{ "scope": "Verbatim complete source files containing copyright/licensing indicators. Includes inactive package sources conservatively; source comments are not rewritten or treated as new grants.", "sources": [ { "package": "adler32@1.2.0", "sourcePath": "src/lib.rs", "sha256": "4f203fd48b12052f950213249a55db0b1c6cde93fe3bdf26b70a6eb42c6c9dee", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "//! A minimal implementation of Adler32 for Rust.\n//!\n//! This provides the simple method adler32(), that exhausts a Read and\n//! computes the Adler32 hash, as well as the RollingAdler32 struct, that can\n//! build a hash byte-by-byte, allowing to 'forget' past bytes in a rolling\n//! fashion.\n//!\n//! The adler32 code has been translated (as accurately as I could manage) from\n//! the zlib implementation.\n\n#![forbid(unsafe_code)]\n#![cfg_attr(not(feature = \"std\"), no_std)]\n\n\n// adler32 algorithm and implementation taken from zlib; http://www.zlib.net/\n// It was translated into Rust as accurately as I could manage\n// The (slow) reference was taken from Wikipedia; https://en.wikipedia.org/\n\n/* zlib.h -- interface of the 'zlib' general purpose compression library\n version 1.2.8, April 28th, 2013\n\n Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler\n\n This software is provided 'as-is', without any express or implied\n warranty. In no event will the authors be held liable for any damages\n arising from the use of this software.\n\n Permission is granted to anyone to use this software for any purpose,\n including commercial applications, and to alter it and redistribute it\n freely, subject to the following restrictions:\n\n 1. The origin of this software must not be misrepresented; you must not\n claim that you wrote the original software. If you use this software\n in a product, an acknowledgment in the product documentation would be\n appreciated but is not required.\n 2. Altered source versions must be plainly marked as such, and must not be\n misrepresented as being the original software.\n 3. This notice may not be removed or altered from any source distribution.\n\n Jean-loup Gailly Mark Adler\n jloup@gzip.org madler@alumni.caltech.edu\n\n*/\n\n// largest prime smaller than 65536\nconst BASE: u32 = 65521;\n\n// NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1\nconst NMAX: usize = 5552;\n\n#[inline(always)]\nfn do1(adler: &mut u32, sum2: &mut u32, buf: &[u8]) {\n *adler += u32::from(buf[0]);\n *sum2 += *adler;\n}\n\n#[inline(always)]\nfn do2(adler: &mut u32, sum2: &mut u32, buf: &[u8]) {\n do1(adler, sum2, &buf[0..1]);\n do1(adler, sum2, &buf[1..2]);\n}\n\n#[inline(always)]\nfn do4(adler: &mut u32, sum2: &mut u32, buf: &[u8]) {\n do2(adler, sum2, &buf[0..2]);\n do2(adler, sum2, &buf[2..4]);\n}\n\n#[inline(always)]\nfn do8(adler: &mut u32, sum2: &mut u32, buf: &[u8]) {\n do4(adler, sum2, &buf[0..4]);\n do4(adler, sum2, &buf[4..8]);\n}\n\n#[inline(always)]\nfn do16(adler: &mut u32, sum2: &mut u32, buf: &[u8]) {\n do8(adler, sum2, &buf[0..8]);\n do8(adler, sum2, &buf[8..16]);\n}\n\n/// A rolling version of the Adler32 hash, which can 'forget' past bytes.\n///\n/// Calling remove() will update the hash to the value it would have if that\n/// past byte had never been fed to the algorithm. This allows you to get the\n/// hash of a rolling window very efficiently.\n#[derive(Clone)]\npub struct RollingAdler32 {\n a: u32,\n b: u32,\n}\n\nimpl Default for RollingAdler32 {\n fn default() -> RollingAdler32 {\n RollingAdler32::new()\n }\n}\n\nimpl RollingAdler32 {\n /// Creates an empty Adler32 context (with hash 1).\n pub fn new() -> RollingAdler32 {\n Self::from_value(1)\n }\n\n /// Creates an Adler32 context with the given initial value.\n pub fn from_value(adler32: u32) -> RollingAdler32 {\n let a = adler32 & 0xFFFF;\n let b = adler32 >> 16;\n RollingAdler32 { a, b }\n }\n\n /// Convenience function initializing a context from the hash of a buffer.\n pub fn from_buffer(buffer: &[u8]) -> RollingAdler32 {\n let mut hash = RollingAdler32::new();\n hash.update_buffer(buffer);\n hash\n }\n\n /// Returns the current hash.\n pub fn hash(&self) -> u32 {\n (self.b << 16) | self.a\n }\n\n /// Removes the given `byte` that was fed to the algorithm `size` bytes ago.\n pub fn remove(&mut self, size: usize, byte: u8) {\n let byte = u32::from(byte);\n self.a = (self.a + BASE - byte) % BASE;\n self.b = ((self.b + BASE - 1)\n .wrapping_add(BASE.wrapping_sub(size as u32).wrapping_mul(byte)))\n % BASE;\n }\n\n /// Feeds a new `byte` to the algorithm to update the hash.\n pub fn update(&mut self, byte: u8) {\n let byte = u32::from(byte);\n self.a = (self.a + byte) % BASE;\n self.b = (self.b + self.a) % BASE;\n }\n\n /// Feeds a vector of bytes to the algorithm to update the hash.\n pub fn update_buffer(&mut self, buffer: &[u8]) {\n let len = buffer.len();\n\n // in case user likes doing a byte at a time, keep it fast\n if len == 1 {\n self.update(buffer[0]);\n return;\n }\n\n // in case short lengths are provided, keep it somewhat fast\n if len < 16 {\n for byte in buffer.iter().take(len) {\n self.a += u32::from(*byte);\n self.b += self.a;\n }\n if self.a >= BASE {\n self.a -= BASE;\n }\n self.b %= BASE;\n return;\n }\n\n let mut pos = 0;\n\n // do length NMAX blocks -- requires just one modulo operation;\n while pos + NMAX <= len {\n let end = pos + NMAX;\n while pos < end {\n // 16 sums unrolled\n do16(&mut self.a, &mut self.b, &buffer[pos..pos + 16]);\n pos += 16;\n }\n self.a %= BASE;\n self.b %= BASE;\n }\n\n // do remaining bytes (less than NMAX, still just one modulo)\n if pos < len {\n // avoid modulos if none remaining\n while len - pos >= 16 {\n do16(&mut self.a, &mut self.b, &buffer[pos..pos + 16]);\n pos += 16;\n }\n while len - pos > 0 {\n self.a += u32::from(buffer[pos]);\n self.b += self.a;\n pos += 1;\n }\n self.a %= BASE;\n self.b %= BASE;\n }\n }\n}\n\n/// Consume a Read object and returns the Adler32 hash.\n#[cfg(feature = \"std\")]\npub fn adler32(mut reader: R) -> std::io::Result {\n let mut hash = RollingAdler32::new();\n let mut buffer = [0u8; NMAX];\n let mut read = reader.read(&mut buffer)?;\n while read > 0 {\n hash.update_buffer(&buffer[..read]);\n read = reader.read(&mut buffer)?;\n }\n Ok(hash.hash())\n}\n\n#[cfg(test)]\nmod test {\n use rand::Rng;\n use std::io;\n #[cfg(target_arch = \"wasm32\")]\n use wasm_bindgen_test::wasm_bindgen_test;\n\n use super::{adler32, RollingAdler32, BASE};\n\n fn adler32_slow(reader: R) -> io::Result {\n let mut a: u32 = 1;\n let mut b: u32 = 0;\n\n for byte in reader.bytes() {\n let byte = byte? as u32;\n a = (a + byte) % BASE;\n b = (b + a) % BASE;\n }\n\n Ok((b << 16) | a)\n }\n\n #[test]\n #[cfg_attr(target_arch = \"wasm32\", wasm_bindgen_test)]\n fn testvectors() {\n fn do_test(v: u32, bytes: &[u8]) {\n let mut hash = RollingAdler32::new();\n hash.update_buffer(&bytes);\n assert_eq!(hash.hash(), v);\n\n let r = io::Cursor::new(bytes);\n assert_eq!(adler32(r).unwrap(), v);\n }\n do_test(0x00000001, b\"\");\n do_test(0x00620062, b\"a\");\n do_test(0x024d0127, b\"abc\");\n do_test(0x29750586, b\"message digest\");\n do_test(0x90860b20, b\"abcdefghijklmnopqrstuvwxyz\");\n do_test(\n 0x8adb150c,\n b\"ABCDEFGHIJKLMNOPQRSTUVWXYZ\\\n abcdefghijklmnopqrstuvwxyz\\\n 0123456789\",\n );\n do_test(\n 0x97b61069,\n b\"1234567890123456789012345678901234567890\\\n 1234567890123456789012345678901234567890\",\n );\n do_test(0xD6251498, &[255; 64000]);\n }\n\n #[test]\n #[cfg_attr(target_arch = \"wasm32\", wasm_bindgen_test)]\n fn compare() {\n let mut rng = rand::thread_rng();\n let mut data = vec![0u8; 5589];\n for size in [\n 0, 1, 3, 4, 5, 31, 32, 33, 67, 5550, 5552, 5553, 5568, 5584, 5589,\n ]\n .iter()\n .cloned()\n {\n rng.fill(&mut data[..size]);\n let r1 = io::Cursor::new(&data[..size]);\n let r2 = r1.clone();\n if adler32_slow(r1).unwrap() != adler32(r2).unwrap() {\n panic!(\"Comparison failed, size={}\", size);\n }\n }\n }\n\n #[test]\n #[cfg_attr(target_arch = \"wasm32\", wasm_bindgen_test)]\n fn rolling() {\n assert_eq!(RollingAdler32::from_value(0x01020304).hash(), 0x01020304);\n\n fn do_test(a: &[u8], b: &[u8]) {\n let mut total = Vec::with_capacity(a.len() + b.len());\n total.extend(a);\n total.extend(b);\n let mut h = RollingAdler32::from_buffer(&total[..(b.len())]);\n for i in 0..(a.len()) {\n h.remove(b.len(), a[i]);\n h.update(total[b.len() + i]);\n }\n assert_eq!(h.hash(), adler32(b).unwrap());\n }\n do_test(b\"a\", b\"b\");\n do_test(b\"\", b\"this a test\");\n do_test(b\"th\", b\"is a test\");\n do_test(b\"this a \", b\"test\");\n }\n\n #[test]\n #[cfg_attr(target_arch = \"wasm32\", wasm_bindgen_test)]\n fn long_window_remove() {\n let mut hash = RollingAdler32::new();\n let w = 65536;\n assert!(w as u32 > BASE);\n\n let mut bytes = vec![0; w * 3];\n for (i, b) in bytes.iter_mut().enumerate() {\n *b = i as u8;\n }\n\n for (i, b) in bytes.iter().enumerate() {\n if i >= w {\n hash.remove(w, bytes[i - w]);\n }\n hash.update(*b);\n if i > 0 && i % w == 0 {\n assert_eq!(hash.hash(), 0x433a8772);\n }\n }\n assert_eq!(hash.hash(), 0xbbba8772);\n }\n}\n" }, { "package": "aho-corasick@1.1.5", "sourcePath": "README.md", "sha256": "afc4d559a98cf190029af0bf320fc0022725e349cd2a303aac860254e28f3c53", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "aho-corasick\n============\nA library for finding occurrences of many patterns at once with SIMD\nacceleration in some cases. This library provides multiple pattern\nsearch principally through an implementation of the\n[Aho-Corasick algorithm](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm),\nwhich builds a finite state machine for executing searches in linear time.\nFeatures include case insensitive matching, overlapping matches, fast searching\nvia SIMD and optional full DFA construction and search & replace in streams.\n\n[![Build status](https://github.com/BurntSushi/aho-corasick/workflows/ci/badge.svg)](https://github.com/BurntSushi/aho-corasick/actions)\n[![crates.io](https://img.shields.io/crates/v/aho-corasick.svg)](https://crates.io/crates/aho-corasick)\n\nDual-licensed under MIT or the [UNLICENSE](https://unlicense.org/).\n\n\n### Documentation\n\nhttps://docs.rs/aho-corasick\n\n\n### Usage\n\nRun `cargo add aho-corasick` to automatically add this crate as a dependency\nin your `Cargo.toml` file.\n\n\n### Example: basic searching\n\nThis example shows how to search for occurrences of multiple patterns\nsimultaneously. Each match includes the pattern that matched along with the\nbyte offsets of the match.\n\n```rust\nuse aho_corasick::{AhoCorasick, PatternID};\n\nlet patterns = &[\"apple\", \"maple\", \"Snapple\"];\nlet haystack = \"Nobody likes maple in their apple flavored Snapple.\";\n\nlet ac = AhoCorasick::new(patterns).unwrap();\nlet mut matches = vec![];\nfor mat in ac.find_iter(haystack) {\n matches.push((mat.pattern(), mat.start(), mat.end()));\n}\nassert_eq!(matches, vec![\n (PatternID::must(1), 13, 18),\n (PatternID::must(0), 28, 33),\n (PatternID::must(2), 43, 50),\n]);\n```\n\n\n### Example: ASCII case insensitivity\n\nThis is like the previous example, but matches `Snapple` case insensitively\nusing `AhoCorasickBuilder`:\n\n```rust\nuse aho_corasick::{AhoCorasick, PatternID};\n\nlet patterns = &[\"apple\", \"maple\", \"snapple\"];\nlet haystack = \"Nobody likes maple in their apple flavored Snapple.\";\n\nlet ac = AhoCorasick::builder()\n .ascii_case_insensitive(true)\n .build(patterns)\n .unwrap();\nlet mut matches = vec![];\nfor mat in ac.find_iter(haystack) {\n matches.push((mat.pattern(), mat.start(), mat.end()));\n}\nassert_eq!(matches, vec![\n (PatternID::must(1), 13, 18),\n (PatternID::must(0), 28, 33),\n (PatternID::must(2), 43, 50),\n]);\n```\n\n\n### Example: replacing matches in a stream\n\nThis example shows how to execute a search and replace on a stream without\nloading the entire stream into memory first.\n\n```rust,ignore\nuse aho_corasick::AhoCorasick;\n\nlet patterns = &[\"fox\", \"brown\", \"quick\"];\nlet replace_with = &[\"sloth\", \"grey\", \"slow\"];\n\n// In a real example, these might be `std::fs::File`s instead. All you need to\n// do is supply a pair of `std::io::Read` and `std::io::Write` implementations.\nlet rdr = \"The quick brown fox.\";\nlet mut wtr = vec![];\n\nlet ac = AhoCorasick::new(patterns).unwrap();\nac.stream_replace_all(rdr.as_bytes(), &mut wtr, replace_with)\n .expect(\"stream_replace_all failed\");\nassert_eq!(b\"The slow grey sloth.\".to_vec(), wtr);\n```\n\n\n### Example: finding the leftmost first match\n\nIn the textbook description of Aho-Corasick, its formulation is typically\nstructured such that it reports all possible matches, even when they overlap\nwith another. In many cases, overlapping matches may not be desired, such as\nthe case of finding all successive non-overlapping matches like you might with\na standard regular expression.\n\nUnfortunately the \"obvious\" way to modify the Aho-Corasick algorithm to do\nthis doesn't always work in the expected way, since it will report matches as\nsoon as they are seen. For example, consider matching the regex `Samwise|Sam`\nagainst the text `Samwise`. Most regex engines (that are Perl-like, or\nnon-POSIX) will report `Samwise` as a match, but the standard Aho-Corasick\nalgorithm modified for reporting non-overlapping matches will report `Sam`.\n\nA novel contribution of this library is the ability to change the match\nsemantics of Aho-Corasick (without additional search time overhead) such that\n`Samwise` is reported instead. For example, here's the standard approach:\n\n```rust\nuse aho_corasick::AhoCorasick;\n\nlet patterns = &[\"Samwise\", \"Sam\"];\nlet haystack = \"Samwise\";\n\nlet ac = AhoCorasick::new(patterns).unwrap();\nlet mat = ac.find(haystack).expect(\"should have a match\");\nassert_eq!(\"Sam\", &haystack[mat.start()..mat.end()]);\n```\n\nAnd now here's the leftmost-first version, which matches how a Perl-like\nregex will work:\n\n```rust\nuse aho_corasick::{AhoCorasick, MatchKind};\n\nlet patterns = &[\"Samwise\", \"Sam\"];\nlet haystack = \"Samwise\";\n\nlet ac = AhoCorasick::builder()\n .match_kind(MatchKind::LeftmostFirst)\n .build(patterns)\n .unwrap();\nlet mat = ac.find(haystack).expect(\"should have a match\");\nassert_eq!(\"Samwise\", &haystack[mat.start()..mat.end()]);\n```\n\nIn addition to leftmost-first semantics, this library also supports\nleftmost-longest semantics, which match the POSIX behavior of a regular\nexpression alternation. See `MatchKind` in the docs for more details.\n\n\n### Minimum Rust version policy\n\nThis crate's minimum supported `rustc` version is `1.60.0`.\n\nThe current policy is that the minimum Rust version required to use this crate\ncan be increased in minor version updates. For example, if `crate 1.0` requires\nRust 1.20.0, then `crate 1.0.z` for all values of `z` will also require Rust\n1.20.0 or newer. However, `crate 1.y` for `y > 0` may require a newer minimum\nversion of Rust.\n\nIn general, this crate will be conservative with respect to the minimum\nsupported version of Rust.\n\n\n### FFI bindings\n\n* [G-Research/ahocorasick_rs](https://github.com/G-Research/ahocorasick_rs/)\nis a Python wrapper for this library.\n* [tmikus/ahocorasick_rs](https://github.com/tmikus/ahocorasick_rs) is a Go\n wrapper for this library.\n" }, { "package": "anstream@1.0.0", "sourcePath": "README.md", "sha256": "7e90b134ad15dc4368d2a27efeb8db9d155dcd0886b72a032ffb367616255807", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# anstream\n\n> A simple cross platform library for writing colored text to a terminal.\n\n*A portmanteau of \"ansi stream\"*\n\n[![Documentation](https://img.shields.io/badge/docs-master-blue.svg)][Documentation]\n![License](https://img.shields.io/crates/l/anstream.svg)\n[![Crates Status](https://img.shields.io/crates/v/anstream.svg)](https://crates.io/crates/anstream)\n\nSpecialized `stdout` and `stderr` that accept ANSI escape codes and adapt them\nbased on the terminal's capabilities.\n\n`anstream::adapter::strip_str` may also be of interest on its own for low\noverhead stripping of ANSI escape codes.\n\n## License\n\nLicensed under either of\n\n* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or )\n* MIT license ([LICENSE-MIT](LICENSE-MIT) or )\n\nat your option.\n\n## [Contribute](../../CONTRIBUTING.md)\n\nUnless you explicitly state otherwise, any contribution intentionally\nsubmitted for inclusion in the work by you, as defined in the Apache-2.0\nlicense, shall be dual-licensed as above, without any additional terms or\nconditions.\n\n[Crates.io]: https://crates.io/crates/anstream\n[Documentation]: https://docs.rs/anstream\n" }, { "package": "anstyle@1.0.14", "sourcePath": "README.md", "sha256": "015245c48ad8fb28bc4b2ed85a992a6e1182aa7fd633ecaac8f69b229d2aacab", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# anstyle\n\n> ANSI text styling\n\n*A portmanteau of \"ansi style\"*\n\n[![Documentation](https://img.shields.io/badge/docs-master-blue.svg)][Documentation]\n![License](https://img.shields.io/crates/l/anstyle.svg)\n[![Crates Status](https://img.shields.io/crates/v/anstyle.svg)](https://crates.io/crates/anstyle)\n\n`anstyle` provides core types describing [ANSI styling escape\ncodes](https://en.wikipedia.org/wiki/ANSI_escape_code) for interoperability\nbetween crates. For example, this would allow a crate to provide an API for\ncustomizing the colors used without putting the underlying text styling crate\nin the API.\n\nFor integration with your text styling crate, see:\n- [anstyle-termcolor](crates/termcolor)\n- [anstyle-owo-colors](crates/owo)\n- [anstyle-yansi](crates/yansi)\n\nGeneral utilities:\n- [anstyle-git](crates/git): Parse Git style descriptions\n\n## License\n\nLicensed under either of\n\n* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or )\n* MIT license ([LICENSE-MIT](LICENSE-MIT) or )\n\nat your option.\n\n## [Contribute](../../CONTRIBUTING.md)\n\nUnless you explicitly state otherwise, any contribution intentionally\nsubmitted for inclusion in the work by you, as defined in the Apache-2.0\nlicense, shall be dual-licensed as above, without any additional terms or\nconditions.\n\n[Crates.io]: https://crates.io/crates/anstyle\n[Documentation]: https://docs.rs/anstyle\n" }, { "package": "anstyle-parse@1.0.0", "sourcePath": "README.md", "sha256": "85c374ff51508a4847ade385c727d5b3d593e374a36e66606543cee390e54963", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# anstyle-parse\n\n> Parse [Parse ANSI Style Escapes](https://vt100.net/emu/dec_ansi_parser)\n\n[![Documentation](https://img.shields.io/badge/docs-master-blue.svg)][Documentation]\n![License](https://img.shields.io/crates/l/anstyle-parse.svg)\n[![Crates Status](https://img.shields.io/crates/v/anstyle-parse.svg)](https://crates.io/crates/anstyle-parse)\n\n## License\n\nLicensed under either of\n\n* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or )\n* MIT license ([LICENSE-MIT](LICENSE-MIT) or )\n\nat your option.\n\n## [Contribute](../../CONTRIBUTING.md)\n\nUnless you explicitly state otherwise, any contribution intentionally\nsubmitted for inclusion in the work by you, as defined in the Apache-2.0\nlicense, shall be dual-licensed as above, without any additional terms or\nconditions.\n\n### Special Thanks\n\n[chrisduerr](https://github.com/alacritty/vte/commits?author=chrisduerr) and the\n[alacritty project](https://github.com/alacritty/alacritty) for\n[vte](https://crates.io/crates/vte) which\n[this was forked from](https://github.com/alacritty/vte/issues/82)\n\n[Crates.io]: https://crates.io/crates/anstyle-parse\n[Documentation]: https://docs.rs/anstyle-parse\n" }, { "package": "anstyle-query@1.1.5", "sourcePath": "README.md", "sha256": "9ed87a71a298ed4925210e66dd06c69f40600c1df97fd082d1763c2220c9f2f1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# anstyle-query\n\n> **Low level terminal capability lookups**\n\n[![Documentation](https://img.shields.io/badge/docs-master-blue.svg)][Documentation]\n![License](https://img.shields.io/crates/l/anstyle-query.svg)\n[![Crates Status](https://img.shields.io/crates/v/anstyle-query.svg)](https://crates.io/crates/anstyle-query)\n\n## License\n\nLicensed under either of\n\n* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or )\n* MIT license ([LICENSE-MIT](LICENSE-MIT) or )\n\nat your option.\n\n## [Contribute](../../CONTRIBUTING.md)\n\nUnless you explicitly state otherwise, any contribution intentionally\nsubmitted for inclusion in the work by you, as defined in the Apache-2.0\nlicense, shall be dual-licensed as above, without any additional terms or\nconditions.\n\n[Documentation]: https://docs.rs/anstyle-query\n" }, { "package": "anstyle-wincon@3.0.11", "sourcePath": "README.md", "sha256": "b979881246703593aad26da21f5d467c03c35142d6f5965f7e7253cb89695c9f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# anstyle-wincon\n\n> Styling legacy Windows terminals\n\n[![Documentation](https://img.shields.io/badge/docs-master-blue.svg)][Documentation]\n![License](https://img.shields.io/crates/l/anstyle-wincon.svg)\n[![Crates Status](https://img.shields.io/crates/v/anstyle-wincon.svg)](https://crates.io/crates/anstyle-wincon)\n\n## License\n\nLicensed under either of\n\n* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or )\n* MIT license ([LICENSE-MIT](LICENSE-MIT) or )\n\nat your option.\n\n## [Contribute](../../CONTRIBUTING.md)\n\nUnless you explicitly state otherwise, any contribution intentionally\nsubmitted for inclusion in the work by you, as defined in the Apache-2.0\nlicense, shall be dual-licensed as above, without any additional terms or\nconditions.\n\n### Special Thanks\n\n[burntsushi](https://github.com/burntsushi) for [termcolor](https://github.com/burntsushi/termcolor) as I don't know Windows either :)\n\n[Crates.io]: https://crates.io/crates/anstyle-wincon\n[Documentation]: https://docs.rs/anstyle-wincon\n" }, { "package": "arrayvec@0.5.2", "sourcePath": "README.md", "sha256": "f42eb73acbc7729825a836a532d1c8c6b71e006e0d87e549ea7a483da7472425", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "\narrayvec\n========\n[![Crates.io: arrayvec](https://img.shields.io/crates/v/arrayvec.svg)](https://crates.io/crates/arrayvec)\n[![Crates.io: nodrop](https://img.shields.io/crates/v/nodrop.svg)](https://crates.io/crates/nodrop)\n[![Documentation](https://docs.rs/arrayvec/badge.svg)](https://docs.rs/arrayvec)\n[![Build Status](https://travis-ci.org/bluss/arrayvec.svg?branch=master)](https://travis-ci.org/bluss/arrayvec)\n[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-red.svg)](LICENSE-APACHE)\nOR\n[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)\n\nA vector with fixed capacity.\n\nPlease read the [`API documentation here`](https://docs.rs/arrayvec)\n\n# License\n\nDual-licensed to be compatible with the Rust project.\n\nLicensed under the Apache License, Version 2.0\nhttp://www.apache.org/licenses/LICENSE-2.0 or the MIT license\nhttp://opensource.org/licenses/MIT, at your\noption. This file may not be copied, modified, or distributed\nexcept according to those terms.\n" }, { "package": "arrayvec@0.5.2", "sourcePath": "src/char.rs", "sha256": "3fe9e9cc68fc7cedb238d53924b552b876a60c4fea85935d2f5d1ca0d41ffa3e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n//\n// Original authors: alexchrichton, bluss\n\nuse std::ptr;\n\n// UTF-8 ranges and tags for encoding characters\nconst TAG_CONT: u8 = 0b1000_0000;\nconst TAG_TWO_B: u8 = 0b1100_0000;\nconst TAG_THREE_B: u8 = 0b1110_0000;\nconst TAG_FOUR_B: u8 = 0b1111_0000;\nconst MAX_ONE_B: u32 = 0x80;\nconst MAX_TWO_B: u32 = 0x800;\nconst MAX_THREE_B: u32 = 0x10000;\n\n/// Placeholder\npub struct EncodeUtf8Error;\n\n#[inline]\nunsafe fn write(ptr: *mut u8, index: usize, byte: u8) {\n ptr::write(ptr.add(index), byte)\n}\n\n/// Encode a char into buf using UTF-8.\n///\n/// On success, return the byte length of the encoding (1, 2, 3 or 4).
\n/// On error, return `EncodeUtf8Error` if the buffer was too short for the char.\n///\n/// Safety: `ptr` must be writable for `len` bytes.\n#[inline]\npub unsafe fn encode_utf8(ch: char, ptr: *mut u8, len: usize) -> Result\n{\n let code = ch as u32;\n if code < MAX_ONE_B && len >= 1 {\n write(ptr, 0, code as u8);\n return Ok(1);\n } else if code < MAX_TWO_B && len >= 2 {\n write(ptr, 0, (code >> 6 & 0x1F) as u8 | TAG_TWO_B);\n write(ptr, 1, (code & 0x3F) as u8 | TAG_CONT);\n return Ok(2);\n } else if code < MAX_THREE_B && len >= 3 {\n write(ptr, 0, (code >> 12 & 0x0F) as u8 | TAG_THREE_B);\n write(ptr, 1, (code >> 6 & 0x3F) as u8 | TAG_CONT);\n write(ptr, 2, (code & 0x3F) as u8 | TAG_CONT);\n return Ok(3);\n } else if len >= 4 {\n write(ptr, 0, (code >> 18 & 0x07) as u8 | TAG_FOUR_B);\n write(ptr, 1, (code >> 12 & 0x3F) as u8 | TAG_CONT);\n write(ptr, 2, (code >> 6 & 0x3F) as u8 | TAG_CONT);\n write(ptr, 3, (code & 0x3F) as u8 | TAG_CONT);\n return Ok(4);\n };\n Err(EncodeUtf8Error)\n}\n\n\n#[test]\n#[cfg_attr(miri, ignore)] // Miri is too slow\nfn test_encode_utf8() {\n // Test that all codepoints are encoded correctly\n let mut data = [0u8; 16];\n for codepoint in 0..=(std::char::MAX as u32) {\n if let Some(ch) = std::char::from_u32(codepoint) {\n for elt in &mut data { *elt = 0; }\n let ptr = data.as_mut_ptr();\n let len = data.len();\n unsafe {\n let res = encode_utf8(ch, ptr, len).ok().unwrap();\n assert_eq!(res, ch.len_utf8());\n }\n let string = std::str::from_utf8(&data).unwrap();\n assert_eq!(string.chars().next(), Some(ch));\n }\n }\n}\n\n#[test]\nfn test_encode_utf8_oob() {\n // test that we report oob if the buffer is too short\n let mut data = [0u8; 16];\n let chars = ['a', 'α', '�', '𐍈'];\n for (len, &ch) in (1..=4).zip(&chars) {\n assert_eq!(len, ch.len_utf8(), \"Len of ch={}\", ch);\n let ptr = data.as_mut_ptr();\n unsafe {\n assert!(matches::matches!(encode_utf8(ch, ptr, len - 1), Err(_)));\n assert!(matches::matches!(encode_utf8(ch, ptr, len), Ok(_)));\n }\n }\n}\n\n" }, { "package": "arrayvec@0.7.8", "sourcePath": "README.md", "sha256": "2264c34c62ea4c617d72047b00749b4786dfb9dff2fac24e0320170ee0cd19c8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "\narrayvec\n========\n[![Crates.io: arrayvec](https://img.shields.io/crates/v/arrayvec.svg)](https://crates.io/crates/arrayvec)\n[![Documentation](https://docs.rs/arrayvec/badge.svg)](https://docs.rs/arrayvec)\n[![Build Status](https://github.com/bluss/arrayvec/workflows/Continuous%20integration/badge.svg?branch=master)](https://github.com/bluss/arrayvec/actions)\n\n\n\n\n[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-red.svg)](LICENSE-APACHE)\nOR\n[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)\n\nA vector with fixed capacity.\n\nPlease read the [`API documentation here`](https://docs.rs/arrayvec)\n\n# License\n\nDual-licensed to be compatible with the Rust project.\n\nLicensed under the Apache License, Version 2.0\nhttp://www.apache.org/licenses/LICENSE-2.0 or the MIT license\nhttp://opensource.org/licenses/MIT, at your\noption. This file may not be copied, modified, or distributed\nexcept according to those terms.\n" }, { "package": "arrayvec@0.7.8", "sourcePath": "src/char.rs", "sha256": "1de50e1d6045af2b3496426492315ba774986f9bc8301ffa391de861a08cc9cb", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n//\n// Original authors: alexchrichton, bluss\n\n// UTF-8 ranges and tags for encoding characters\nconst TAG_CONT: u8 = 0b1000_0000;\nconst TAG_TWO_B: u8 = 0b1100_0000;\nconst TAG_THREE_B: u8 = 0b1110_0000;\nconst TAG_FOUR_B: u8 = 0b1111_0000;\nconst MAX_ONE_B: u32 = 0x80;\nconst MAX_TWO_B: u32 = 0x800;\nconst MAX_THREE_B: u32 = 0x10000;\n\n/// Placeholder\npub struct EncodeUtf8Error;\n\n/// Encode a char into buf using UTF-8.\n///\n/// On success, return the byte length of the encoding (1, 2, 3 or 4).
\n/// On error, return `EncodeUtf8Error` if the buffer was too short for the char.\n///\n/// Safety: `ptr` must be writable for `len` bytes.\n#[inline]\npub unsafe fn encode_utf8(ch: char, ptr: *mut u8, len: usize) -> Result\n{\n let code = ch as u32;\n if code < MAX_ONE_B && len >= 1 {\n ptr.add(0).write(code as u8);\n return Ok(1);\n } else if code < MAX_TWO_B && len >= 2 {\n ptr.add(0).write((code >> 6 & 0x1F) as u8 | TAG_TWO_B);\n ptr.add(1).write((code & 0x3F) as u8 | TAG_CONT);\n return Ok(2);\n } else if code < MAX_THREE_B && len >= 3 {\n ptr.add(0).write((code >> 12 & 0x0F) as u8 | TAG_THREE_B);\n ptr.add(1).write((code >> 6 & 0x3F) as u8 | TAG_CONT);\n ptr.add(2).write((code & 0x3F) as u8 | TAG_CONT);\n return Ok(3);\n } else if len >= 4 {\n ptr.add(0).write((code >> 18 & 0x07) as u8 | TAG_FOUR_B);\n ptr.add(1).write((code >> 12 & 0x3F) as u8 | TAG_CONT);\n ptr.add(2).write((code >> 6 & 0x3F) as u8 | TAG_CONT);\n ptr.add(3).write((code & 0x3F) as u8 | TAG_CONT);\n return Ok(4);\n };\n Err(EncodeUtf8Error)\n}\n\n\n#[test]\n#[cfg_attr(miri, ignore)] // Miri is too slow\nfn test_encode_utf8() {\n // Test that all codepoints are encoded correctly\n let mut data = [0u8; 16];\n for codepoint in 0..=(std::char::MAX as u32) {\n if let Some(ch) = std::char::from_u32(codepoint) {\n for elt in &mut data { *elt = 0; }\n let ptr = data.as_mut_ptr();\n let len = data.len();\n unsafe {\n let res = encode_utf8(ch, ptr, len).ok().unwrap();\n assert_eq!(res, ch.len_utf8());\n }\n let string = std::str::from_utf8(&data).unwrap();\n assert_eq!(string.chars().next(), Some(ch));\n }\n }\n}\n\n#[test]\nfn test_encode_utf8_oob() {\n // test that we report oob if the buffer is too short\n let mut data = [0u8; 16];\n let chars = ['a', 'α', '�', '𐍈'];\n for (len, &ch) in (1..=4).zip(&chars) {\n assert_eq!(len, ch.len_utf8(), \"Len of ch={}\", ch);\n let ptr = data.as_mut_ptr();\n unsafe {\n assert!(matches::matches!(encode_utf8(ch, ptr, len - 1), Err(_)));\n assert!(matches::matches!(encode_utf8(ch, ptr, len), Ok(_)));\n }\n }\n}\n\n" }, { "package": "base64@0.21.7", "sourcePath": "README.md", "sha256": "df01f5b4317d601e7de86743f9818aec9196abf9e298f5e47679b7a966ecd945", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# [base64](https://crates.io/crates/base64)\n\n[![](https://img.shields.io/crates/v/base64.svg)](https://crates.io/crates/base64) [![Docs](https://docs.rs/base64/badge.svg)](https://docs.rs/base64) [![CircleCI](https://circleci.com/gh/marshallpierce/rust-base64/tree/master.svg?style=shield)](https://circleci.com/gh/marshallpierce/rust-base64/tree/master) [![codecov](https://codecov.io/gh/marshallpierce/rust-base64/branch/master/graph/badge.svg)](https://codecov.io/gh/marshallpierce/rust-base64) [![unsafe forbidden](https://img.shields.io/badge/unsafe-forbidden-success.svg)](https://github.com/rust-secure-code/safety-dance/)\n\n\n\nMade with CLion. Thanks to JetBrains for supporting open source!\n\nIt's base64. What more could anyone want?\n\nThis library's goals are to be *correct* and *fast*. It's thoroughly tested and widely used. It exposes functionality at\nmultiple levels of abstraction so you can choose the level of convenience vs performance that you want,\ne.g. `decode_engine_slice` decodes into an existing `&mut [u8]` and is pretty fast (2.6GiB/s for a 3 KiB input),\nwhereas `decode_engine` allocates a new `Vec` and returns it, which might be more convenient in some cases, but is\nslower (although still fast enough for almost any purpose) at 2.1 GiB/s.\n\nSee the [docs](https://docs.rs/base64) for all the details.\n\n## FAQ\n\n### I need to decode base64 with whitespace/null bytes/other random things interspersed in it. What should I do?\n\nRemove non-base64 characters from your input before decoding.\n\nIf you have a `Vec` of base64, [retain](https://doc.rust-lang.org/std/vec/struct.Vec.html#method.retain) can be used to\nstrip out whatever you need removed.\n\nIf you have a `Read` (e.g. reading a file or network socket), there are various approaches.\n\n- Use [iter_read](https://crates.io/crates/iter-read) together with `Read`'s `bytes()` to filter out unwanted bytes.\n- Implement `Read` with a `read()` impl that delegates to your actual `Read`, and then drops any bytes you don't want.\n\n### I need to line-wrap base64, e.g. for MIME/PEM.\n\n[line-wrap](https://crates.io/crates/line-wrap) does just that.\n\n### I want canonical base64 encoding/decoding.\n\nFirst, don't do this. You should no more expect Base64 to be canonical than you should expect compression algorithms to\nproduce canonical output across all usage in the wild (hint: they don't).\nHowever, [people are drawn to their own destruction like moths to a flame](https://eprint.iacr.org/2022/361), so here we\nare.\n\nThere are two opportunities for non-canonical encoding (and thus, detection of the same during decoding): the final bits\nof the last encoded token in two or three token suffixes, and the `=` token used to inflate the suffix to a full four\ntokens.\n\nThe trailing bits issue is unavoidable: with 6 bits available in each encoded token, 1 input byte takes 2 tokens,\nwith the second one having some bits unused. Same for two input bytes: 16 bits, but 3 tokens have 18 bits. Unless we\ndecide to stop shipping whole bytes around, we're stuck with those extra bits that a sneaky or buggy encoder might set\nto 1 instead of 0.\n\nThe `=` pad bytes, on the other hand, are entirely a self-own by the Base64 standard. They do not affect decoding other\nthan to provide an opportunity to say \"that padding is incorrect\". Exabytes of storage and transfer have no doubt been\nwasted on pointless `=` bytes. Somehow we all seem to be quite comfortable with, say, hex-encoded data just stopping\nwhen it's done rather than requiring a confirmation that the author of the encoder could count to four. Anyway, there\nare two ways to make pad bytes predictable: require canonical padding to the next multiple of four bytes as per the RFC,\nor, if you control all producers and consumers, save a few bytes by requiring no padding (especially applicable to the\nurl-safe alphabet).\n\nAll `Engine` implementations must at a minimum support treating non-canonical padding of both types as an error, and\noptionally may allow other behaviors.\n\n## Rust version compatibility\n\nThe minimum supported Rust version is 1.48.0.\n\n# Contributing\n\nContributions are very welcome. However, because this library is used widely, and in security-sensitive contexts, all\nPRs will be carefully scrutinized. Beyond that, this sort of low level library simply needs to be 100% correct. Nobody\nwants to chase bugs in encoding of any sort.\n\nAll this means that it takes me a fair amount of time to review each PR, so it might take quite a while to carve out the\nfree time to give each PR the attention it deserves. I will get to everyone eventually!\n\n## Developing\n\nBenchmarks are in `benches/`.\n\n```bash\ncargo bench\n```\n\n## no_std\n\nThis crate supports no_std. By default the crate targets std via the `std` feature. You can deactivate\nthe `default-features` to target `core` instead. In that case you lose out on all the functionality revolving\naround `std::io`, `std::error::Error`, and heap allocations. There is an additional `alloc` feature that you can activate\nto bring back the support for heap allocations.\n\n## Profiling\n\nOn Linux, you can use [perf](https://perf.wiki.kernel.org/index.php/Main_Page) for profiling. Then compile the\nbenchmarks with `cargo bench --no-run`.\n\nRun the benchmark binary with `perf` (shown here filtering to one particular benchmark, which will make the results\neasier to read). `perf` is only available to the root user on most systems as it fiddles with event counters in your\nCPU, so use `sudo`. We need to run the actual benchmark binary, hence the path into `target`. You can see the actual\nfull path with `cargo bench -v`; it will print out the commands it runs. If you use the exact path\nthat `bench` outputs, make sure you get the one that's for the benchmarks, not the tests. You may also want\nto `cargo clean` so you have only one `benchmarks-` binary (they tend to accumulate).\n\n```bash\nsudo perf record target/release/deps/benchmarks-* --bench decode_10mib_reuse\n```\n\nThen analyze the results, again with perf:\n\n```bash\nsudo perf annotate -l\n```\n\nYou'll see a bunch of interleaved rust source and assembly like this. The section with `lib.rs:327` is telling us that\n4.02% of samples saw the `movzbl` aka bit shift as the active instruction. However, this percentage is not as exact as\nit seems due to a phenomenon called *skid*. Basically, a consequence of how fancy modern CPUs are is that this sort of\ninstruction profiling is inherently inaccurate, especially in branch-heavy code.\n\n```text\n lib.rs:322 0.70 : 10698: mov %rdi,%rax\n 2.82 : 1069b: shr $0x38,%rax\n : if morsel == decode_tables::INVALID_VALUE {\n : bad_byte_index = input_index;\n : break;\n : };\n : accum = (morsel as u64) << 58;\n lib.rs:327 4.02 : 1069f: movzbl (%r9,%rax,1),%r15d\n : // fast loop of 8 bytes at a time\n : while input_index < length_of_full_chunks {\n : let mut accum: u64;\n :\n : let input_chunk = BigEndian::read_u64(&input_bytes[input_index..(input_index + 8)]);\n : morsel = decode_table[(input_chunk >> 56) as usize];\n lib.rs:322 3.68 : 106a4: cmp $0xff,%r15\n : if morsel == decode_tables::INVALID_VALUE {\n 0.00 : 106ab: je 1090e \n```\n\n## Fuzzing\n\nThis uses [cargo-fuzz](https://github.com/rust-fuzz/cargo-fuzz). See `fuzz/fuzzers` for the available fuzzing scripts.\nTo run, use an invocation like these:\n\n```bash\ncargo +nightly fuzz run roundtrip\ncargo +nightly fuzz run roundtrip_no_pad\ncargo +nightly fuzz run roundtrip_random_config -- -max_len=10240\ncargo +nightly fuzz run decode_random\n```\n\n## License\n\nThis project is dual-licensed under MIT and Apache 2.0.\n\n" }, { "package": "bitflags@1.3.2", "sourcePath": "src/lib.rs", "sha256": "e6477688535ee326d27238aeedc9cb4320ac35b9d17a4deda09e0587b0ccdbd4", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! A typesafe bitmask flag generator useful for sets of C-style bitmask flags.\n//! It can be used for creating typesafe wrappers around C APIs.\n//!\n//! The `bitflags!` macro generates `struct`s that manage a set of flags. The\n//! flags should only be defined for integer types, otherwise unexpected type\n//! errors may occur at compile time.\n//!\n//! # Example\n//!\n//! ```\n//! use bitflags::bitflags;\n//!\n//! bitflags! {\n//! struct Flags: u32 {\n//! const A = 0b00000001;\n//! const B = 0b00000010;\n//! const C = 0b00000100;\n//! const ABC = Self::A.bits | Self::B.bits | Self::C.bits;\n//! }\n//! }\n//!\n//! fn main() {\n//! let e1 = Flags::A | Flags::C;\n//! let e2 = Flags::B | Flags::C;\n//! assert_eq!((e1 | e2), Flags::ABC); // union\n//! assert_eq!((e1 & e2), Flags::C); // intersection\n//! assert_eq!((e1 - e2), Flags::A); // set difference\n//! assert_eq!(!e2, Flags::A); // set complement\n//! }\n//! ```\n//!\n//! See [`example_generated::Flags`](./example_generated/struct.Flags.html) for documentation of code\n//! generated by the above `bitflags!` expansion.\n//!\n//! The generated `struct`s can also be extended with type and trait\n//! implementations:\n//!\n//! ```\n//! use std::fmt;\n//!\n//! use bitflags::bitflags;\n//!\n//! bitflags! {\n//! struct Flags: u32 {\n//! const A = 0b00000001;\n//! const B = 0b00000010;\n//! }\n//! }\n//!\n//! impl Flags {\n//! pub fn clear(&mut self) {\n//! self.bits = 0; // The `bits` field can be accessed from within the\n//! // same module where the `bitflags!` macro was invoked.\n//! }\n//! }\n//!\n//! impl fmt::Display for Flags {\n//! fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n//! write!(f, \"hi!\")\n//! }\n//! }\n//!\n//! fn main() {\n//! let mut flags = Flags::A | Flags::B;\n//! flags.clear();\n//! assert!(flags.is_empty());\n//! assert_eq!(format!(\"{}\", flags), \"hi!\");\n//! assert_eq!(format!(\"{:?}\", Flags::A | Flags::B), \"A | B\");\n//! assert_eq!(format!(\"{:?}\", Flags::B), \"B\");\n//! }\n//! ```\n//!\n//! # Visibility\n//!\n//! The generated structs and their associated flag constants are not exported\n//! out of the current module by default. A definition can be exported out of\n//! the current module by adding `pub` before `struct`:\n//!\n//! ```\n//! mod example {\n//! use bitflags::bitflags;\n//!\n//! bitflags! {\n//! pub struct Flags1: u32 {\n//! const A = 0b00000001;\n//! }\n//!\n//! # pub\n//! struct Flags2: u32 {\n//! const B = 0b00000010;\n//! }\n//! }\n//! }\n//!\n//! fn main() {\n//! let flag1 = example::Flags1::A;\n//! let flag2 = example::Flags2::B; // error: const `B` is private\n//! }\n//! ```\n//!\n//! # Attributes\n//!\n//! Attributes can be attached to the generated `struct`s by placing them\n//! before the `struct` keyword.\n//!\n//! ## Representations\n//!\n//! It's valid to add a `#[repr(C)]` or `#[repr(transparent)]` attribute to a type\n//! generated by `bitflags!`. In these cases, the type is guaranteed to be a newtype.\n//!\n//! ```\n//! use bitflags::bitflags;\n//!\n//! bitflags! {\n//! #[repr(transparent)]\n//! struct Flags: u32 {\n//! const A = 0b00000001;\n//! const B = 0b00000010;\n//! const C = 0b00000100;\n//! }\n//! }\n//! ```\n//!\n//! # Trait implementations\n//!\n//! The `Copy`, `Clone`, `PartialEq`, `Eq`, `PartialOrd`, `Ord` and `Hash`\n//! traits are automatically derived for the `struct`s using the `derive` attribute.\n//! Additional traits can be derived by providing an explicit `derive`\n//! attribute on `struct`.\n//!\n//! The `Extend` and `FromIterator` traits are implemented for the `struct`s,\n//! too: `Extend` adds the union of the instances of the `struct` iterated over,\n//! while `FromIterator` calculates the union.\n//!\n//! The `Binary`, `Debug`, `LowerHex`, `Octal` and `UpperHex` traits are also\n//! implemented by displaying the bits value of the internal struct.\n//!\n//! ## Operators\n//!\n//! The following operator traits are implemented for the generated `struct`s:\n//!\n//! - `BitOr` and `BitOrAssign`: union\n//! - `BitAnd` and `BitAndAssign`: intersection\n//! - `BitXor` and `BitXorAssign`: toggle\n//! - `Sub` and `SubAssign`: set difference\n//! - `Not`: set complement\n//!\n//! # Methods\n//!\n//! The following methods are defined for the generated `struct`s:\n//!\n//! - `empty`: an empty set of flags\n//! - `all`: the set of all defined flags\n//! - `bits`: the raw value of the flags currently stored\n//! - `from_bits`: convert from underlying bit representation, unless that\n//! representation contains bits that do not correspond to a\n//! defined flag\n//! - `from_bits_truncate`: convert from underlying bit representation, dropping\n//! any bits that do not correspond to defined flags\n//! - `from_bits_unchecked`: convert from underlying bit representation, keeping\n//! all bits (even those not corresponding to defined\n//! flags)\n//! - `is_empty`: `true` if no flags are currently stored\n//! - `is_all`: `true` if currently set flags exactly equal all defined flags\n//! - `intersects`: `true` if there are flags common to both `self` and `other`\n//! - `contains`: `true` if all of the flags in `other` are contained within `self`\n//! - `insert`: inserts the specified flags in-place\n//! - `remove`: removes the specified flags in-place\n//! - `toggle`: the specified flags will be inserted if not present, and removed\n//! if they are.\n//! - `set`: inserts or removes the specified flags depending on the passed value\n//! - `intersection`: returns a new set of flags, containing only the flags present\n//! in both `self` and `other` (the argument to the function).\n//! - `union`: returns a new set of flags, containing any flags present in\n//! either `self` or `other` (the argument to the function).\n//! - `difference`: returns a new set of flags, containing all flags present in\n//! `self` without any of the flags present in `other` (the\n//! argument to the function).\n//! - `symmetric_difference`: returns a new set of flags, containing all flags\n//! present in either `self` or `other` (the argument\n//! to the function), but not both.\n//! - `complement`: returns a new set of flags, containing all flags which are\n//! not set in `self`, but which are allowed for this type.\n//!\n//! ## Default\n//!\n//! The `Default` trait is not automatically implemented for the generated structs.\n//!\n//! If your default value is equal to `0` (which is the same value as calling `empty()`\n//! on the generated struct), you can simply derive `Default`:\n//!\n//! ```\n//! use bitflags::bitflags;\n//!\n//! bitflags! {\n//! // Results in default value with bits: 0\n//! #[derive(Default)]\n//! struct Flags: u32 {\n//! const A = 0b00000001;\n//! const B = 0b00000010;\n//! const C = 0b00000100;\n//! }\n//! }\n//!\n//! fn main() {\n//! let derived_default: Flags = Default::default();\n//! assert_eq!(derived_default.bits(), 0);\n//! }\n//! ```\n//!\n//! If your default value is not equal to `0` you need to implement `Default` yourself:\n//!\n//! ```\n//! use bitflags::bitflags;\n//!\n//! bitflags! {\n//! struct Flags: u32 {\n//! const A = 0b00000001;\n//! const B = 0b00000010;\n//! const C = 0b00000100;\n//! }\n//! }\n//!\n//! // explicit `Default` implementation\n//! impl Default for Flags {\n//! fn default() -> Flags {\n//! Flags::A | Flags::C\n//! }\n//! }\n//!\n//! fn main() {\n//! let implemented_default: Flags = Default::default();\n//! assert_eq!(implemented_default, (Flags::A | Flags::C));\n//! }\n//! ```\n//!\n//! # Zero Flags\n//!\n//! Flags with a value equal to zero will have some strange behavior that one should be aware of.\n//!\n//! ```\n//! use bitflags::bitflags;\n//!\n//! bitflags! {\n//! struct Flags: u32 {\n//! const NONE = 0b00000000;\n//! const SOME = 0b00000001;\n//! }\n//! }\n//!\n//! fn main() {\n//! let empty = Flags::empty();\n//! let none = Flags::NONE;\n//! let some = Flags::SOME;\n//!\n//! // Zero flags are treated as always present\n//! assert!(empty.contains(Flags::NONE));\n//! assert!(none.contains(Flags::NONE));\n//! assert!(some.contains(Flags::NONE));\n//!\n//! // Zero flags will be ignored when testing for emptiness\n//! assert!(none.is_empty());\n//! }\n//! ```\n//!\n//! Users should generally avoid defining a flag with a value of zero.\n\n#![cfg_attr(not(test), no_std)]\n#![doc(html_root_url = \"https://docs.rs/bitflags/1.3.2\")]\n\n#[doc(hidden)]\npub extern crate core as _core;\n\n/// The macro used to generate the flag structures.\n///\n/// See the [crate level docs](../bitflags/index.html) for complete documentation.\n///\n/// # Example\n///\n/// ```\n/// use bitflags::bitflags;\n///\n/// bitflags! {\n/// struct Flags: u32 {\n/// const A = 0b00000001;\n/// const B = 0b00000010;\n/// const C = 0b00000100;\n/// const ABC = Self::A.bits | Self::B.bits | Self::C.bits;\n/// }\n/// }\n///\n/// fn main() {\n/// let e1 = Flags::A | Flags::C;\n/// let e2 = Flags::B | Flags::C;\n/// assert_eq!((e1 | e2), Flags::ABC); // union\n/// assert_eq!((e1 & e2), Flags::C); // intersection\n/// assert_eq!((e1 - e2), Flags::A); // set difference\n/// assert_eq!(!e2, Flags::A); // set complement\n/// }\n/// ```\n///\n/// The generated `struct`s can also be extended with type and trait\n/// implementations:\n///\n/// ```\n/// use std::fmt;\n///\n/// use bitflags::bitflags;\n///\n/// bitflags! {\n/// struct Flags: u32 {\n/// const A = 0b00000001;\n/// const B = 0b00000010;\n/// }\n/// }\n///\n/// impl Flags {\n/// pub fn clear(&mut self) {\n/// self.bits = 0; // The `bits` field can be accessed from within the\n/// // same module where the `bitflags!` macro was invoked.\n/// }\n/// }\n///\n/// impl fmt::Display for Flags {\n/// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n/// write!(f, \"hi!\")\n/// }\n/// }\n///\n/// fn main() {\n/// let mut flags = Flags::A | Flags::B;\n/// flags.clear();\n/// assert!(flags.is_empty());\n/// assert_eq!(format!(\"{}\", flags), \"hi!\");\n/// assert_eq!(format!(\"{:?}\", Flags::A | Flags::B), \"A | B\");\n/// assert_eq!(format!(\"{:?}\", Flags::B), \"B\");\n/// }\n/// ```\n#[macro_export(local_inner_macros)]\nmacro_rules! bitflags {\n (\n $(#[$outer:meta])*\n $vis:vis struct $BitFlags:ident: $T:ty {\n $(\n $(#[$inner:ident $($args:tt)*])*\n const $Flag:ident = $value:expr;\n )*\n }\n\n $($t:tt)*\n ) => {\n $(#[$outer])*\n #[derive(Copy, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]\n $vis struct $BitFlags {\n bits: $T,\n }\n\n __impl_bitflags! {\n $BitFlags: $T {\n $(\n $(#[$inner $($args)*])*\n $Flag = $value;\n )*\n }\n }\n\n bitflags! {\n $($t)*\n }\n };\n () => {};\n}\n\n// A helper macro to implement the `all` function.\n#[macro_export(local_inner_macros)]\n#[doc(hidden)]\nmacro_rules! __impl_all_bitflags {\n (\n $BitFlags:ident: $T:ty {\n $(\n $(#[$attr:ident $($args:tt)*])*\n $Flag:ident = $value:expr;\n )+\n }\n ) => {\n // See `Debug::fmt` for why this approach is taken.\n #[allow(non_snake_case)]\n trait __BitFlags {\n $(\n const $Flag: $T = 0;\n )+\n }\n #[allow(non_snake_case)]\n impl __BitFlags for $BitFlags {\n $(\n __impl_bitflags! {\n #[allow(deprecated)]\n $(? #[$attr $($args)*])*\n const $Flag: $T = Self::$Flag.bits;\n }\n )+\n }\n Self { bits: $(::$Flag)|+ }\n };\n (\n $BitFlags:ident: $T:ty { }\n ) => {\n Self { bits: 0 }\n };\n}\n\n#[macro_export(local_inner_macros)]\n#[doc(hidden)]\nmacro_rules! __impl_bitflags {\n (\n $BitFlags:ident: $T:ty {\n $(\n $(#[$attr:ident $($args:tt)*])*\n $Flag:ident = $value:expr;\n )*\n }\n ) => {\n impl $crate::_core::fmt::Debug for $BitFlags {\n fn fmt(&self, f: &mut $crate::_core::fmt::Formatter) -> $crate::_core::fmt::Result {\n // This convoluted approach is to handle #[cfg]-based flag\n // omission correctly. For example it needs to support:\n //\n // #[cfg(unix)] const A: Flag = /* ... */;\n // #[cfg(windows)] const B: Flag = /* ... */;\n\n // Unconditionally define a check for every flag, even disabled\n // ones.\n #[allow(non_snake_case)]\n trait __BitFlags {\n $(\n #[inline]\n fn $Flag(&self) -> bool { false }\n )*\n }\n\n // Conditionally override the check for just those flags that\n // are not #[cfg]ed away.\n #[allow(non_snake_case)]\n impl __BitFlags for $BitFlags {\n $(\n __impl_bitflags! {\n #[allow(deprecated)]\n #[inline]\n $(? #[$attr $($args)*])*\n fn $Flag(&self) -> bool {\n if Self::$Flag.bits == 0 && self.bits != 0 {\n false\n } else {\n self.bits & Self::$Flag.bits == Self::$Flag.bits\n }\n }\n }\n )*\n }\n\n let mut first = true;\n $(\n if ::$Flag(self) {\n if !first {\n f.write_str(\" | \")?;\n }\n first = false;\n f.write_str($crate::_core::stringify!($Flag))?;\n }\n )*\n let extra_bits = self.bits & !Self::all().bits();\n if extra_bits != 0 {\n if !first {\n f.write_str(\" | \")?;\n }\n first = false;\n f.write_str(\"0x\")?;\n $crate::_core::fmt::LowerHex::fmt(&extra_bits, f)?;\n }\n if first {\n f.write_str(\"(empty)\")?;\n }\n Ok(())\n }\n }\n impl $crate::_core::fmt::Binary for $BitFlags {\n fn fmt(&self, f: &mut $crate::_core::fmt::Formatter) -> $crate::_core::fmt::Result {\n $crate::_core::fmt::Binary::fmt(&self.bits, f)\n }\n }\n impl $crate::_core::fmt::Octal for $BitFlags {\n fn fmt(&self, f: &mut $crate::_core::fmt::Formatter) -> $crate::_core::fmt::Result {\n $crate::_core::fmt::Octal::fmt(&self.bits, f)\n }\n }\n impl $crate::_core::fmt::LowerHex for $BitFlags {\n fn fmt(&self, f: &mut $crate::_core::fmt::Formatter) -> $crate::_core::fmt::Result {\n $crate::_core::fmt::LowerHex::fmt(&self.bits, f)\n }\n }\n impl $crate::_core::fmt::UpperHex for $BitFlags {\n fn fmt(&self, f: &mut $crate::_core::fmt::Formatter) -> $crate::_core::fmt::Result {\n $crate::_core::fmt::UpperHex::fmt(&self.bits, f)\n }\n }\n\n #[allow(dead_code)]\n impl $BitFlags {\n $(\n $(#[$attr $($args)*])*\n pub const $Flag: Self = Self { bits: $value };\n )*\n\n /// Returns an empty set of flags.\n #[inline]\n pub const fn empty() -> Self {\n Self { bits: 0 }\n }\n\n /// Returns the set containing all flags.\n #[inline]\n pub const fn all() -> Self {\n __impl_all_bitflags! {\n $BitFlags: $T {\n $(\n $(#[$attr $($args)*])*\n $Flag = $value;\n )*\n }\n }\n }\n\n /// Returns the raw value of the flags currently stored.\n #[inline]\n pub const fn bits(&self) -> $T {\n self.bits\n }\n\n /// Convert from underlying bit representation, unless that\n /// representation contains bits that do not correspond to a flag.\n #[inline]\n pub const fn from_bits(bits: $T) -> $crate::_core::option::Option {\n if (bits & !Self::all().bits()) == 0 {\n $crate::_core::option::Option::Some(Self { bits })\n } else {\n $crate::_core::option::Option::None\n }\n }\n\n /// Convert from underlying bit representation, dropping any bits\n /// that do not correspond to flags.\n #[inline]\n pub const fn from_bits_truncate(bits: $T) -> Self {\n Self { bits: bits & Self::all().bits }\n }\n\n /// Convert from underlying bit representation, preserving all\n /// bits (even those not corresponding to a defined flag).\n ///\n /// # Safety\n ///\n /// The caller of the `bitflags!` macro can chose to allow or\n /// disallow extra bits for their bitflags type.\n ///\n /// The caller of `from_bits_unchecked()` has to ensure that\n /// all bits correspond to a defined flag or that extra bits\n /// are valid for this bitflags type.\n #[inline]\n pub const unsafe fn from_bits_unchecked(bits: $T) -> Self {\n Self { bits }\n }\n\n /// Returns `true` if no flags are currently stored.\n #[inline]\n pub const fn is_empty(&self) -> bool {\n self.bits() == Self::empty().bits()\n }\n\n /// Returns `true` if all flags are currently set.\n #[inline]\n pub const fn is_all(&self) -> bool {\n Self::all().bits | self.bits == self.bits\n }\n\n /// Returns `true` if there are flags common to both `self` and `other`.\n #[inline]\n pub const fn intersects(&self, other: Self) -> bool {\n !(Self { bits: self.bits & other.bits}).is_empty()\n }\n\n /// Returns `true` if all of the flags in `other` are contained within `self`.\n #[inline]\n pub const fn contains(&self, other: Self) -> bool {\n (self.bits & other.bits) == other.bits\n }\n\n /// Inserts the specified flags in-place.\n #[inline]\n pub fn insert(&mut self, other: Self) {\n self.bits |= other.bits;\n }\n\n /// Removes the specified flags in-place.\n #[inline]\n pub fn remove(&mut self, other: Self) {\n self.bits &= !other.bits;\n }\n\n /// Toggles the specified flags in-place.\n #[inline]\n pub fn toggle(&mut self, other: Self) {\n self.bits ^= other.bits;\n }\n\n /// Inserts or removes the specified flags depending on the passed value.\n #[inline]\n pub fn set(&mut self, other: Self, value: bool) {\n if value {\n self.insert(other);\n } else {\n self.remove(other);\n }\n }\n\n /// Returns the intersection between the flags in `self` and\n /// `other`.\n ///\n /// Specifically, the returned set contains only the flags which are\n /// present in *both* `self` *and* `other`.\n ///\n /// This is equivalent to using the `&` operator (e.g.\n /// [`ops::BitAnd`]), as in `flags & other`.\n ///\n /// [`ops::BitAnd`]: https://doc.rust-lang.org/std/ops/trait.BitAnd.html\n #[inline]\n #[must_use]\n pub const fn intersection(self, other: Self) -> Self {\n Self { bits: self.bits & other.bits }\n }\n\n /// Returns the union of between the flags in `self` and `other`.\n ///\n /// Specifically, the returned set contains all flags which are\n /// present in *either* `self` *or* `other`, including any which are\n /// present in both (see [`Self::symmetric_difference`] if that\n /// is undesirable).\n ///\n /// This is equivalent to using the `|` operator (e.g.\n /// [`ops::BitOr`]), as in `flags | other`.\n ///\n /// [`ops::BitOr`]: https://doc.rust-lang.org/std/ops/trait.BitOr.html\n #[inline]\n #[must_use]\n pub const fn union(self, other: Self) -> Self {\n Self { bits: self.bits | other.bits }\n }\n\n /// Returns the difference between the flags in `self` and `other`.\n ///\n /// Specifically, the returned set contains all flags present in\n /// `self`, except for the ones present in `other`.\n ///\n /// It is also conceptually equivalent to the \"bit-clear\" operation:\n /// `flags & !other` (and this syntax is also supported).\n ///\n /// This is equivalent to using the `-` operator (e.g.\n /// [`ops::Sub`]), as in `flags - other`.\n ///\n /// [`ops::Sub`]: https://doc.rust-lang.org/std/ops/trait.Sub.html\n #[inline]\n #[must_use]\n pub const fn difference(self, other: Self) -> Self {\n Self { bits: self.bits & !other.bits }\n }\n\n /// Returns the [symmetric difference][sym-diff] between the flags\n /// in `self` and `other`.\n ///\n /// Specifically, the returned set contains the flags present which\n /// are present in `self` or `other`, but that are not present in\n /// both. Equivalently, it contains the flags present in *exactly\n /// one* of the sets `self` and `other`.\n ///\n /// This is equivalent to using the `^` operator (e.g.\n /// [`ops::BitXor`]), as in `flags ^ other`.\n ///\n /// [sym-diff]: https://en.wikipedia.org/wiki/Symmetric_difference\n /// [`ops::BitXor`]: https://doc.rust-lang.org/std/ops/trait.BitXor.html\n #[inline]\n #[must_use]\n pub const fn symmetric_difference(self, other: Self) -> Self {\n Self { bits: self.bits ^ other.bits }\n }\n\n /// Returns the complement of this set of flags.\n ///\n /// Specifically, the returned set contains all the flags which are\n /// not set in `self`, but which are allowed for this type.\n ///\n /// Alternatively, it can be thought of as the set difference\n /// between [`Self::all()`] and `self` (e.g. `Self::all() - self`)\n ///\n /// This is equivalent to using the `!` operator (e.g.\n /// [`ops::Not`]), as in `!flags`.\n ///\n /// [`Self::all()`]: Self::all\n /// [`ops::Not`]: https://doc.rust-lang.org/std/ops/trait.Not.html\n #[inline]\n #[must_use]\n pub const fn complement(self) -> Self {\n Self::from_bits_truncate(!self.bits)\n }\n\n }\n\n impl $crate::_core::ops::BitOr for $BitFlags {\n type Output = Self;\n\n /// Returns the union of the two sets of flags.\n #[inline]\n fn bitor(self, other: $BitFlags) -> Self {\n Self { bits: self.bits | other.bits }\n }\n }\n\n impl $crate::_core::ops::BitOrAssign for $BitFlags {\n /// Adds the set of flags.\n #[inline]\n fn bitor_assign(&mut self, other: Self) {\n self.bits |= other.bits;\n }\n }\n\n impl $crate::_core::ops::BitXor for $BitFlags {\n type Output = Self;\n\n /// Returns the left flags, but with all the right flags toggled.\n #[inline]\n fn bitxor(self, other: Self) -> Self {\n Self { bits: self.bits ^ other.bits }\n }\n }\n\n impl $crate::_core::ops::BitXorAssign for $BitFlags {\n /// Toggles the set of flags.\n #[inline]\n fn bitxor_assign(&mut self, other: Self) {\n self.bits ^= other.bits;\n }\n }\n\n impl $crate::_core::ops::BitAnd for $BitFlags {\n type Output = Self;\n\n /// Returns the intersection between the two sets of flags.\n #[inline]\n fn bitand(self, other: Self) -> Self {\n Self { bits: self.bits & other.bits }\n }\n }\n\n impl $crate::_core::ops::BitAndAssign for $BitFlags {\n /// Disables all flags disabled in the set.\n #[inline]\n fn bitand_assign(&mut self, other: Self) {\n self.bits &= other.bits;\n }\n }\n\n impl $crate::_core::ops::Sub for $BitFlags {\n type Output = Self;\n\n /// Returns the set difference of the two sets of flags.\n #[inline]\n fn sub(self, other: Self) -> Self {\n Self { bits: self.bits & !other.bits }\n }\n }\n\n impl $crate::_core::ops::SubAssign for $BitFlags {\n /// Disables all flags enabled in the set.\n #[inline]\n fn sub_assign(&mut self, other: Self) {\n self.bits &= !other.bits;\n }\n }\n\n impl $crate::_core::ops::Not for $BitFlags {\n type Output = Self;\n\n /// Returns the complement of this set of flags.\n #[inline]\n fn not(self) -> Self {\n Self { bits: !self.bits } & Self::all()\n }\n }\n\n impl $crate::_core::iter::Extend<$BitFlags> for $BitFlags {\n fn extend>(&mut self, iterator: T) {\n for item in iterator {\n self.insert(item)\n }\n }\n }\n\n impl $crate::_core::iter::FromIterator<$BitFlags> for $BitFlags {\n fn from_iter>(iterator: T) -> Self {\n let mut result = Self::empty();\n result.extend(iterator);\n result\n }\n }\n };\n\n // Every attribute that the user writes on a const is applied to the\n // corresponding const that we generate, but within the implementation of\n // Debug and all() we want to ignore everything but #[cfg] attributes. In\n // particular, including a #[deprecated] attribute on those items would fail\n // to compile.\n // https://github.com/bitflags/bitflags/issues/109\n //\n // Input:\n //\n // ? #[cfg(feature = \"advanced\")]\n // ? #[deprecated(note = \"Use something else.\")]\n // ? #[doc = r\"High quality documentation.\"]\n // fn f() -> i32 { /* ... */ }\n //\n // Output:\n //\n // #[cfg(feature = \"advanced\")]\n // fn f() -> i32 { /* ... */ }\n (\n $(#[$filtered:meta])*\n ? #[cfg $($cfgargs:tt)*]\n $(? #[$rest:ident $($restargs:tt)*])*\n fn $($item:tt)*\n ) => {\n __impl_bitflags! {\n $(#[$filtered])*\n #[cfg $($cfgargs)*]\n $(? #[$rest $($restargs)*])*\n fn $($item)*\n }\n };\n (\n $(#[$filtered:meta])*\n // $next != `cfg`\n ? #[$next:ident $($nextargs:tt)*]\n $(? #[$rest:ident $($restargs:tt)*])*\n fn $($item:tt)*\n ) => {\n __impl_bitflags! {\n $(#[$filtered])*\n // $next filtered out\n $(? #[$rest $($restargs)*])*\n fn $($item)*\n }\n };\n (\n $(#[$filtered:meta])*\n fn $($item:tt)*\n ) => {\n $(#[$filtered])*\n fn $($item)*\n };\n\n // Every attribute that the user writes on a const is applied to the\n // corresponding const that we generate, but within the implementation of\n // Debug and all() we want to ignore everything but #[cfg] attributes. In\n // particular, including a #[deprecated] attribute on those items would fail\n // to compile.\n // https://github.com/bitflags/bitflags/issues/109\n //\n // const version\n //\n // Input:\n //\n // ? #[cfg(feature = \"advanced\")]\n // ? #[deprecated(note = \"Use something else.\")]\n // ? #[doc = r\"High quality documentation.\"]\n // const f: i32 { /* ... */ }\n //\n // Output:\n //\n // #[cfg(feature = \"advanced\")]\n // const f: i32 { /* ... */ }\n (\n $(#[$filtered:meta])*\n ? #[cfg $($cfgargs:tt)*]\n $(? #[$rest:ident $($restargs:tt)*])*\n const $($item:tt)*\n ) => {\n __impl_bitflags! {\n $(#[$filtered])*\n #[cfg $($cfgargs)*]\n $(? #[$rest $($restargs)*])*\n const $($item)*\n }\n };\n (\n $(#[$filtered:meta])*\n // $next != `cfg`\n ? #[$next:ident $($nextargs:tt)*]\n $(? #[$rest:ident $($restargs:tt)*])*\n const $($item:tt)*\n ) => {\n __impl_bitflags! {\n $(#[$filtered])*\n // $next filtered out\n $(? #[$rest $($restargs)*])*\n const $($item)*\n }\n };\n (\n $(#[$filtered:meta])*\n const $($item:tt)*\n ) => {\n $(#[$filtered])*\n const $($item)*\n };\n}\n\n#[cfg(feature = \"example_generated\")]\npub mod example_generated;\n\n#[cfg(test)]\nmod tests {\n use std::collections::hash_map::DefaultHasher;\n use std::hash::{Hash, Hasher};\n\n bitflags! {\n #[doc = \"> The first principle is that you must not fool yourself — and\"]\n #[doc = \"> you are the easiest person to fool.\"]\n #[doc = \"> \"]\n #[doc = \"> - Richard Feynman\"]\n #[derive(Default)]\n struct Flags: u32 {\n const A = 0b00000001;\n #[doc = \" macros are way better at generating code than trans is\"]\n const B = 0b00000010;\n const C = 0b00000100;\n #[doc = \"* cmr bed\"]\n #[doc = \"* strcat table\"]\n #[doc = \" wait what?\"]\n const ABC = Self::A.bits | Self::B.bits | Self::C.bits;\n }\n\n struct _CfgFlags: u32 {\n #[cfg(unix)]\n const _CFG_A = 0b01;\n #[cfg(windows)]\n const _CFG_B = 0b01;\n #[cfg(unix)]\n const _CFG_C = Self::_CFG_A.bits | 0b10;\n }\n\n struct AnotherSetOfFlags: i8 {\n const ANOTHER_FLAG = -1_i8;\n }\n\n struct LongFlags: u32 {\n const LONG_A = 0b1111111111111111;\n }\n }\n\n bitflags! {\n struct EmptyFlags: u32 {\n }\n }\n\n #[test]\n fn test_bits() {\n assert_eq!(Flags::empty().bits(), 0b00000000);\n assert_eq!(Flags::A.bits(), 0b00000001);\n assert_eq!(Flags::ABC.bits(), 0b00000111);\n\n assert_eq!(AnotherSetOfFlags::empty().bits(), 0b00);\n assert_eq!(AnotherSetOfFlags::ANOTHER_FLAG.bits(), !0_i8);\n\n assert_eq!(EmptyFlags::empty().bits(), 0b00000000);\n }\n\n #[test]\n fn test_from_bits() {\n assert_eq!(Flags::from_bits(0), Some(Flags::empty()));\n assert_eq!(Flags::from_bits(0b1), Some(Flags::A));\n assert_eq!(Flags::from_bits(0b10), Some(Flags::B));\n assert_eq!(Flags::from_bits(0b11), Some(Flags::A | Flags::B));\n assert_eq!(Flags::from_bits(0b1000), None);\n\n assert_eq!(\n AnotherSetOfFlags::from_bits(!0_i8),\n Some(AnotherSetOfFlags::ANOTHER_FLAG)\n );\n\n assert_eq!(EmptyFlags::from_bits(0), Some(EmptyFlags::empty()));\n assert_eq!(EmptyFlags::from_bits(0b1), None);\n }\n\n #[test]\n fn test_from_bits_truncate() {\n assert_eq!(Flags::from_bits_truncate(0), Flags::empty());\n assert_eq!(Flags::from_bits_truncate(0b1), Flags::A);\n assert_eq!(Flags::from_bits_truncate(0b10), Flags::B);\n assert_eq!(Flags::from_bits_truncate(0b11), (Flags::A | Flags::B));\n assert_eq!(Flags::from_bits_truncate(0b1000), Flags::empty());\n assert_eq!(Flags::from_bits_truncate(0b1001), Flags::A);\n\n assert_eq!(\n AnotherSetOfFlags::from_bits_truncate(0_i8),\n AnotherSetOfFlags::empty()\n );\n\n assert_eq!(EmptyFlags::from_bits_truncate(0), EmptyFlags::empty());\n assert_eq!(EmptyFlags::from_bits_truncate(0b1), EmptyFlags::empty());\n }\n\n #[test]\n fn test_from_bits_unchecked() {\n let extra = unsafe { Flags::from_bits_unchecked(0b1000) };\n assert_eq!(unsafe { Flags::from_bits_unchecked(0) }, Flags::empty());\n assert_eq!(unsafe { Flags::from_bits_unchecked(0b1) }, Flags::A);\n assert_eq!(unsafe { Flags::from_bits_unchecked(0b10) }, Flags::B);\n\n assert_eq!(\n unsafe { Flags::from_bits_unchecked(0b11) },\n (Flags::A | Flags::B)\n );\n assert_eq!(\n unsafe { Flags::from_bits_unchecked(0b1000) },\n (extra | Flags::empty())\n );\n assert_eq!(\n unsafe { Flags::from_bits_unchecked(0b1001) },\n (extra | Flags::A)\n );\n\n let extra = unsafe { EmptyFlags::from_bits_unchecked(0b1000) };\n assert_eq!(\n unsafe { EmptyFlags::from_bits_unchecked(0b1000) },\n (extra | EmptyFlags::empty())\n );\n }\n\n #[test]\n fn test_is_empty() {\n assert!(Flags::empty().is_empty());\n assert!(!Flags::A.is_empty());\n assert!(!Flags::ABC.is_empty());\n\n assert!(!AnotherSetOfFlags::ANOTHER_FLAG.is_empty());\n\n assert!(EmptyFlags::empty().is_empty());\n assert!(EmptyFlags::all().is_empty());\n }\n\n #[test]\n fn test_is_all() {\n assert!(Flags::all().is_all());\n assert!(!Flags::A.is_all());\n assert!(Flags::ABC.is_all());\n\n let extra = unsafe { Flags::from_bits_unchecked(0b1000) };\n assert!(!extra.is_all());\n assert!(!(Flags::A | extra).is_all());\n assert!((Flags::ABC | extra).is_all());\n\n assert!(AnotherSetOfFlags::ANOTHER_FLAG.is_all());\n\n assert!(EmptyFlags::all().is_all());\n assert!(EmptyFlags::empty().is_all());\n }\n\n #[test]\n fn test_two_empties_do_not_intersect() {\n let e1 = Flags::empty();\n let e2 = Flags::empty();\n assert!(!e1.intersects(e2));\n\n assert!(AnotherSetOfFlags::ANOTHER_FLAG.intersects(AnotherSetOfFlags::ANOTHER_FLAG));\n }\n\n #[test]\n fn test_empty_does_not_intersect_with_full() {\n let e1 = Flags::empty();\n let e2 = Flags::ABC;\n assert!(!e1.intersects(e2));\n }\n\n #[test]\n fn test_disjoint_intersects() {\n let e1 = Flags::A;\n let e2 = Flags::B;\n assert!(!e1.intersects(e2));\n }\n\n #[test]\n fn test_overlapping_intersects() {\n let e1 = Flags::A;\n let e2 = Flags::A | Flags::B;\n assert!(e1.intersects(e2));\n }\n\n #[test]\n fn test_contains() {\n let e1 = Flags::A;\n let e2 = Flags::A | Flags::B;\n assert!(!e1.contains(e2));\n assert!(e2.contains(e1));\n assert!(Flags::ABC.contains(e2));\n\n assert!(AnotherSetOfFlags::ANOTHER_FLAG.contains(AnotherSetOfFlags::ANOTHER_FLAG));\n\n assert!(EmptyFlags::empty().contains(EmptyFlags::empty()));\n }\n\n #[test]\n fn test_insert() {\n let mut e1 = Flags::A;\n let e2 = Flags::A | Flags::B;\n e1.insert(e2);\n assert_eq!(e1, e2);\n\n let mut e3 = AnotherSetOfFlags::empty();\n e3.insert(AnotherSetOfFlags::ANOTHER_FLAG);\n assert_eq!(e3, AnotherSetOfFlags::ANOTHER_FLAG);\n }\n\n #[test]\n fn test_remove() {\n let mut e1 = Flags::A | Flags::B;\n let e2 = Flags::A | Flags::C;\n e1.remove(e2);\n assert_eq!(e1, Flags::B);\n\n let mut e3 = AnotherSetOfFlags::ANOTHER_FLAG;\n e3.remove(AnotherSetOfFlags::ANOTHER_FLAG);\n assert_eq!(e3, AnotherSetOfFlags::empty());\n }\n\n #[test]\n fn test_operators() {\n let e1 = Flags::A | Flags::C;\n let e2 = Flags::B | Flags::C;\n assert_eq!((e1 | e2), Flags::ABC); // union\n assert_eq!((e1 & e2), Flags::C); // intersection\n assert_eq!((e1 - e2), Flags::A); // set difference\n assert_eq!(!e2, Flags::A); // set complement\n assert_eq!(e1 ^ e2, Flags::A | Flags::B); // toggle\n let mut e3 = e1;\n e3.toggle(e2);\n assert_eq!(e3, Flags::A | Flags::B);\n\n let mut m4 = AnotherSetOfFlags::empty();\n m4.toggle(AnotherSetOfFlags::empty());\n assert_eq!(m4, AnotherSetOfFlags::empty());\n }\n\n #[test]\n fn test_operators_unchecked() {\n let extra = unsafe { Flags::from_bits_unchecked(0b1000) };\n let e1 = Flags::A | Flags::C | extra;\n let e2 = Flags::B | Flags::C;\n assert_eq!((e1 | e2), (Flags::ABC | extra)); // union\n assert_eq!((e1 & e2), Flags::C); // intersection\n assert_eq!((e1 - e2), (Flags::A | extra)); // set difference\n assert_eq!(!e2, Flags::A); // set complement\n assert_eq!(!e1, Flags::B); // set complement\n assert_eq!(e1 ^ e2, Flags::A | Flags::B | extra); // toggle\n let mut e3 = e1;\n e3.toggle(e2);\n assert_eq!(e3, Flags::A | Flags::B | extra);\n }\n\n #[test]\n fn test_set_ops_basic() {\n let ab = Flags::A.union(Flags::B);\n let ac = Flags::A.union(Flags::C);\n let bc = Flags::B.union(Flags::C);\n assert_eq!(ab.bits, 0b011);\n assert_eq!(bc.bits, 0b110);\n assert_eq!(ac.bits, 0b101);\n\n assert_eq!(ab, Flags::B.union(Flags::A));\n assert_eq!(ac, Flags::C.union(Flags::A));\n assert_eq!(bc, Flags::C.union(Flags::B));\n\n assert_eq!(ac, Flags::A | Flags::C);\n assert_eq!(bc, Flags::B | Flags::C);\n assert_eq!(ab.union(bc), Flags::ABC);\n\n assert_eq!(ac, Flags::A | Flags::C);\n assert_eq!(bc, Flags::B | Flags::C);\n\n assert_eq!(ac.union(bc), ac | bc);\n assert_eq!(ac.union(bc), Flags::ABC);\n assert_eq!(bc.union(ac), Flags::ABC);\n\n assert_eq!(ac.intersection(bc), ac & bc);\n assert_eq!(ac.intersection(bc), Flags::C);\n assert_eq!(bc.intersection(ac), Flags::C);\n\n assert_eq!(ac.difference(bc), ac - bc);\n assert_eq!(bc.difference(ac), bc - ac);\n assert_eq!(ac.difference(bc), Flags::A);\n assert_eq!(bc.difference(ac), Flags::B);\n\n assert_eq!(bc.complement(), !bc);\n assert_eq!(bc.complement(), Flags::A);\n assert_eq!(ac.symmetric_difference(bc), Flags::A.union(Flags::B));\n assert_eq!(bc.symmetric_difference(ac), Flags::A.union(Flags::B));\n }\n\n #[test]\n fn test_set_ops_const() {\n // These just test that these compile and don't cause use-site panics\n // (would be possible if we had some sort of UB)\n const INTERSECT: Flags = Flags::all().intersection(Flags::C);\n const UNION: Flags = Flags::A.union(Flags::C);\n const DIFFERENCE: Flags = Flags::all().difference(Flags::A);\n const COMPLEMENT: Flags = Flags::C.complement();\n const SYM_DIFFERENCE: Flags = UNION.symmetric_difference(DIFFERENCE);\n assert_eq!(INTERSECT, Flags::C);\n assert_eq!(UNION, Flags::A | Flags::C);\n assert_eq!(DIFFERENCE, Flags::all() - Flags::A);\n assert_eq!(COMPLEMENT, !Flags::C);\n assert_eq!(SYM_DIFFERENCE, (Flags::A | Flags::C) ^ (Flags::all() - Flags::A));\n }\n\n #[test]\n fn test_set_ops_unchecked() {\n let extra = unsafe { Flags::from_bits_unchecked(0b1000) };\n let e1 = Flags::A.union(Flags::C).union(extra);\n let e2 = Flags::B.union(Flags::C);\n assert_eq!(e1.bits, 0b1101);\n assert_eq!(e1.union(e2), (Flags::ABC | extra));\n assert_eq!(e1.intersection(e2), Flags::C);\n assert_eq!(e1.difference(e2), Flags::A | extra);\n assert_eq!(e2.difference(e1), Flags::B);\n assert_eq!(e2.complement(), Flags::A);\n assert_eq!(e1.complement(), Flags::B);\n assert_eq!(e1.symmetric_difference(e2), Flags::A | Flags::B | extra); // toggle\n }\n\n #[test]\n fn test_set_ops_exhaustive() {\n // Define a flag that contains gaps to help exercise edge-cases,\n // especially around \"unknown\" flags (e.g. ones outside of `all()`\n // `from_bits_unchecked`).\n // - when lhs and rhs both have different sets of unknown flags.\n // - unknown flags at both ends, and in the middle\n // - cases with \"gaps\".\n bitflags! {\n struct Test: u16 {\n // Intentionally no `A`\n const B = 0b000000010;\n // Intentionally no `C`\n const D = 0b000001000;\n const E = 0b000010000;\n const F = 0b000100000;\n const G = 0b001000000;\n // Intentionally no `H`\n const I = 0b100000000;\n }\n }\n let iter_test_flags =\n || (0..=0b111_1111_1111).map(|bits| unsafe { Test::from_bits_unchecked(bits) });\n\n for a in iter_test_flags() {\n assert_eq!(\n a.complement(),\n Test::from_bits_truncate(!a.bits),\n \"wrong result: !({:?})\",\n a,\n );\n assert_eq!(a.complement(), !a, \"named != op: !({:?})\", a);\n for b in iter_test_flags() {\n // Check that the named operations produce the expected bitwise\n // values.\n assert_eq!(\n a.union(b).bits,\n a.bits | b.bits,\n \"wrong result: `{:?}` | `{:?}`\",\n a,\n b,\n );\n assert_eq!(\n a.intersection(b).bits,\n a.bits & b.bits,\n \"wrong result: `{:?}` & `{:?}`\",\n a,\n b,\n );\n assert_eq!(\n a.symmetric_difference(b).bits,\n a.bits ^ b.bits,\n \"wrong result: `{:?}` ^ `{:?}`\",\n a,\n b,\n );\n assert_eq!(\n a.difference(b).bits,\n a.bits & !b.bits,\n \"wrong result: `{:?}` - `{:?}`\",\n a,\n b,\n );\n // Note: Difference is checked as both `a - b` and `b - a`\n assert_eq!(\n b.difference(a).bits,\n b.bits & !a.bits,\n \"wrong result: `{:?}` - `{:?}`\",\n b,\n a,\n );\n // Check that the named set operations are equivalent to the\n // bitwise equivalents\n assert_eq!(a.union(b), a | b, \"named != op: `{:?}` | `{:?}`\", a, b,);\n assert_eq!(\n a.intersection(b),\n a & b,\n \"named != op: `{:?}` & `{:?}`\",\n a,\n b,\n );\n assert_eq!(\n a.symmetric_difference(b),\n a ^ b,\n \"named != op: `{:?}` ^ `{:?}`\",\n a,\n b,\n );\n assert_eq!(a.difference(b), a - b, \"named != op: `{:?}` - `{:?}`\", a, b,);\n // Note: Difference is checked as both `a - b` and `b - a`\n assert_eq!(b.difference(a), b - a, \"named != op: `{:?}` - `{:?}`\", b, a,);\n // Verify that the operations which should be symmetric are\n // actually symmetric.\n assert_eq!(a.union(b), b.union(a), \"asymmetry: `{:?}` | `{:?}`\", a, b,);\n assert_eq!(\n a.intersection(b),\n b.intersection(a),\n \"asymmetry: `{:?}` & `{:?}`\",\n a,\n b,\n );\n assert_eq!(\n a.symmetric_difference(b),\n b.symmetric_difference(a),\n \"asymmetry: `{:?}` ^ `{:?}`\",\n a,\n b,\n );\n }\n }\n }\n\n #[test]\n fn test_set() {\n let mut e1 = Flags::A | Flags::C;\n e1.set(Flags::B, true);\n e1.set(Flags::C, false);\n\n assert_eq!(e1, Flags::A | Flags::B);\n }\n\n #[test]\n fn test_assignment_operators() {\n let mut m1 = Flags::empty();\n let e1 = Flags::A | Flags::C;\n // union\n m1 |= Flags::A;\n assert_eq!(m1, Flags::A);\n // intersection\n m1 &= e1;\n assert_eq!(m1, Flags::A);\n // set difference\n m1 -= m1;\n assert_eq!(m1, Flags::empty());\n // toggle\n m1 ^= e1;\n assert_eq!(m1, e1);\n }\n\n #[test]\n fn test_const_fn() {\n const _M1: Flags = Flags::empty();\n\n const M2: Flags = Flags::A;\n assert_eq!(M2, Flags::A);\n\n const M3: Flags = Flags::C;\n assert_eq!(M3, Flags::C);\n }\n\n #[test]\n fn test_extend() {\n let mut flags;\n\n flags = Flags::empty();\n flags.extend([].iter().cloned());\n assert_eq!(flags, Flags::empty());\n\n flags = Flags::empty();\n flags.extend([Flags::A, Flags::B].iter().cloned());\n assert_eq!(flags, Flags::A | Flags::B);\n\n flags = Flags::A;\n flags.extend([Flags::A, Flags::B].iter().cloned());\n assert_eq!(flags, Flags::A | Flags::B);\n\n flags = Flags::B;\n flags.extend([Flags::A, Flags::ABC].iter().cloned());\n assert_eq!(flags, Flags::ABC);\n }\n\n #[test]\n fn test_from_iterator() {\n assert_eq!([].iter().cloned().collect::(), Flags::empty());\n assert_eq!(\n [Flags::A, Flags::B].iter().cloned().collect::(),\n Flags::A | Flags::B\n );\n assert_eq!(\n [Flags::A, Flags::ABC].iter().cloned().collect::(),\n Flags::ABC\n );\n }\n\n #[test]\n fn test_lt() {\n let mut a = Flags::empty();\n let mut b = Flags::empty();\n\n assert!(!(a < b) && !(b < a));\n b = Flags::B;\n assert!(a < b);\n a = Flags::C;\n assert!(!(a < b) && b < a);\n b = Flags::C | Flags::B;\n assert!(a < b);\n }\n\n #[test]\n fn test_ord() {\n let mut a = Flags::empty();\n let mut b = Flags::empty();\n\n assert!(a <= b && a >= b);\n a = Flags::A;\n assert!(a > b && a >= b);\n assert!(b < a && b <= a);\n b = Flags::B;\n assert!(b > a && b >= a);\n assert!(a < b && a <= b);\n }\n\n fn hash(t: &T) -> u64 {\n let mut s = DefaultHasher::new();\n t.hash(&mut s);\n s.finish()\n }\n\n #[test]\n fn test_hash() {\n let mut x = Flags::empty();\n let mut y = Flags::empty();\n assert_eq!(hash(&x), hash(&y));\n x = Flags::all();\n y = Flags::ABC;\n assert_eq!(hash(&x), hash(&y));\n }\n\n #[test]\n fn test_default() {\n assert_eq!(Flags::empty(), Flags::default());\n }\n\n #[test]\n fn test_debug() {\n assert_eq!(format!(\"{:?}\", Flags::A | Flags::B), \"A | B\");\n assert_eq!(format!(\"{:?}\", Flags::empty()), \"(empty)\");\n assert_eq!(format!(\"{:?}\", Flags::ABC), \"A | B | C | ABC\");\n let extra = unsafe { Flags::from_bits_unchecked(0xb8) };\n assert_eq!(format!(\"{:?}\", extra), \"0xb8\");\n assert_eq!(format!(\"{:?}\", Flags::A | extra), \"A | 0xb8\");\n\n assert_eq!(\n format!(\"{:?}\", Flags::ABC | extra),\n \"A | B | C | ABC | 0xb8\"\n );\n\n assert_eq!(format!(\"{:?}\", EmptyFlags::empty()), \"(empty)\");\n }\n\n #[test]\n fn test_binary() {\n assert_eq!(format!(\"{:b}\", Flags::ABC), \"111\");\n assert_eq!(format!(\"{:#b}\", Flags::ABC), \"0b111\");\n let extra = unsafe { Flags::from_bits_unchecked(0b1010000) };\n assert_eq!(format!(\"{:b}\", Flags::ABC | extra), \"1010111\");\n assert_eq!(format!(\"{:#b}\", Flags::ABC | extra), \"0b1010111\");\n }\n\n #[test]\n fn test_octal() {\n assert_eq!(format!(\"{:o}\", LongFlags::LONG_A), \"177777\");\n assert_eq!(format!(\"{:#o}\", LongFlags::LONG_A), \"0o177777\");\n let extra = unsafe { LongFlags::from_bits_unchecked(0o5000000) };\n assert_eq!(format!(\"{:o}\", LongFlags::LONG_A | extra), \"5177777\");\n assert_eq!(format!(\"{:#o}\", LongFlags::LONG_A | extra), \"0o5177777\");\n }\n\n #[test]\n fn test_lowerhex() {\n assert_eq!(format!(\"{:x}\", LongFlags::LONG_A), \"ffff\");\n assert_eq!(format!(\"{:#x}\", LongFlags::LONG_A), \"0xffff\");\n let extra = unsafe { LongFlags::from_bits_unchecked(0xe00000) };\n assert_eq!(format!(\"{:x}\", LongFlags::LONG_A | extra), \"e0ffff\");\n assert_eq!(format!(\"{:#x}\", LongFlags::LONG_A | extra), \"0xe0ffff\");\n }\n\n #[test]\n fn test_upperhex() {\n assert_eq!(format!(\"{:X}\", LongFlags::LONG_A), \"FFFF\");\n assert_eq!(format!(\"{:#X}\", LongFlags::LONG_A), \"0xFFFF\");\n let extra = unsafe { LongFlags::from_bits_unchecked(0xe00000) };\n assert_eq!(format!(\"{:X}\", LongFlags::LONG_A | extra), \"E0FFFF\");\n assert_eq!(format!(\"{:#X}\", LongFlags::LONG_A | extra), \"0xE0FFFF\");\n }\n\n mod submodule {\n bitflags! {\n pub struct PublicFlags: i8 {\n const X = 0;\n }\n\n struct PrivateFlags: i8 {\n const Y = 0;\n }\n }\n\n #[test]\n fn test_private() {\n let _ = PrivateFlags::Y;\n }\n }\n\n #[test]\n fn test_public() {\n let _ = submodule::PublicFlags::X;\n }\n\n mod t1 {\n mod foo {\n pub type Bar = i32;\n }\n\n bitflags! {\n /// baz\n struct Flags: foo::Bar {\n const A = 0b00000001;\n #[cfg(foo)]\n const B = 0b00000010;\n #[cfg(foo)]\n const C = 0b00000010;\n }\n }\n }\n\n #[test]\n fn test_in_function() {\n bitflags! {\n struct Flags: u8 {\n const A = 1;\n #[cfg(any())] // false\n const B = 2;\n }\n }\n assert_eq!(Flags::all(), Flags::A);\n assert_eq!(format!(\"{:?}\", Flags::A), \"A\");\n }\n\n #[test]\n fn test_deprecated() {\n bitflags! {\n pub struct TestFlags: u32 {\n #[deprecated(note = \"Use something else.\")]\n const ONE = 1;\n }\n }\n }\n\n #[test]\n fn test_pub_crate() {\n mod module {\n bitflags! {\n pub (crate) struct Test: u8 {\n const FOO = 1;\n }\n }\n }\n\n assert_eq!(module::Test::FOO.bits(), 1);\n }\n\n #[test]\n fn test_pub_in_module() {\n mod module {\n mod submodule {\n bitflags! {\n // `pub (in super)` means only the module `module` will\n // be able to access this.\n pub (in super) struct Test: u8 {\n const FOO = 1;\n }\n }\n }\n\n mod test {\n // Note: due to `pub (in super)`,\n // this cannot be accessed directly by the testing code.\n pub(super) fn value() -> u8 {\n super::submodule::Test::FOO.bits()\n }\n }\n\n pub fn value() -> u8 {\n test::value()\n }\n }\n\n assert_eq!(module::value(), 1)\n }\n\n #[test]\n fn test_zero_value_flags() {\n bitflags! {\n struct Flags: u32 {\n const NONE = 0b0;\n const SOME = 0b1;\n }\n }\n\n assert!(Flags::empty().contains(Flags::NONE));\n assert!(Flags::SOME.contains(Flags::NONE));\n assert!(Flags::NONE.is_empty());\n\n assert_eq!(format!(\"{:?}\", Flags::empty()), \"NONE\");\n assert_eq!(format!(\"{:?}\", Flags::SOME), \"SOME\");\n }\n\n #[test]\n fn test_empty_bitflags() {\n bitflags! {}\n }\n\n #[test]\n fn test_u128_bitflags() {\n bitflags! {\n struct Flags128: u128 {\n const A = 0x0000_0000_0000_0000_0000_0000_0000_0001;\n const B = 0x0000_0000_0000_1000_0000_0000_0000_0000;\n const C = 0x8000_0000_0000_0000_0000_0000_0000_0000;\n const ABC = Self::A.bits | Self::B.bits | Self::C.bits;\n }\n }\n\n assert_eq!(Flags128::ABC, Flags128::A | Flags128::B | Flags128::C);\n assert_eq!(Flags128::A.bits, 0x0000_0000_0000_0000_0000_0000_0000_0001);\n assert_eq!(Flags128::B.bits, 0x0000_0000_0000_1000_0000_0000_0000_0000);\n assert_eq!(Flags128::C.bits, 0x8000_0000_0000_0000_0000_0000_0000_0000);\n assert_eq!(\n Flags128::ABC.bits,\n 0x8000_0000_0000_1000_0000_0000_0000_0001\n );\n assert_eq!(format!(\"{:?}\", Flags128::A), \"A\");\n assert_eq!(format!(\"{:?}\", Flags128::B), \"B\");\n assert_eq!(format!(\"{:?}\", Flags128::C), \"C\");\n assert_eq!(format!(\"{:?}\", Flags128::ABC), \"A | B | C | ABC\");\n }\n\n #[test]\n fn test_serde_bitflags_serialize() {\n let flags = SerdeFlags::A | SerdeFlags::B;\n\n let serialized = serde_json::to_string(&flags).unwrap();\n\n assert_eq!(serialized, r#\"{\"bits\":3}\"#);\n }\n\n #[test]\n fn test_serde_bitflags_deserialize() {\n let deserialized: SerdeFlags = serde_json::from_str(r#\"{\"bits\":12}\"#).unwrap();\n\n let expected = SerdeFlags::C | SerdeFlags::D;\n\n assert_eq!(deserialized.bits, expected.bits);\n }\n\n #[test]\n fn test_serde_bitflags_roundtrip() {\n let flags = SerdeFlags::A | SerdeFlags::B;\n\n let deserialized: SerdeFlags = serde_json::from_str(&serde_json::to_string(&flags).unwrap()).unwrap();\n\n assert_eq!(deserialized.bits, flags.bits);\n }\n\n bitflags! {\n #[derive(serde::Serialize, serde::Deserialize)]\n struct SerdeFlags: u32 {\n const A = 1;\n const B = 2;\n const C = 4;\n const D = 8;\n }\n }\n}\n" }, { "package": "bitflags@2.13.2", "sourcePath": "src/lib.rs", "sha256": "f8d141a1ba6b1996b33eb34669786ef389d4d5226edad46cec22a561588c324b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n/*!\nGenerate types for C-style flags with ergonomic APIs.\n\n# Getting started\n\nAdd `bitflags` to your `Cargo.toml`:\n\n```toml\n[dependencies.bitflags]\nversion = \"2.13.2\"\n```\n\n## Crate features\n\nThe `bitflags` library defines a few Cargo features that you can opt-in to:\n\n- `std`: Implement the `Error` trait on error types used by `bitflags`.\n- `serde`: Support deriving `serde` traits on generated flags types.\n- `arbitrary`: Support deriving `arbitrary` traits on generated flags types.\n- `bytemuck`: Support deriving `bytemuck` traits on generated flags types.\n\n## Generating flags types\n\nUse the [`bitflags`] macro to generate flags types:\n\n```rust\nuse bitflags::bitflags;\n\nbitflags! {\n pub struct Flags: u32 {\n const A = 0b00000001;\n const B = 0b00000010;\n const C = 0b00000100;\n }\n}\n```\n\nSee the docs for the `bitflags` macro for the full syntax.\n\nAlso see the [`example_generated`](./example_generated/index.html) module for an example of what the `bitflags` macro generates for a flags type.\n\n### Externally defined flags\n\nIf you're generating flags types for an external source, such as a C API, you can define\nan extra unnamed flag as a mask of all bits the external source may ever set. Usually this would be all bits (`!0`):\n\n```rust\n# use bitflags::bitflags;\nbitflags! {\n pub struct Flags: u32 {\n const A = 0b00000001;\n const B = 0b00000010;\n const C = 0b00000100;\n\n // The source may set any bits\n const _ = !0;\n }\n}\n```\n\nWhy should you do this? Generated methods like `all` and truncating operators like `!` only consider\nbits in defined flags. Adding an unnamed flag makes those methods consider additional bits,\nwithout generating additional constants for them. It helps compatibility when the external source\nmay start setting additional bits at any time. The [known and unknown bits](#known-and-unknown-bits)\nsection has more details on this behavior.\n\n### Custom derives\n\nYou can derive some traits on generated flags types if you enable Cargo features. The following\nlibraries are currently supported:\n\n- `serde`: Support `#[derive(Serialize, Deserialize)]`, using text for human-readable formats,\n and a raw number for binary formats.\n- `arbitrary`: Support `#[derive(Arbitrary)]`, only generating flags values with known bits.\n- `bytemuck`: Support `#[derive(Pod, Zeroable)]`, for casting between flags values and their\n underlying bits values.\n\nYou can also define your own flags type outside of the [`bitflags`] macro and then use it to generate methods.\nThis can be useful if you need a custom `#[derive]` attribute for a library that `bitflags` doesn't\nnatively support:\n\n```rust\n# use std::fmt::Debug as SomeTrait;\n# use bitflags::bitflags;\n#[derive(SomeTrait)]\npub struct Flags(u32);\n\nbitflags! {\n impl Flags: u32 {\n const A = 0b00000001;\n const B = 0b00000010;\n const C = 0b00000100;\n }\n}\n```\n\n### Adding custom methods\n\nThe [`bitflags`] macro supports attributes on generated flags types within the macro itself, while\n`impl` blocks can be added outside of it:\n\n```rust\n# use bitflags::bitflags;\nbitflags! {\n // Attributes can be applied to flags types\n #[repr(transparent)]\n #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]\n pub struct Flags: u32 {\n const A = 0b00000001;\n const B = 0b00000010;\n const C = 0b00000100;\n }\n}\n\n// Impl blocks can be added to flags types\nimpl Flags {\n pub fn as_u64(&self) -> u64 {\n self.bits() as u64\n }\n}\n```\n\n### Renaming flags\n\nThe [`bitflags`] macro recognizes a special `#[bitflags(flag_name = \"\")]` attribute on flags values to rename them:\n\n```rust\n# use bitflags::bitflags;\nbitflags! {\n pub struct Flags: u32 {\n // Add the attribute to a flag to change its name\n #[bitflags(flag_name = \"a\")]\n const A = 0b00000001;\n #[bitflags(flag_name = \"b\")]\n const B = 0b00000010;\n #[bitflags(flag_name = \"c\")]\n const C = 0b00000100;\n }\n}\n```\n\nWhen applied to a flag value, instead of using its identifier, like `A` as the name, it'll use the given string. This\ndoesn't affect the identifier of the constant itself, just the name recognized when parsing and formatting.\n\n## Working with flags values\n\nUse generated constants and standard bitwise operators to interact with flags values:\n\n```rust\n# use bitflags::bitflags;\n# bitflags! {\n# #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]\n# pub struct Flags: u32 {\n# const A = 0b00000001;\n# const B = 0b00000010;\n# const C = 0b00000100;\n# }\n# }\n// union\nlet ab = Flags::A | Flags::B;\n\n// intersection\nlet a = ab & Flags::A;\n\n// difference\nlet b = ab - Flags::A;\n\n// complement\nlet c = !ab;\n```\n\nSee the docs for the [`Flags`] trait for more details on operators and how they behave.\n\n# Formatting and parsing\n\n`bitflags` defines a text format that can be used to convert any flags value to and from strings.\n\nSee the [`parser`] module for more details.\n\n# Specification\n\nThe terminology and behavior of generated flags types is\n[specified in the source repository](https://github.com/bitflags/bitflags/blob/main/spec.md).\nDetails are repeated in these docs where appropriate, but is exhaustively listed in the spec. Some\nthings are worth calling out explicitly here.\n\n## Flags types, flags values, flags\n\nThe spec and these docs use consistent terminology to refer to things in the bitflags domain:\n\n- **Bits type**: A type that defines a fixed number of bits at specific locations.\n- **Flag**: A set of bits in a bits type that may have a unique name.\n- **Flags type**: A set of defined flags over a specific bits type.\n- **Flags value**: An instance of a flags type using its specific bits value for storage.\n\n```\n# use bitflags::bitflags;\nbitflags! {\n struct FlagsType: u8 {\n// -- Bits type\n// --------- Flags type\n const A = 1;\n// ----- Flag\n }\n}\n\nlet flag = FlagsType::A;\n// ---- Flags value\n```\n\n## Known and unknown bits\n\nAny bits in a flag you define are called _known bits_. Any other bits are _unknown bits_.\nIn the following flags type:\n\n```\n# use bitflags::bitflags;\nbitflags! {\n struct Flags: u8 {\n const A = 1;\n const B = 1 << 1;\n const C = 1 << 2;\n }\n}\n```\n\nThe known bits are `0b0000_0111` and the unknown bits are `0b1111_1000`.\n\n`bitflags` doesn't guarantee that a flags value will only ever have known bits set, but some operators\nwill unset any unknown bits they encounter. In a future version of `bitflags`, all operators will\nunset unknown bits.\n\nIf you're using `bitflags` for flags types defined externally, such as from C, you probably want all\nbits to be considered known, in case that external source changes. You can do this using an unnamed\nflag, as described in [externally defined flags](#externally-defined-flags).\n\n## Zero-bit flags\n\nFlags with no bits set should be avoided because they interact strangely with [`Flags::contains`]\nand [`Flags::intersects`]. A zero-bit flag is always contained, but is never intersected. The\nnames of zero-bit flags can be parsed, but are never formatted.\n\n## Multi-bit flags\n\nFlags that set multiple bits should be avoided unless each bit is also in a single-bit flag.\nTake the following flags type as an example:\n\n```\n# use bitflags::bitflags;\nbitflags! {\n struct Flags: u8 {\n const A = 1;\n const B = 1 | 1 << 1;\n }\n}\n```\n\nThe result of `Flags::A ^ Flags::B` is `0b0000_0010`, which doesn't correspond to either\n`Flags::A` or `Flags::B` even though it's still a known bit.\n*/\n\n#![cfg_attr(not(any(feature = \"std\", test)), no_std)]\n#![cfg_attr(not(test), forbid(unsafe_code))]\n#![cfg_attr(test, allow(mixed_script_confusables))]\n\n#[doc(inline)]\npub use traits::{Bits, Flag, Flags};\n\npub mod iter;\npub mod parser;\n\nmod traits;\n\n#[doc(hidden)]\npub mod __private {\n #[allow(unused_imports)]\n // Easier than conditionally checking any optional external dependencies\n pub use crate::{external::__private::*, traits::__private::*};\n\n pub use core;\n}\n\n#[allow(unused_imports)]\npub use external::*;\n\n#[allow(deprecated)]\npub use traits::BitFlags;\n\n/*\nHow does the bitflags crate work?\n\nThis library generates a `struct` in the end-user's crate with a bunch of constants on it that represent flags.\nThe difference between `bitflags` and a lot of other libraries is that we don't actually control the generated `struct` in the end.\nIt's part of the end-user's crate, so it belongs to them. That makes it difficult to extend `bitflags` with new functionality\nbecause we could end up breaking valid code that was already written.\n\nOur solution is to split the type we generate into two: the public struct owned by the end-user, and an internal struct owned by `bitflags` (us).\nTo give you an example, let's say we had a crate that called `bitflags!`:\n\n```rust\nbitflags! {\n pub struct MyFlags: u32 {\n const A = 1;\n const B = 2;\n }\n}\n```\n\nWhat they'd end up with looks something like this:\n\n```rust\npub struct MyFlags(::InternalBitFlags);\n\nconst _: () = {\n #[repr(transparent)]\n #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]\n pub struct MyInternalBitFlags {\n bits: u32,\n }\n\n impl PublicFlags for MyFlags {\n type Internal = InternalBitFlags;\n }\n};\n```\n\nIf we want to expose something like a new trait impl for generated flags types, we add it to our generated `MyInternalBitFlags`,\nand let `#[derive]` on `MyFlags` pick up that implementation, if an end-user chooses to add one.\n\nThe public API is generated in the `__impl_public_flags!` macro, and the internal API is generated in\nthe `__impl_internal_flags!` macro.\n\nThe macros are split into 3 modules:\n\n- `public`: where the user-facing flags types are generated.\n- `internal`: where the `bitflags`-facing flags types are generated.\n- `external`: where external library traits are implemented conditionally.\n*/\n\n/**\nGenerate a flags type.\n\n# `struct` mode\n\nA declaration that begins with `$vis struct` will generate a `struct` for a flags type, along with\nmethods and trait implementations for it. The body of the declaration defines flags as constants,\nwhere each constant is a flags value of the generated flags type.\n\n## Examples\n\nGenerate a flags type using `u8` as the bits type:\n\n```\n# use bitflags::bitflags;\nbitflags! {\n struct Flags: u8 {\n const A = 1;\n const B = 1 << 1;\n const C = 0b0000_0100;\n }\n}\n```\n\nFlags types are private by default and accept standard visibility modifiers. Flags themselves\nare always public:\n\n```\n# use bitflags::bitflags;\nbitflags! {\n pub struct Flags: u8 {\n // Constants are always `pub`\n const A = 1;\n }\n}\n```\n\nFlags may refer to other flags using their [`Flags::bits`] value:\n\n```\n# use bitflags::bitflags;\nbitflags! {\n struct Flags: u8 {\n const A = 1;\n const B = 1 << 1;\n const AB = Flags::A.bits() | Flags::B.bits();\n }\n}\n```\n\nA single `bitflags` invocation may include zero or more flags type declarations:\n\n```\n# use bitflags::bitflags;\nbitflags! {}\n\nbitflags! {\n struct Flags1: u8 {\n const A = 1;\n }\n\n struct Flags2: u8 {\n const A = 1;\n }\n}\n```\n\n# `impl` mode\n\nA declaration that begins with `impl` will only generate methods and trait implementations for the\n`struct` defined outside of the `bitflags` macro.\n\nThe struct itself must be a newtype using the bits type as its field.\n\nThe syntax for `impl` mode is identical to `struct` mode besides the starting token.\n\n## Examples\n\nImplement flags methods and traits for a custom flags type using `u8` as its underlying bits type:\n\n```\n# use bitflags::bitflags;\nstruct Flags(u8);\n\nbitflags! {\n impl Flags: u8 {\n const A = 1;\n const B = 1 << 1;\n const C = 0b0000_0100;\n }\n}\n```\n\n# Named and unnamed flags\n\nConstants in the body of a declaration are flags. The identifier of the constant is the name of\nthe flag. If the identifier is `_`, then the flag is unnamed. Unnamed flags don't appear in the\ngenerated API, but affect how bits are truncated.\n\n## Examples\n\nAdding an unnamed flag that makes all bits known:\n\n```\n# use bitflags::bitflags;\nbitflags! {\n struct Flags: u8 {\n const A = 1;\n const B = 1 << 1;\n\n const _ = !0;\n }\n}\n```\n\nFlags types may define multiple unnamed flags:\n\n```\n# use bitflags::bitflags;\nbitflags! {\n struct Flags: u8 {\n const _ = 1;\n const _ = 1 << 1;\n }\n}\n```\n*/\n#[macro_export]\nmacro_rules! bitflags {\n (\n $(#[$outer:meta])*\n $vis:vis struct $BitFlags:ident: $T:ty {\n $(\n $(#[$inner:ident $($args:tt)*])*\n const $Flag:tt = $value:expr;\n )*\n }\n\n $($t:tt)*\n ) => {\n // Declared in the scope of the `bitflags!` call\n // This type appears in the end-user's API\n $crate::__declare_public_bitflags! {\n $(#[$outer])*\n $vis struct $BitFlags\n }\n\n // Workaround for: https://github.com/bitflags/bitflags/issues/320\n // Pulled outside of the `const _: () = {}` block to avoid triggering an ICE\n $crate::__impl_public_bitflags_consts! {\n #[allow(\n dead_code,\n deprecated,\n unused_doc_comments,\n unused_attributes,\n unused_mut,\n unused_imports,\n non_upper_case_globals,\n clippy::min_ident_chars,\n clippy::assign_op_pattern,\n clippy::indexing_slicing,\n clippy::same_name_method,\n clippy::iter_without_into_iter,\n )]\n $BitFlags: $T {\n $(\n $(#[$inner $($args)*])*\n const $Flag = $value;\n )*\n }\n }\n\n #[allow(\n dead_code,\n deprecated,\n unused_doc_comments,\n unused_attributes,\n unused_mut,\n unused_imports,\n non_upper_case_globals,\n clippy::min_ident_chars,\n clippy::assign_op_pattern,\n clippy::indexing_slicing,\n clippy::same_name_method,\n clippy::iter_without_into_iter,\n )]\n const _: () = {\n // Declared in a \"hidden\" scope that can't be reached directly\n // These types don't appear in the end-user's API\n $crate::__declare_internal_bitflags! {\n $vis struct InternalBitFlags: $T\n }\n\n $crate::__impl_internal_bitflags! {\n InternalBitFlags: $T, $BitFlags {\n $(\n $(#[$inner $($args)*])*\n const $Flag = $value;\n )*\n }\n }\n\n // This is where new library trait implementations can be added\n $crate::__impl_external_bitflags! {\n InternalBitFlags: $T, $BitFlags {\n $(\n $(#[$inner $($args)*])*\n const $Flag;\n )*\n }\n }\n\n $crate::__impl_public_bitflags_forward! {\n $BitFlags: $T, InternalBitFlags\n }\n\n $crate::__impl_public_bitflags_ops! {\n $BitFlags\n }\n\n $crate::__impl_public_bitflags_iter! {\n $BitFlags: $T, $BitFlags\n }\n };\n\n $crate::bitflags! {\n $($t)*\n }\n };\n (\n $(#[$outer:meta])*\n impl $BitFlags:ident: $T:ty {\n $(\n $(#[$inner:ident $($args:tt)*])*\n const $Flag:tt = $value:expr;\n )*\n }\n\n $($t:tt)*\n ) => {\n // Workaround for: https://github.com/bitflags/bitflags/issues/320\n // Pulled outside of the `const _: () = {}` block to avoid triggering an ICE\n $crate::__impl_public_bitflags_consts! {\n #[allow(\n dead_code,\n deprecated,\n unused_doc_comments,\n unused_attributes,\n unused_mut,\n unused_imports,\n non_upper_case_globals,\n clippy::min_ident_chars,\n clippy::assign_op_pattern,\n clippy::iter_without_into_iter,\n )]\n $BitFlags: $T {\n $(\n $(#[$inner $($args)*])*\n const $Flag = $value;\n )*\n }\n }\n\n #[allow(\n dead_code,\n deprecated,\n unused_doc_comments,\n unused_attributes,\n unused_mut,\n unused_imports,\n non_upper_case_globals,\n clippy::min_ident_chars,\n clippy::assign_op_pattern,\n clippy::iter_without_into_iter,\n )]\n const _: () = {\n $crate::__impl_public_bitflags! {\n $(#[$outer])*\n $BitFlags: $T, $BitFlags {\n $(\n $(#[$inner $($args)*])*\n const $Flag = $value;\n )*\n }\n }\n\n $crate::__impl_public_bitflags_ops! {\n $BitFlags\n }\n\n $crate::__impl_public_bitflags_iter! {\n $BitFlags: $T, $BitFlags\n }\n };\n\n $crate::bitflags! {\n $($t)*\n }\n };\n () => {};\n}\n\n/// Implement functions on bitflags types.\n///\n/// We need to be careful about adding new methods and trait implementations here because they\n/// could conflict with items added by the end-user.\n#[macro_export]\n#[doc(hidden)]\nmacro_rules! __impl_bitflags {\n (\n // These param names must be passed in to make the macro work.\n // Just use `params: self, bits, name, other, value;`.\n params: $self:ident, $bits:ident, $name:ident, $other:ident, $value:ident;\n $(#[$outer:meta])*\n $PublicBitFlags:ident: $T:ty {\n fn empty() $empty_body:block\n fn all() $all_body:block\n fn bits(&self) $bits_body:block\n fn from_bits(bits) $from_bits_body:block\n fn from_bits_truncate(bits) $from_bits_truncate_body:block\n fn from_bits_retain(bits) $from_bits_retain_body:block\n fn from_name(name) $from_name_body:block\n fn is_empty(&self) $is_empty_body:block\n fn is_all(&self) $is_all_body:block\n fn intersects(&self, other) $intersects_body:block\n fn contains(&self, other) $contains_body:block\n fn insert(&mut self, other) $insert_body:block\n fn remove(&mut self, other) $remove_body:block\n fn toggle(&mut self, other) $toggle_body:block\n fn set(&mut self, other, value) $set_body:block\n fn intersection(self, other) $intersection_body:block\n fn union(self, other) $union_body:block\n fn difference(self, other) $difference_body:block\n fn symmetric_difference(self, other) $symmetric_difference_body:block\n fn complement(self) $complement_body:block\n }\n ) => {\n $(#[$outer])*\n impl $PublicBitFlags {\n /// Get a flags value with all bits unset.\n #[inline]\n pub const fn empty() -> Self\n $empty_body\n\n /// Get a flags value with all known bits set.\n #[inline]\n pub const fn all() -> Self\n $all_body\n\n /// Get the underlying bits value.\n ///\n /// The returned value is exactly the bits set in this flags value.\n #[inline]\n pub const fn bits(&$self) -> $T\n $bits_body\n\n /// Convert from a bits value.\n ///\n /// This method will return `None` if any unknown bits are set.\n #[inline]\n pub const fn from_bits($bits: $T) -> $crate::__private::core::option::Option\n $from_bits_body\n\n /// Convert from a bits value, unsetting any unknown bits.\n #[inline]\n pub const fn from_bits_truncate($bits: $T) -> Self\n $from_bits_truncate_body\n\n /// Convert from a bits value exactly.\n #[inline]\n pub const fn from_bits_retain($bits: $T) -> Self\n $from_bits_retain_body\n\n /// Get a flags value with the bits of a flag with the given name set.\n ///\n /// This method will return `None` if `name` is empty or doesn't\n /// correspond to any named flag.\n #[inline]\n pub fn from_name($name: &str) -> $crate::__private::core::option::Option\n $from_name_body\n\n /// Whether all bits in `self` are unset.\n #[inline]\n pub const fn is_empty(&$self) -> bool\n $is_empty_body\n\n /// Whether all known bits in this flags value are set.\n #[inline]\n pub const fn is_all(&$self) -> bool\n $is_all_body\n\n /// Whether any set bits in `other` are also set in `self`.\n #[inline]\n pub const fn intersects(&$self, $other: Self) -> bool\n $intersects_body\n\n /// Whether all set bits in `other` are also set in `self`.\n #[inline]\n pub const fn contains(&$self, $other: Self) -> bool\n $contains_body\n\n /// The bitwise or (`|`) of the bits in `self` and `other`.\n #[inline]\n pub fn insert(&mut $self, $other: Self)\n $insert_body\n\n /// The intersection of `self` with the complement of `other` (`&!`).\n ///\n /// This method is not equivalent to `self & !other` when `other` has unknown bits set.\n /// `remove` won't truncate `other`, but the `!` operator will.\n #[inline]\n pub fn remove(&mut $self, $other: Self)\n $remove_body\n\n /// The bitwise exclusive-or (`^`) of the bits in `self` and `other`.\n #[inline]\n pub fn toggle(&mut $self, $other: Self)\n $toggle_body\n\n /// Call `insert` when `value` is `true` or `remove` when `value` is `false`.\n #[inline]\n pub fn set(&mut $self, $other: Self, $value: bool)\n $set_body\n\n /// The bitwise and (`&`) of the bits in `self` and `other`.\n #[inline]\n #[must_use]\n pub const fn intersection($self, $other: Self) -> Self\n $intersection_body\n\n /// The bitwise or (`|`) of the bits in `self` and `other`.\n #[inline]\n #[must_use]\n pub const fn union($self, $other: Self) -> Self\n $union_body\n\n /// The intersection of `self` with the complement of `other` (`&!`).\n ///\n /// This method is not equivalent to `self & !other` when `other` has unknown bits set.\n /// `difference` won't truncate `other`, but the `!` operator will.\n #[inline]\n #[must_use]\n pub const fn difference($self, $other: Self) -> Self\n $difference_body\n\n /// The bitwise exclusive-or (`^`) of the bits in `self` and `other`.\n #[inline]\n #[must_use]\n pub const fn symmetric_difference($self, $other: Self) -> Self\n $symmetric_difference_body\n\n /// The bitwise negation (`!`) of the bits in `self`, truncating the result.\n #[inline]\n #[must_use]\n pub const fn complement($self) -> Self\n $complement_body\n }\n };\n}\n\n/// A macro that matches flags values, similar to Rust's `match` statement.\n///\n/// In a regular `match` statement, the syntax `Flag::A | Flag::B` is interpreted as an or-pattern,\n/// instead of the bitwise-or of `Flag::A` and `Flag::B`. This can be surprising when combined with flags types\n/// because `Flag::A | Flag::B` won't match the pattern `Flag::A | Flag::B`. This macro is an alternative to\n/// `match` for flags values that doesn't have this issue.\n///\n/// # Syntax\n///\n/// ```ignore\n/// bitflags_match!(expression, {\n/// pattern1 => result1,\n/// pattern2 => result2,\n/// ..\n/// _ => default_result,\n/// })\n/// ```\n///\n/// The final `_ => default_result` arm is required, otherwise the macro will fail to compile.\n///\n/// # Examples\n///\n/// ```rust\n/// use bitflags::{bitflags, bitflags_match};\n///\n/// bitflags! {\n/// #[derive(PartialEq)]\n/// struct Flags: u8 {\n/// const A = 1 << 0;\n/// const B = 1 << 1;\n/// const C = 1 << 2;\n/// }\n/// }\n///\n/// let flags = Flags::A | Flags::B;\n///\n/// // Prints `the value is A and B`\n/// bitflags_match!(flags, {\n/// Flags::A | Flags::B => println!(\"the value is A and B\"),\n/// _ => println!(\"the value is not A and B\"),\n/// });\n///\n/// // Prints `the value is not A`\n/// bitflags_match!(flags, {\n/// Flags::A => println!(\"the value is A\"),\n/// _ => println!(\"the value is not A\"),\n/// });\n/// ```\n///\n/// # How it works\n///\n/// The macro expands to a series of `if` statements, **checking equality** between the input expression\n/// and each pattern. This allows for correct matching of bitflag combinations, which is not possible\n/// with a regular match expression due to the way bitflags are implemented.\n///\n/// Patterns are evaluated in the order they appear in the macro.\n#[macro_export]\nmacro_rules! bitflags_match {\n ($operation:expr, {\n $($t:tt)*\n }) => {\n // Expand to a closure so we can use `return`\n // This makes it possible to apply attributes to the \"match arms\"\n (|| {\n $crate::__bitflags_match!($operation, { $($t)* })\n })()\n };\n}\n\n/// Expand the `bitflags_match` macro\n#[macro_export]\n#[doc(hidden)]\nmacro_rules! __bitflags_match {\n // Eat an optional `,` following a block match arm\n ($operation:expr, { $pattern:expr => { $($body:tt)* } , $($t:tt)+ }) => {\n $crate::__bitflags_match!($operation, { $pattern => { $($body)* } $($t)+ })\n };\n // Expand a block match arm `A => { .. }`\n ($operation:expr, { $pattern:expr => { $($body:tt)* } $($t:tt)+ }) => {\n {\n if $operation == $pattern {\n return {\n $($body)*\n };\n }\n\n $crate::__bitflags_match!($operation, { $($t)+ })\n }\n };\n // Expand an expression match arm `A => x,`\n ($operation:expr, { $pattern:expr => $body:expr , $($t:tt)+ }) => {\n {\n if $operation == $pattern {\n return $body;\n }\n\n $crate::__bitflags_match!($operation, { $($t)+ })\n }\n };\n // Expand the default case\n ($operation:expr, { _ => $default:expr $(,)? }) => {\n $default\n }\n}\n\n/// Implement a flag, which may be a wildcard `_`.\n///\n/// Named flags will emit the `named` block, and unnamed flags will emit the `unnamed` block.\n#[macro_export]\n#[doc(hidden)]\nmacro_rules! __bitflags_flag {\n (\n {\n name: _,\n named: { $($named:tt)* },\n unnamed: { $($unnamed:tt)* },\n }\n ) => {\n $($unnamed)*\n };\n (\n {\n name: $Flag:ident,\n named: { $($named:tt)* },\n unnamed: { $($unnamed:tt)* },\n }\n ) => {\n $($named)*\n };\n}\n\n/*\nAttribute inspection macros\n\nThe following macros all use the same pattern for searching for specific attributes and transforming\na target token tree. They're implementations of _token-tree munchers_, where each token from a source\nset is matched one-at-a-time until the input is exhausted, at which point the final result is emitted.\n\nThe first match is the entrypoint for the macro with user syntax.\n\nSubsequent matches pull tokens from `unprocessed` and do something with them. That might be moving\nthem into `processed` to be emitted later, or manipulating a target item/expression. The logic of\nthe macro is implemented in these middle matches.\n\nThe final match is the exitpoint, where `unprocessed` is empty.\n*/\n\n/// A macro that processes the input to `bitflags!` and shuffles attributes around\n/// based on whether or not they're \"expression-safe\".\n///\n/// This macro is a token-tree muncher that works on 2 levels:\n///\n/// For each attribute, we explicitly match on its identifier, like `cfg` to determine\n/// whether or not it should be considered expression-safe.\n///\n/// If you find yourself with an attribute that should be considered expression-safe\n/// and isn't, it can be added here.\n#[macro_export]\n#[doc(hidden)]\nmacro_rules! __bitflags_expr_safe_attrs {\n (\n $(#[$inner:ident $($args:tt)*])*\n { $e:expr }\n ) => {\n $crate::__bitflags_expr_safe_attrs! {\n expr: { $e },\n attrs: {\n // All attributes start here\n unprocessed: [$(#[$inner $($args)*])*],\n // Attributes that are safe on expressions go here\n processed: [],\n },\n }\n };\n // `cfg`: propagate\n (\n expr: { $e:expr },\n attrs: {\n unprocessed: [\n #[cfg $($args:tt)*]\n $($attrs_rest:tt)*\n ],\n processed: [$($expr:tt)*],\n },\n ) => {\n $crate::__bitflags_expr_safe_attrs! {\n expr: { $e },\n attrs: {\n unprocessed: [\n $($attrs_rest)*\n ],\n processed: [\n $($expr)*\n #[cfg $($args)*]\n ],\n },\n }\n };\n // Other: discard\n (\n expr: { $e:expr },\n attrs: {\n unprocessed: [\n #[$other:ident $($args:tt)*]\n $($attrs_rest:tt)*\n ],\n processed: [$($expr:tt)*],\n },\n ) => {\n $crate::__bitflags_expr_safe_attrs! {\n expr: { $e },\n attrs: {\n unprocessed: [\n $($attrs_rest)*\n ],\n processed: [\n $($expr)*\n ],\n },\n }\n };\n // Finished\n (\n expr: { $e:expr },\n attrs: {\n unprocessed: [],\n processed: [$(#[$expr:ident $($exprargs:tt)*])*],\n },\n ) => {\n $(#[$expr $($exprargs)*])*\n { $e }\n }\n}\n\n/// A macro that processes the input to `bitflags!` and shuffles attributes around\n/// based on whether or not they're \"item-safe\".\n///\n/// This macro follows the same pattern as expr-safe above, but assumes all attributes\n/// are safe on items. It only filters out any `bitflags`-defined attributes.\n#[macro_export]\n#[doc(hidden)]\nmacro_rules! __bitflags_item_safe_attrs {\n (\n $(#[$inner:ident $($args:tt)*])*\n { $i:item }\n ) => {\n $crate::__bitflags_item_safe_attrs! {\n item: { $i },\n attrs: {\n // All attributes start here\n unprocessed: [$(#[$inner $($args)*])*],\n // Attributes that are safe on items go here\n processed: [],\n },\n }\n };\n // `bitflags`: discard\n (\n item: { $i:item },\n attrs: {\n unprocessed: [\n #[bitflags $($args:tt)*]\n $($attrs_rest:tt)*\n ],\n processed: [$($item:tt)*],\n },\n ) => {\n $crate::__bitflags_item_safe_attrs! {\n item: { $i },\n attrs: {\n unprocessed: [\n $($attrs_rest)*\n ],\n processed: [\n $($item)*\n ],\n },\n }\n };\n // Other: propagate\n (\n item: { $i:item },\n attrs: {\n unprocessed: [\n // $other matched here\n #[$other:ident $($args:tt)*]\n $($attrs_rest:tt)*\n ],\n processed: [$($item:tt)*],\n },\n ) => {\n $crate::__bitflags_item_safe_attrs! {\n item: { $i },\n attrs: {\n unprocessed: [\n $($attrs_rest)*\n ],\n processed: [\n $($item)*\n #[$other $($args)*]\n ],\n },\n }\n };\n // Finished\n (\n item: { $i:item },\n attrs: {\n unprocessed: [],\n processed: [$(#[$item:ident $($itemargs:tt)*])*],\n },\n ) => {\n $(#[$item $($itemargs)*])*\n $i\n }\n}\n\n/// Determine the name to assign to a flag.\n#[macro_export]\n#[doc(hidden)]\nmacro_rules! __bitflags_flag_name {\n // Unnamed\n (\n $(#[$inner:ident $($args:tt)*])*\n { $vis:vis const _ = _ }\n ) => {\n\n };\n (\n $(#[$inner:ident $($args:tt)*])*\n { $vis:vis const $binding:ident = $name:expr }\n ) => {\n $crate::__bitflags_flag_name! {\n item: { $vis const $binding = $crate::__private::core::stringify!($name) },\n attrs: {\n // All attributes start here\n unprocessed: [$(#[$inner $($args)*])*],\n // Attributes that are safe on the flag name go here\n processed: [],\n },\n }\n };\n // `bitflags(flag_name)`: set the name\n (\n item: { $vis:vis const $binding:ident = $name:expr },\n attrs: {\n unprocessed: [\n #[bitflags(flag_name = $flag_name:expr)]\n $($attrs_rest:tt)*\n ],\n processed: [$($item:tt)*],\n },\n ) => {\n $crate::__bitflags_flag_name! {\n item: { $vis const $binding = $flag_name },\n attrs: {\n unprocessed: [\n $($attrs_rest)*\n ],\n processed: [\n $($item)*\n ],\n },\n }\n };\n // `cfg`: propagate\n (\n item: { $vis:vis const $binding:ident = $name:expr },\n attrs: {\n unprocessed: [\n #[cfg $($args:tt)*]\n $($attrs_rest:tt)*\n ],\n processed: [$($item:tt)*],\n },\n ) => {\n $crate::__bitflags_flag_name! {\n item: { $vis const $binding = $name },\n attrs: {\n unprocessed: [\n $($attrs_rest)*\n ],\n processed: [\n $($item)*\n #[cfg $($args)*]\n ],\n },\n }\n };\n // Other: discard\n (\n item: { $vis:vis const $binding:ident = $name:expr },\n attrs: {\n unprocessed: [\n #[$other:ident $($args:tt)*]\n $($attrs_rest:tt)*\n ],\n processed: [$($item:tt)*],\n },\n ) => {\n $crate::__bitflags_flag_name! {\n item: { $vis const $binding = $name },\n attrs: {\n unprocessed: [\n $($attrs_rest)*\n ],\n processed: [$($item)*],\n },\n }\n };\n // Finished\n (\n item: { $vis:vis const $binding:ident = $name:expr },\n attrs: {\n unprocessed: [],\n processed: [$(#[$item:ident $($itemargs:tt)*])*],\n },\n ) => {\n $(#[$item $($itemargs)*])*\n $vis const $binding: &'static str = $name;\n }\n}\n\n#[macro_use]\nmod public;\n#[macro_use]\nmod internal;\n#[macro_use]\nmod external;\n\n#[cfg(feature = \"example_generated\")]\npub mod example_generated;\n\n#[cfg(test)]\nmod tests;\n" }, { "package": "bitvec@1.1.1", "sourcePath": "LICENSE.txt", "sha256": "411781fd38700f2357a14126d0ab048164ab881f1dcb335c1bb932e232c9a2f5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "MIT License\n\nCopyright (c) 2018 myrrlyn (Alexander Payne)\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "package": "brotli@3.5.0", "sourcePath": "src/enc/entropy_encode.rs", "sha256": "46fe7c70f68ed2146933ddc63d9c5bbe40ff53c62b5d1c19e4ffdc32b9421197", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* Copyright 2010 Google Inc. All Rights Reserved.\n\n Distributed under MIT license.\n See file LICENSE for detail or copy at https://opensource.org/licenses/MIT\n*/\n\n/* Entropy encoding (Huffman) utilities. */\n#![allow(dead_code)]\nuse super::util::brotli_max_uint32_t;\n\n#[derive(Clone, Copy, Default)]\npub struct HuffmanTree {\n pub total_count_: u32,\n pub index_left_: i16,\n pub index_right_or_value_: i16,\n}\n\npub fn NewHuffmanTree(count: u32, left: i16, right: i16) -> HuffmanTree {\n HuffmanTree {\n total_count_: count,\n index_left_: left,\n index_right_or_value_: right,\n }\n}\npub fn InitHuffmanTree(xself: &mut HuffmanTree, count: u32, left: i16, right: i16) {\n *xself = NewHuffmanTree(count, left, right);\n}\n\npub fn BrotliSetDepth(p0: i32, pool: &mut [HuffmanTree], depth: &mut [u8], max_depth: i32) -> bool {\n let mut stack: [i32; 16] = [0; 16];\n let mut level: i32 = 0i32;\n let mut p: i32 = p0;\n 0i32;\n stack[0] = -1i32;\n loop {\n if (pool[(p as usize)]).index_left_ as i32 >= 0i32 {\n level += 1;\n if level > max_depth {\n return false;\n }\n stack[level as usize] = (pool[(p as usize)]).index_right_or_value_ as i32;\n p = (pool[(p as usize)]).index_left_ as i32;\n {\n {\n continue;\n }\n }\n } else {\n let pp = pool[(p as usize)];\n depth[((pp).index_right_or_value_ as usize)] = level as u8;\n }\n while level >= 0i32 && (stack[level as usize] == -1i32) {\n level -= 1;\n }\n if level < 0i32 {\n return true;\n }\n p = stack[level as usize];\n stack[level as usize] = -1i32;\n }\n}\n\npub trait HuffmanComparator {\n fn Cmp(&self, a: &HuffmanTree, b: &HuffmanTree) -> bool;\n}\npub struct SortHuffmanTree {}\nimpl HuffmanComparator for SortHuffmanTree {\n fn Cmp(&self, v0: &HuffmanTree, v1: &HuffmanTree) -> bool {\n if v0.total_count_ != v1.total_count_ {\n v0.total_count_ < v1.total_count_\n } else {\n v0.index_right_or_value_ > v1.index_right_or_value_\n }\n }\n}\npub fn SortHuffmanTreeItems(\n items: &mut [HuffmanTree],\n n: usize,\n comparator: Comparator,\n) {\n static gaps: [usize; 6] = [132, 57, 23, 10, 4, 1];\n if n < 13usize {\n let mut i: usize;\n i = 1;\n while i < n {\n {\n let mut tmp: HuffmanTree = items[i];\n let mut k: usize = i;\n let mut j: usize = i.wrapping_sub(1);\n while comparator.Cmp(&mut tmp, &mut items[j]) {\n items[k] = items[j];\n k = j;\n if {\n let _old = j;\n j = j.wrapping_sub(1);\n _old\n } == 0\n {\n {\n break;\n }\n }\n }\n items[k] = tmp;\n }\n i = i.wrapping_add(1);\n }\n } else {\n let mut g: i32 = if n < 57usize { 2i32 } else { 0i32 };\n while g < 6i32 {\n {\n let gap: usize = gaps[g as usize];\n let mut i: usize;\n i = gap;\n while i < n {\n {\n let mut j: usize = i;\n let mut tmp: HuffmanTree = items[i];\n while j >= gap\n && (comparator.Cmp(&mut tmp, &mut items[j.wrapping_sub(gap)]))\n {\n {\n items[j] = items[j.wrapping_sub(gap)];\n }\n j = j.wrapping_sub(gap);\n }\n items[j] = tmp;\n }\n i = i.wrapping_add(1);\n }\n }\n g += 1;\n }\n }\n}\n\n/* This function will create a Huffman tree.\n\nThe catch here is that the tree cannot be arbitrarily deep.\nBrotli specifies a maximum depth of 15 bits for \"code trees\"\nand 7 bits for \"code length code trees.\"\n\ncount_limit is the value that is to be faked as the minimum value\nand this minimum value is raised until the tree matches the\nmaximum length requirement.\n\nThis algorithm is not of excellent performance for very long data blocks,\nespecially when population counts are longer than 2**tree_limit, but\nwe are not planning to use this with extremely long blocks.\n\nSee https://en.wikipedia.org/wiki/Huffman_coding */\npub fn BrotliCreateHuffmanTree(\n data: &[u32],\n length: usize,\n tree_limit: i32,\n tree: &mut [HuffmanTree],\n depth: &mut [u8],\n) {\n let mut count_limit: u32;\n let mut sentinel: HuffmanTree = HuffmanTree {\n total_count_: 0,\n index_left_: 0,\n index_right_or_value_: 0,\n };\n InitHuffmanTree(&mut sentinel, !(0u32), -1i16, -1i16);\n count_limit = 1u32;\n 'break1: loop {\n {\n let mut n: usize = 0usize;\n let mut i: usize;\n let mut j: usize;\n let mut k: usize;\n i = length;\n while i != 0usize {\n i = i.wrapping_sub(1);\n if data[i] != 0 {\n let count: u32 = brotli_max_uint32_t(data[i], count_limit);\n InitHuffmanTree(&mut tree[n], count, -1i16, i as i16);\n n = n.wrapping_add(1);\n }\n }\n if n == 1 {\n depth[((tree[0]).index_right_or_value_ as usize)] = 1u8;\n {\n {\n break 'break1;\n }\n }\n }\n SortHuffmanTreeItems(tree, n, SortHuffmanTree {});\n tree[n] = sentinel;\n tree[n.wrapping_add(1)] = sentinel;\n i = 0usize;\n j = n.wrapping_add(1);\n k = n.wrapping_sub(1);\n while k != 0usize {\n {\n let left: usize;\n let right: usize;\n if (tree[i]).total_count_ <= (tree[j]).total_count_ {\n left = i;\n i = i.wrapping_add(1);\n } else {\n left = j;\n j = j.wrapping_add(1);\n }\n if (tree[i]).total_count_ <= (tree[j]).total_count_ {\n right = i;\n i = i.wrapping_add(1);\n } else {\n right = j;\n j = j.wrapping_add(1);\n }\n {\n let j_end: usize = (2usize).wrapping_mul(n).wrapping_sub(k);\n (tree[j_end]).total_count_ = (tree[left])\n .total_count_\n .wrapping_add((tree[right]).total_count_);\n (tree[j_end]).index_left_ = left as i16;\n (tree[j_end]).index_right_or_value_ = right as i16;\n tree[j_end.wrapping_add(1)] = sentinel;\n }\n }\n k = k.wrapping_sub(1);\n }\n if BrotliSetDepth(\n (2usize).wrapping_mul(n).wrapping_sub(1) as i32,\n tree,\n depth,\n tree_limit,\n ) {\n {\n break 'break1;\n }\n }\n }\n count_limit = count_limit.wrapping_mul(2);\n }\n}\npub fn BrotliOptimizeHuffmanCountsForRle(\n mut length: usize,\n counts: &mut [u32],\n good_for_rle: &mut [u8],\n) {\n let mut nonzero_count: usize = 0usize;\n let mut stride: usize;\n let mut limit: usize;\n let mut sum: usize;\n let streak_limit: usize = 1240usize;\n let mut i: usize;\n i = 0usize;\n while i < length {\n {\n if counts[i] != 0 {\n nonzero_count = nonzero_count.wrapping_add(1);\n }\n }\n i = i.wrapping_add(1);\n }\n if nonzero_count < 16usize {\n return;\n }\n while length != 0usize && (counts[length.wrapping_sub(1)] == 0u32) {\n length = length.wrapping_sub(1);\n }\n if length == 0usize {\n return;\n }\n {\n let mut nonzeros: usize = 0usize;\n let mut smallest_nonzero: u32 = (1i32 << 30) as u32;\n i = 0usize;\n while i < length {\n {\n if counts[i] != 0u32 {\n nonzeros = nonzeros.wrapping_add(1);\n if smallest_nonzero > counts[i] {\n smallest_nonzero = counts[i];\n }\n }\n }\n i = i.wrapping_add(1);\n }\n if nonzeros < 5usize {\n return;\n }\n if smallest_nonzero < 4u32 {\n let zeros: usize = length.wrapping_sub(nonzeros);\n if zeros < 6usize {\n i = 1;\n while i < length.wrapping_sub(1) {\n {\n if counts[i.wrapping_sub(1)] != 0u32\n && (counts[i] == 0u32)\n && (counts[i.wrapping_add(1)] != 0u32)\n {\n counts[i] = 1u32;\n }\n }\n i = i.wrapping_add(1);\n }\n }\n }\n if nonzeros < 28usize {\n return;\n }\n }\n for rle_item in good_for_rle.iter_mut() {\n *rle_item = 0;\n }\n {\n let mut symbol: u32 = counts[0];\n let mut step: usize = 0usize;\n i = 0usize;\n while i <= length {\n {\n if i == length || counts[i] != symbol {\n if symbol == 0u32 && (step >= 5usize) || symbol != 0u32 && (step >= 7usize) {\n let mut k: usize;\n k = 0usize;\n while k < step {\n {\n good_for_rle[i.wrapping_sub(k).wrapping_sub(1)] = 1u8;\n }\n k = k.wrapping_add(1);\n }\n }\n step = 1;\n if i != length {\n symbol = counts[i];\n }\n } else {\n step = step.wrapping_add(1);\n }\n }\n i = i.wrapping_add(1);\n }\n }\n stride = 0usize;\n limit = (256u32)\n .wrapping_mul((counts[0]).wrapping_add(counts[1]).wrapping_add(counts[2]))\n .wrapping_div(3)\n .wrapping_add(420) as usize;\n sum = 0usize;\n i = 0usize;\n while i <= length {\n {\n if i == length\n || good_for_rle[i] != 0\n || i != 0usize && (good_for_rle[i.wrapping_sub(1)] != 0)\n || ((256u32).wrapping_mul(counts[i]) as usize)\n .wrapping_sub(limit)\n .wrapping_add(streak_limit)\n >= (2usize).wrapping_mul(streak_limit)\n {\n if stride >= 4usize || stride >= 3usize && (sum == 0usize) {\n let mut k: usize;\n let mut count: usize = sum\n .wrapping_add(stride.wrapping_div(2))\n .wrapping_div(stride);\n if count == 0usize {\n count = 1;\n }\n if sum == 0usize {\n count = 0usize;\n }\n k = 0usize;\n while k < stride {\n {\n counts[i.wrapping_sub(k).wrapping_sub(1)] = count as u32;\n }\n k = k.wrapping_add(1);\n }\n }\n stride = 0usize;\n sum = 0usize;\n if i < length.wrapping_sub(2) {\n limit = (256u32)\n .wrapping_mul(\n (counts[i])\n .wrapping_add(counts[i.wrapping_add(1)])\n .wrapping_add(counts[i.wrapping_add(2)]),\n )\n .wrapping_div(3)\n .wrapping_add(420) as usize;\n } else if i < length {\n limit = (256u32).wrapping_mul(counts[i]) as usize;\n } else {\n limit = 0usize;\n }\n }\n stride = stride.wrapping_add(1);\n if i != length {\n sum = sum.wrapping_add(counts[i] as usize);\n if stride >= 4usize {\n limit = (256usize)\n .wrapping_mul(sum)\n .wrapping_add(stride.wrapping_div(2))\n .wrapping_div(stride);\n }\n if stride == 4usize {\n limit = limit.wrapping_add(120);\n }\n }\n }\n i = i.wrapping_add(1);\n }\n}\n\npub fn DecideOverRleUse(\n depth: &[u8],\n length: usize,\n use_rle_for_non_zero: &mut i32,\n use_rle_for_zero: &mut i32,\n) {\n let mut total_reps_zero: usize = 0usize;\n let mut total_reps_non_zero: usize = 0usize;\n let mut count_reps_zero: usize = 1;\n let mut count_reps_non_zero: usize = 1;\n let mut i: usize;\n i = 0usize;\n while i < length {\n let value: u8 = depth[i];\n let mut reps: usize = 1;\n let mut k: usize;\n k = i.wrapping_add(1);\n while k < length && (depth[k] as i32 == value as i32) {\n {\n reps = reps.wrapping_add(1);\n }\n k = k.wrapping_add(1);\n }\n if reps >= 3usize && (value as i32 == 0i32) {\n total_reps_zero = total_reps_zero.wrapping_add(reps);\n count_reps_zero = count_reps_zero.wrapping_add(1);\n }\n if reps >= 4usize && (value as i32 != 0i32) {\n total_reps_non_zero = total_reps_non_zero.wrapping_add(reps);\n count_reps_non_zero = count_reps_non_zero.wrapping_add(1);\n }\n i = i.wrapping_add(reps);\n }\n *use_rle_for_non_zero = if total_reps_non_zero > count_reps_non_zero.wrapping_mul(2) {\n 1i32\n } else {\n 0i32\n };\n *use_rle_for_zero = if total_reps_zero > count_reps_zero.wrapping_mul(2) {\n 1i32\n } else {\n 0i32\n };\n}\n\nfn Reverse(v: &mut [u8], mut start: usize, mut end: usize) {\n end = end.wrapping_sub(1);\n while start < end {\n v.swap(start, end);\n start = start.wrapping_add(1);\n end = end.wrapping_sub(1);\n }\n}\n\nfn BrotliWriteHuffmanTreeRepetitions(\n previous_value: u8,\n value: u8,\n mut repetitions: usize,\n tree_size: &mut usize,\n tree: &mut [u8],\n extra_bits_data: &mut [u8],\n) {\n 0i32;\n if previous_value as i32 != value as i32 {\n tree[*tree_size] = value;\n extra_bits_data[*tree_size] = 0u8;\n *tree_size = tree_size.wrapping_add(1);\n repetitions = repetitions.wrapping_sub(1);\n }\n if repetitions == 7usize {\n tree[*tree_size] = value;\n extra_bits_data[*tree_size] = 0u8;\n *tree_size = tree_size.wrapping_add(1);\n repetitions = repetitions.wrapping_sub(1);\n }\n if repetitions < 3usize {\n let mut i: usize;\n i = 0usize;\n while i < repetitions {\n {\n tree[*tree_size] = value;\n extra_bits_data[*tree_size] = 0u8;\n *tree_size = tree_size.wrapping_add(1);\n }\n i = i.wrapping_add(1);\n }\n } else {\n let start: usize = *tree_size;\n repetitions = repetitions.wrapping_sub(3);\n loop {\n tree[*tree_size] = 16u8;\n extra_bits_data[*tree_size] = (repetitions & 0x3usize) as u8;\n *tree_size = tree_size.wrapping_add(1);\n repetitions >>= 2i32;\n if repetitions == 0usize {\n {\n break;\n }\n }\n repetitions = repetitions.wrapping_sub(1);\n }\n Reverse(tree, start, *tree_size);\n Reverse(extra_bits_data, start, *tree_size);\n }\n}\n\nfn BrotliWriteHuffmanTreeRepetitionsZeros(\n mut repetitions: usize,\n tree_size: &mut usize,\n tree: &mut [u8],\n extra_bits_data: &mut [u8],\n) {\n if repetitions == 11 {\n tree[*tree_size] = 0u8;\n extra_bits_data[*tree_size] = 0u8;\n *tree_size = tree_size.wrapping_add(1);\n repetitions = repetitions.wrapping_sub(1);\n }\n if repetitions < 3usize {\n let mut i: usize;\n i = 0usize;\n while i < repetitions {\n {\n tree[*tree_size] = 0u8;\n extra_bits_data[*tree_size] = 0u8;\n *tree_size = tree_size.wrapping_add(1);\n }\n i = i.wrapping_add(1);\n }\n } else {\n let start: usize = *tree_size;\n repetitions = repetitions.wrapping_sub(3);\n loop {\n tree[*tree_size] = 17u8;\n extra_bits_data[*tree_size] = (repetitions & 0x7usize) as u8;\n *tree_size = tree_size.wrapping_add(1);\n repetitions >>= 3i32;\n if repetitions == 0usize {\n {\n break;\n }\n }\n repetitions = repetitions.wrapping_sub(1);\n }\n Reverse(tree, start, *tree_size);\n Reverse(extra_bits_data, start, *tree_size);\n }\n}\n\npub fn BrotliWriteHuffmanTree(\n depth: &[u8],\n length: usize,\n tree_size: &mut usize,\n tree: &mut [u8],\n extra_bits_data: &mut [u8],\n) {\n let mut previous_value: u8 = 8u8;\n let mut i: usize;\n let mut use_rle_for_non_zero: i32 = 0i32;\n let mut use_rle_for_zero: i32 = 0i32;\n let mut new_length: usize = length;\n i = 0usize;\n 'break27: while i < length {\n {\n if depth[length.wrapping_sub(i).wrapping_sub(1)] as i32 == 0i32 {\n new_length = new_length.wrapping_sub(1);\n } else {\n break 'break27;\n }\n }\n i = i.wrapping_add(1);\n }\n if length > 50usize {\n DecideOverRleUse(\n depth,\n new_length,\n &mut use_rle_for_non_zero,\n &mut use_rle_for_zero,\n );\n }\n i = 0usize;\n while i < new_length {\n let value: u8 = depth[i];\n let mut reps: usize = 1;\n if value as i32 != 0i32 && (use_rle_for_non_zero != 0)\n || value as i32 == 0i32 && (use_rle_for_zero != 0)\n {\n let mut k: usize;\n k = i.wrapping_add(1);\n while k < new_length && (depth[k] as i32 == value as i32) {\n {\n reps = reps.wrapping_add(1);\n }\n k = k.wrapping_add(1);\n }\n }\n if value as i32 == 0i32 {\n BrotliWriteHuffmanTreeRepetitionsZeros(reps, tree_size, tree, extra_bits_data);\n } else {\n BrotliWriteHuffmanTreeRepetitions(\n previous_value,\n value,\n reps,\n tree_size,\n tree,\n extra_bits_data,\n );\n previous_value = value;\n }\n i = i.wrapping_add(reps);\n }\n}\n\nfn BrotliReverseBits(num_bits: usize, mut bits: u16) -> u16 {\n static kLut: [usize; 16] = [\n 0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe, 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf,\n ];\n let mut retval: usize = kLut[(bits as i32 & 0xfi32) as usize];\n let mut i: usize;\n i = 4usize;\n while i < num_bits {\n {\n retval <<= 4i32;\n bits = (bits as i32 >> 4) as u16;\n retval |= kLut[(bits as i32 & 0xfi32) as usize];\n }\n i = i.wrapping_add(4);\n }\n retval >>= (0usize.wrapping_sub(num_bits) & 0x3usize);\n retval as u16\n}\nconst MAX_HUFFMAN_BITS: usize = 16;\npub fn BrotliConvertBitDepthsToSymbols(depth: &[u8], len: usize, bits: &mut [u16]) {\n /* In Brotli, all bit depths are [1..15]\n 0 bit depth means that the symbol does not exist. */\n\n let mut bl_count: [u16; MAX_HUFFMAN_BITS] = [0; MAX_HUFFMAN_BITS];\n let mut next_code: [u16; MAX_HUFFMAN_BITS] = [0; MAX_HUFFMAN_BITS];\n let mut i: usize;\n let mut code: i32 = 0i32;\n i = 0usize;\n while i < len {\n {\n let _rhs = 1;\n let _lhs = &mut bl_count[depth[i] as usize];\n *_lhs = (*_lhs as i32 + _rhs) as u16;\n }\n i = i.wrapping_add(1);\n }\n bl_count[0] = 0u16;\n next_code[0] = 0u16;\n i = 1;\n while i < MAX_HUFFMAN_BITS {\n {\n code = (code + bl_count[i.wrapping_sub(1)] as i32) << 1;\n next_code[i] = code as u16;\n }\n i = i.wrapping_add(1);\n }\n i = 0usize;\n while i < len {\n {\n if depth[i] != 0 {\n bits[i] = BrotliReverseBits(depth[i] as usize, {\n let _rhs = 1;\n let _lhs = &mut next_code[depth[i] as usize];\n let _old = *_lhs;\n *_lhs = (*_lhs as i32 + _rhs) as u16;\n _old\n });\n }\n }\n i = i.wrapping_add(1);\n }\n}\n" }, { "package": "brotli-decompressor@2.5.1", "sourcePath": "src/context.rs", "sha256": "8fe7865962916fa7ce820b9fb407e6a2dd9b9c373191ad4a41c507b6be373e40", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "#![allow(non_snake_case)]\n#![allow(non_upper_case_globals)]\n#![allow(non_camel_case_types)]\n\n// Copyright 2013 Google Inc. All Rights Reserved.\n//\n// Distributed under MIT license.\n// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT\n//\n\n// Lookup table to map the previous two bytes to a context id.\n//\n// There are four different context modeling modes defined here:\n// CONTEXT_LSB6: context id is the least significant 6 bits of the last byte,\n// CONTEXT_MSB6: context id is the most significant 6 bits of the last byte,\n// CONTEXT_UTF8: second-order context model tuned for UTF8-encoded text,\n// CONTEXT_SIGNED: second-order context model tuned for signed integers.\n//\n// The context id for the UTF8 context model is calculated as follows. If p1\n// and p2 are the previous two bytes, we calculate the context as\n//\n// context = kContextLookup[p1] | kContextLookup[p2 + 256].\n//\n// If the previous two bytes are ASCII characters (i.e. < 128), this will be\n// equivalent to\n//\n// context = 4 * context1(p1) + context2(p2),\n//\n// where context1 is based on the previous byte in the following way:\n//\n// 0 : non-ASCII control\n// 1 : \\t, \\n, \\r\n// 2 : space\n// 3 : other punctuation\n// 4 : \" '\n// 5 : %\n// 6 : ( < [ {\n// 7 : ) > ] }\n// 8 : , ; :\n// 9 : .\n// 10 : =\n// 11 : number\n// 12 : upper-case vowel\n// 13 : upper-case consonant\n// 14 : lower-case vowel\n// 15 : lower-case consonant\n//\n// and context2 is based on the second last byte:\n//\n// 0 : control, space\n// 1 : punctuation\n// 2 : upper-case letter, number\n// 3 : lower-case letter\n//\n// If the last byte is ASCII, and the second last byte is not (in a valid UTF8\n// stream it will be a continuation byte, value between 128 and 191), the\n// context is the same as if the second last byte was an ASCII control or space.\n//\n// If the last byte is a UTF8 lead byte (value >= 192), then the next byte will\n// be a continuation byte and the context id is 2 or 3 depending on the LSB of\n// the last byte and to a lesser extent on the second last byte if it is ASCII.\n//\n// If the last byte is a UTF8 continuation byte, the second last byte can be:\n// - continuation byte: the next byte is probably ASCII or lead byte (assuming\n// 4-byte UTF8 characters are rare) and the context id is 0 or 1.\n// - lead byte (192 - 207): next byte is ASCII or lead byte, context is 0 or 1\n// - lead byte (208 - 255): next byte is continuation byte, context is 2 or 3\n//\n// The possible value combinations of the previous two bytes, the range of\n// context ids and the type of the next byte is summarized in the table below:\n//\n// |--------\\-----------------------------------------------------------------|\n// | \\ Last byte |\n// | Second \\---------------------------------------------------------------|\n// | last byte \\ ASCII | cont. byte | lead byte |\n// | \\ (0-127) | (128-191) | (192-) |\n// |=============|===================|=====================|==================|\n// | ASCII | next: ASCII/lead | not valid | next: cont. |\n// | (0-127) | context: 4 - 63 | | context: 2 - 3 |\n// |-------------|-------------------|---------------------|------------------|\n// | cont. byte | next: ASCII/lead | next: ASCII/lead | next: cont. |\n// | (128-191) | context: 4 - 63 | context: 0 - 1 | context: 2 - 3 |\n// |-------------|-------------------|---------------------|------------------|\n// | lead byte | not valid | next: ASCII/lead | not valid |\n// | (192-207) | | context: 0 - 1 | |\n// |-------------|-------------------|---------------------|------------------|\n// | lead byte | not valid | next: cont. | not valid |\n// | (208-) | | context: 2 - 3 | |\n// |-------------|-------------------|---------------------|------------------|\n//\n// The context id for the signed context mode is calculated as:\n//\n// context = (kContextLookup[512 + p1] << 3) | kContextLookup[512 + p2].\n//\n// For any context modeling modes, the context ids can be calculated by |-ing\n// together two lookups from one table using context model dependent offsets:\n//\n// context = kContextLookup[offset1 + p1] | kContextLookup[offset2 + p2].\n//\n// where offset1 and offset2 are dependent on the context mode.\n//\n\n#[allow(dead_code)]\npub enum ContextType {\n CONTEXT_LSB6,\n CONTEXT_MSB6,\n CONTEXT_UTF8,\n CONTEXT_SIGNED,\n}\n\n/* Common context lookup table for all context modes. */\npub static kContextLookup:[[u8;512];4] = [\n[\n/* CONTEXT_LSB6, last byte. */\n 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,\n 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,\n 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,\n 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,\n 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,\n 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,\n 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,\n 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,\n 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,\n 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,\n 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,\n 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,\n 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,\n 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,\n 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,\n 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,\n\n /* CONTEXT_LSB6, second last byte, */\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n],[\n /* CONTEXT_MSB6, last byte. */\n 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,\n 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,\n 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11,\n 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15,\n 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, 19, 19, 19, 19,\n 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23,\n 24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27,\n 28, 28, 28, 28, 29, 29, 29, 29, 30, 30, 30, 30, 31, 31, 31, 31,\n 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 35,\n 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, 39, 39, 39,\n 40, 40, 40, 40, 41, 41, 41, 41, 42, 42, 42, 42, 43, 43, 43, 43,\n 44, 44, 44, 44, 45, 45, 45, 45, 46, 46, 46, 46, 47, 47, 47, 47,\n 48, 48, 48, 48, 49, 49, 49, 49, 50, 50, 50, 50, 51, 51, 51, 51,\n 52, 52, 52, 52, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 55, 55,\n 56, 56, 56, 56, 57, 57, 57, 57, 58, 58, 58, 58, 59, 59, 59, 59,\n 60, 60, 60, 60, 61, 61, 61, 61, 62, 62, 62, 62, 63, 63, 63, 63,\n\n /* CONTEXT_MSB6, second last byte, */\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n],[\n /* CONTEXT_UTF8, last byte. */\n /* ASCII range. */\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12,\n 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12,\n 12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48,\n 52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12,\n 12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56,\n 60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0,\n /* UTF8 continuation byte range. */\n 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n /* UTF8 lead byte range. */\n 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n\n /* CONTEXT_UTF8 second last byte. */\n /* ASCII range. */\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1,\n 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,\n 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0,\n /* UTF8 continuation byte range. */\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n /* UTF8 lead byte range. */\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n],[\n /* CONTEXT_SIGNED, last byte, same as the above values shifted by 3 bits. */\n 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,\n 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,\n 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,\n 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,\n 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,\n 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,\n 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,\n 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,\n 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,\n 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,\n 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,\n 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,\n 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,\n 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,\n 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,\n 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 56,\n\n /* CONTEXT_SIGNED, second last byte. */\n 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,\n 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7,\n]\n];\n\n" }, { "package": "bumpalo@3.20.3", "sourcePath": "src/alloc.rs", "sha256": "3a9645d9e8db1f2a8549ee928cafa5263a828f25c88ce4d2b07996ecc14bfa81", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n#![allow(unstable_name_collisions)]\n#![allow(dead_code)]\n#![allow(deprecated)]\n\n//! Memory allocation APIs\n\nuse core::cmp;\nuse core::fmt;\nuse core::mem;\nuse core::ptr::{self, NonNull};\nuse core::usize;\n\npub use core::alloc::{Layout, LayoutErr};\n\nfn new_layout_err() -> LayoutErr {\n Layout::from_size_align(1, 3).unwrap_err()\n}\n\npub fn handle_alloc_error(layout: Layout) -> ! {\n panic!(\"encountered allocation error: {:?}\", layout)\n}\n\npub trait UnstableLayoutMethods {\n fn padding_needed_for(&self, align: usize) -> usize;\n fn repeat(&self, n: usize) -> Result<(Layout, usize), LayoutErr>;\n fn array(n: usize) -> Result;\n}\n\nimpl UnstableLayoutMethods for Layout {\n fn padding_needed_for(&self, align: usize) -> usize {\n let len = self.size();\n\n // Rounded up value is:\n // len_rounded_up = (len + align - 1) & !(align - 1);\n // and then we return the padding difference: `len_rounded_up - len`.\n //\n // We use modular arithmetic throughout:\n //\n // 1. align is guaranteed to be > 0, so align - 1 is always\n // valid.\n //\n // 2. `len + align - 1` can overflow by at most `align - 1`,\n // so the &-mask with `!(align - 1)` will ensure that in the\n // case of overflow, `len_rounded_up` will itself be 0.\n // Thus the returned padding, when added to `len`, yields 0,\n // which trivially satisfies the alignment `align`.\n //\n // (Of course, attempts to allocate blocks of memory whose\n // size and padding overflow in the above manner should cause\n // the allocator to yield an error anyway.)\n\n let len_rounded_up = len.wrapping_add(align).wrapping_sub(1) & !align.wrapping_sub(1);\n len_rounded_up.wrapping_sub(len)\n }\n\n fn repeat(&self, n: usize) -> Result<(Layout, usize), LayoutErr> {\n let padded_size = self\n .size()\n .checked_add(self.padding_needed_for(self.align()))\n .ok_or_else(new_layout_err)?;\n let alloc_size = padded_size.checked_mul(n).ok_or_else(new_layout_err)?;\n\n unsafe {\n // self.align is already known to be valid and alloc_size has been\n // padded already.\n Ok((\n Layout::from_size_align_unchecked(alloc_size, self.align()),\n padded_size,\n ))\n }\n }\n\n fn array(n: usize) -> Result {\n Layout::new::().repeat(n).map(|(k, offs)| {\n debug_assert!(offs == mem::size_of::());\n k\n })\n }\n}\n\n/// Represents the combination of a starting address and\n/// a total capacity of the returned block.\n// #[unstable(feature = \"allocator_api\", issue = \"32838\")]\n#[derive(Debug)]\npub struct Excess(pub NonNull, pub usize);\n\nfn size_align() -> (usize, usize) {\n (mem::size_of::(), mem::align_of::())\n}\n\n/// The `AllocErr` error indicates an allocation failure\n/// that may be due to resource exhaustion or to\n/// something wrong when combining the given input arguments with this\n/// allocator.\n// #[unstable(feature = \"allocator_api\", issue = \"32838\")]\n#[derive(Clone, PartialEq, Eq, Debug)]\npub struct AllocErr;\n\n// (we need this for downstream impl of trait Error)\n// #[unstable(feature = \"allocator_api\", issue = \"32838\")]\nimpl fmt::Display for AllocErr {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n f.write_str(\"memory allocation failed\")\n }\n}\n\n/// The `CannotReallocInPlace` error is used when `grow_in_place` or\n/// `shrink_in_place` were unable to reuse the given memory block for\n/// a requested layout.\n// #[unstable(feature = \"allocator_api\", issue = \"32838\")]\n#[derive(Clone, PartialEq, Eq, Debug)]\npub struct CannotReallocInPlace;\n\n// #[unstable(feature = \"allocator_api\", issue = \"32838\")]\nimpl CannotReallocInPlace {\n pub fn description(&self) -> &str {\n \"cannot reallocate allocator's memory in place\"\n }\n}\n\n// (we need this for downstream impl of trait Error)\n// #[unstable(feature = \"allocator_api\", issue = \"32838\")]\nimpl fmt::Display for CannotReallocInPlace {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n write!(f, \"{}\", self.description())\n }\n}\n\n/// An implementation of `Alloc` can allocate, reallocate, and\n/// deallocate arbitrary blocks of data described via `Layout`.\n///\n/// Some of the methods require that a memory block be *currently\n/// allocated* via an allocator. This means that:\n///\n/// * the starting address for that memory block was previously\n/// returned by a previous call to an allocation method (`alloc`,\n/// `alloc_zeroed`, `alloc_excess`, `alloc_one`, `alloc_array`) or\n/// reallocation method (`realloc`, `realloc_excess`, or\n/// `realloc_array`), and\n///\n/// * the memory block has not been subsequently deallocated, where\n/// blocks are deallocated either by being passed to a deallocation\n/// method (`dealloc`, `dealloc_one`, `dealloc_array`) or by being\n/// passed to a reallocation method (see above) that returns `Ok`.\n///\n/// A note regarding zero-sized types and zero-sized layouts: many\n/// methods in the `Alloc` trait state that allocation requests\n/// must be non-zero size, or else undefined behavior can result.\n///\n/// * However, some higher-level allocation methods (`alloc_one`,\n/// `alloc_array`) are well-defined on zero-sized types and can\n/// optionally support them: it is left up to the implementor\n/// whether to return `Err`, or to return `Ok` with some pointer.\n///\n/// * If an `Alloc` implementation chooses to return `Ok` in this\n/// case (i.e. the pointer denotes a zero-sized inaccessible block)\n/// then that returned pointer must be considered \"currently\n/// allocated\". On such an allocator, *all* methods that take\n/// currently-allocated pointers as inputs must accept these\n/// zero-sized pointers, *without* causing undefined behavior.\n///\n/// * In other words, if a zero-sized pointer can flow out of an\n/// allocator, then that allocator must likewise accept that pointer\n/// flowing back into its deallocation and reallocation methods.\n///\n/// Some of the methods require that a layout *fit* a memory block.\n/// What it means for a layout to \"fit\" a memory block means (or\n/// equivalently, for a memory block to \"fit\" a layout) is that the\n/// following two conditions must hold:\n///\n/// 1. The block's starting address must be aligned to `layout.align()`.\n///\n/// 2. The block's size must fall in the range `[use_min, use_max]`, where:\n///\n/// * `use_min` is `self.usable_size(layout).0`, and\n///\n/// * `use_max` is the capacity that was (or would have been)\n/// returned when (if) the block was allocated via a call to\n/// `alloc_excess` or `realloc_excess`.\n///\n/// Note that:\n///\n/// * the size of the layout most recently used to allocate the block\n/// is guaranteed to be in the range `[use_min, use_max]`, and\n///\n/// * a lower-bound on `use_max` can be safely approximated by a call to\n/// `usable_size`.\n///\n/// * if a layout `k` fits a memory block (denoted by `ptr`)\n/// currently allocated via an allocator `a`, then it is legal to\n/// use that layout to deallocate it, i.e. `a.dealloc(ptr, k);`.\n///\n/// # Unsafety\n///\n/// The `Alloc` trait is an `unsafe` trait for a number of reasons, and\n/// implementors must ensure that they adhere to these contracts:\n///\n/// * Pointers returned from allocation functions must point to valid memory and\n/// retain their validity until at least the instance of `Alloc` is dropped\n/// itself.\n///\n/// * `Layout` queries and calculations in general must be correct. Callers of\n/// this trait are allowed to rely on the contracts defined on each method,\n/// and implementors must ensure such contracts remain true.\n///\n/// Note that this list may get tweaked over time as clarifications are made in\n/// the future.\n// #[unstable(feature = \"allocator_api\", issue = \"32838\")]\npub unsafe trait Alloc {\n // (Note: some existing allocators have unspecified but well-defined\n // behavior in response to a zero size allocation request ;\n // e.g. in C, `malloc` of 0 will either return a null pointer or a\n // unique pointer, but will not have arbitrary undefined\n // behavior.\n // However in jemalloc for example,\n // `mallocx(0)` is documented as undefined behavior.)\n\n /// Returns a pointer meeting the size and alignment guarantees of\n /// `layout`.\n ///\n /// If this method returns an `Ok(addr)`, then the `addr` returned\n /// will be non-null address pointing to a block of storage\n /// suitable for holding an instance of `layout`.\n ///\n /// The returned block of storage may or may not have its contents\n /// initialized. (Extension subtraits might restrict this\n /// behavior, e.g. to ensure initialization to particular sets of\n /// bit patterns.)\n ///\n /// # Safety\n ///\n /// This function is unsafe because undefined behavior can result\n /// if the caller does not ensure that `layout` has non-zero size.\n ///\n /// (Extension subtraits might provide more specific bounds on\n /// behavior, e.g. guarantee a sentinel address or a null pointer\n /// in response to a zero-size allocation request.)\n ///\n /// # Errors\n ///\n /// Returning `Err` indicates that either memory is exhausted or\n /// `layout` does not meet allocator's size or alignment\n /// constraints.\n ///\n /// Implementations are encouraged to return `Err` on memory\n /// exhaustion rather than panicking or aborting, but this is not\n /// a strict requirement. (Specifically: it is *legal* to\n /// implement this trait atop an underlying native allocation\n /// library that aborts on memory exhaustion.)\n ///\n /// Clients wishing to abort computation in response to an\n /// allocation error are encouraged to call the [`handle_alloc_error`] function,\n /// rather than directly invoking `panic!` or similar.\n ///\n /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html\n unsafe fn alloc(&mut self, layout: Layout) -> Result, AllocErr>;\n\n /// Deallocate the memory referenced by `ptr`.\n ///\n /// # Safety\n ///\n /// This function is unsafe because undefined behavior can result\n /// if the caller does not ensure all of the following:\n ///\n /// * `ptr` must denote a block of memory currently allocated via\n /// this allocator,\n ///\n /// * `layout` must *fit* that block of memory,\n ///\n /// * In addition to fitting the block of memory `layout`, the\n /// alignment of the `layout` must match the alignment used\n /// to allocate that block of memory.\n unsafe fn dealloc(&mut self, ptr: NonNull, layout: Layout);\n\n // == ALLOCATOR-SPECIFIC QUANTITIES AND LIMITS ==\n // usable_size\n\n /// Returns bounds on the guaranteed usable size of a successful\n /// allocation created with the specified `layout`.\n ///\n /// In particular, if one has a memory block allocated via a given\n /// allocator `a` and layout `k` where `a.usable_size(k)` returns\n /// `(l, u)`, then one can pass that block to `a.dealloc()` with a\n /// layout in the size range [l, u].\n ///\n /// (All implementors of `usable_size` must ensure that\n /// `l <= k.size() <= u`)\n ///\n /// Both the lower- and upper-bounds (`l` and `u` respectively)\n /// are provided, because an allocator based on size classes could\n /// misbehave if one attempts to deallocate a block without\n /// providing a correct value for its size (i.e., one within the\n /// range `[l, u]`).\n ///\n /// Clients who wish to make use of excess capacity are encouraged\n /// to use the `alloc_excess` and `realloc_excess` instead, as\n /// this method is constrained to report conservative values that\n /// serve as valid bounds for *all possible* allocation method\n /// calls.\n ///\n /// However, for clients that do not wish to track the capacity\n /// returned by `alloc_excess` locally, this method is likely to\n /// produce useful results.\n #[inline]\n fn usable_size(&self, layout: &Layout) -> (usize, usize) {\n (layout.size(), layout.size())\n }\n\n // == METHODS FOR MEMORY REUSE ==\n // realloc. alloc_excess, realloc_excess\n\n /// Returns a pointer suitable for holding data described by\n /// a new layout with `layout`’s alignment and a size given\n /// by `new_size`. To\n /// accomplish this, this may extend or shrink the allocation\n /// referenced by `ptr` to fit the new layout.\n ///\n /// If this returns `Ok`, then ownership of the memory block\n /// referenced by `ptr` has been transferred to this\n /// allocator. The memory may or may not have been freed, and\n /// should be considered unusable (unless of course it was\n /// transferred back to the caller again via the return value of\n /// this method).\n ///\n /// If this method returns `Err`, then ownership of the memory\n /// block has not been transferred to this allocator, and the\n /// contents of the memory block are unaltered.\n ///\n /// # Safety\n ///\n /// This function is unsafe because undefined behavior can result\n /// if the caller does not ensure all of the following:\n ///\n /// * `ptr` must be currently allocated via this allocator,\n ///\n /// * `layout` must *fit* the `ptr` (see above). (The `new_size`\n /// argument need not fit it.)\n ///\n /// * `new_size` must be greater than zero.\n ///\n /// * `new_size`, when rounded up to the nearest multiple of `layout.align()`,\n /// must not overflow (i.e. the rounded value must be less than `usize::MAX`).\n ///\n /// (Extension subtraits might provide more specific bounds on\n /// behavior, e.g. guarantee a sentinel address or a null pointer\n /// in response to a zero-size allocation request.)\n ///\n /// # Errors\n ///\n /// Returns `Err` only if the new layout\n /// does not meet the allocator's size\n /// and alignment constraints of the allocator, or if reallocation\n /// otherwise fails.\n ///\n /// Implementations are encouraged to return `Err` on memory\n /// exhaustion rather than panicking or aborting, but this is not\n /// a strict requirement. (Specifically: it is *legal* to\n /// implement this trait atop an underlying native allocation\n /// library that aborts on memory exhaustion.)\n ///\n /// Clients wishing to abort computation in response to a\n /// reallocation error are encouraged to call the [`handle_alloc_error`] function,\n /// rather than directly invoking `panic!` or similar.\n ///\n /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html\n unsafe fn realloc(\n &mut self,\n ptr: NonNull,\n layout: Layout,\n new_size: usize,\n ) -> Result, AllocErr> {\n let old_size = layout.size();\n\n if new_size >= old_size {\n if let Ok(()) = self.grow_in_place(ptr, layout, new_size) {\n return Ok(ptr);\n }\n } else if new_size < old_size {\n if let Ok(()) = self.shrink_in_place(ptr, layout, new_size) {\n return Ok(ptr);\n }\n }\n\n // otherwise, fall back on alloc + copy + dealloc.\n let new_layout = Layout::from_size_align_unchecked(new_size, layout.align());\n let result = self.alloc(new_layout);\n if let Ok(new_ptr) = result {\n ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_ptr(), cmp::min(old_size, new_size));\n self.dealloc(ptr, layout);\n }\n result\n }\n\n /// Behaves like `alloc`, but also ensures that the contents\n /// are set to zero before being returned.\n ///\n /// # Safety\n ///\n /// This function is unsafe for the same reasons that `alloc` is.\n ///\n /// # Errors\n ///\n /// Returning `Err` indicates that either memory is exhausted or\n /// `layout` does not meet allocator's size or alignment\n /// constraints, just as in `alloc`.\n ///\n /// Clients wishing to abort computation in response to an\n /// allocation error are encouraged to call the [`handle_alloc_error`] function,\n /// rather than directly invoking `panic!` or similar.\n ///\n /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html\n unsafe fn alloc_zeroed(&mut self, layout: Layout) -> Result, AllocErr> {\n let size = layout.size();\n let p = self.alloc(layout);\n if let Ok(p) = p {\n ptr::write_bytes(p.as_ptr(), 0, size);\n }\n p\n }\n\n /// Behaves like `alloc`, but also returns the whole size of\n /// the returned block. For some `layout` inputs, like arrays, this\n /// may include extra storage usable for additional data.\n ///\n /// # Safety\n ///\n /// This function is unsafe for the same reasons that `alloc` is.\n ///\n /// # Errors\n ///\n /// Returning `Err` indicates that either memory is exhausted or\n /// `layout` does not meet allocator's size or alignment\n /// constraints, just as in `alloc`.\n ///\n /// Clients wishing to abort computation in response to an\n /// allocation error are encouraged to call the [`handle_alloc_error`] function,\n /// rather than directly invoking `panic!` or similar.\n ///\n /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html\n unsafe fn alloc_excess(&mut self, layout: Layout) -> Result {\n let usable_size = self.usable_size(&layout);\n self.alloc(layout).map(|p| Excess(p, usable_size.1))\n }\n\n /// Behaves like `realloc`, but also returns the whole size of\n /// the returned block. For some `layout` inputs, like arrays, this\n /// may include extra storage usable for additional data.\n ///\n /// # Safety\n ///\n /// This function is unsafe for the same reasons that `realloc` is.\n ///\n /// # Errors\n ///\n /// Returning `Err` indicates that either memory is exhausted or\n /// `layout` does not meet allocator's size or alignment\n /// constraints, just as in `realloc`.\n ///\n /// Clients wishing to abort computation in response to a\n /// reallocation error are encouraged to call the [`handle_alloc_error`] function,\n /// rather than directly invoking `panic!` or similar.\n ///\n /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html\n unsafe fn realloc_excess(\n &mut self,\n ptr: NonNull,\n layout: Layout,\n new_size: usize,\n ) -> Result {\n let new_layout = Layout::from_size_align_unchecked(new_size, layout.align());\n let usable_size = self.usable_size(&new_layout);\n self.realloc(ptr, layout, new_size)\n .map(|p| Excess(p, usable_size.1))\n }\n\n /// Attempts to extend the allocation referenced by `ptr` to fit `new_size`.\n ///\n /// If this returns `Ok`, then the allocator has asserted that the\n /// memory block referenced by `ptr` now fits `new_size`, and thus can\n /// be used to carry data of a layout of that size and same alignment as\n /// `layout`. (The allocator is allowed to\n /// expend effort to accomplish this, such as extending the memory block to\n /// include successor blocks, or virtual memory tricks.)\n ///\n /// Regardless of what this method returns, ownership of the\n /// memory block referenced by `ptr` has not been transferred, and\n /// the contents of the memory block are unaltered.\n ///\n /// # Safety\n ///\n /// This function is unsafe because undefined behavior can result\n /// if the caller does not ensure all of the following:\n ///\n /// * `ptr` must be currently allocated via this allocator,\n ///\n /// * `layout` must *fit* the `ptr` (see above); note the\n /// `new_size` argument need not fit it,\n ///\n /// * `new_size` must not be less than `layout.size()`,\n ///\n /// # Errors\n ///\n /// Returns `Err(CannotReallocInPlace)` when the allocator is\n /// unable to assert that the memory block referenced by `ptr`\n /// could fit `layout`.\n ///\n /// Note that one cannot pass `CannotReallocInPlace` to the `handle_alloc_error`\n /// function; clients are expected either to be able to recover from\n /// `grow_in_place` failures without aborting, or to fall back on\n /// another reallocation method before resorting to an abort.\n unsafe fn grow_in_place(\n &mut self,\n ptr: NonNull,\n layout: Layout,\n new_size: usize,\n ) -> Result<(), CannotReallocInPlace> {\n let _ = ptr; // this default implementation doesn't care about the actual address.\n debug_assert!(new_size >= layout.size());\n let (_l, u) = self.usable_size(&layout);\n // _l <= layout.size() [guaranteed by usable_size()]\n // layout.size() <= new_layout.size() [required by this method]\n if new_size <= u {\n Ok(())\n } else {\n Err(CannotReallocInPlace)\n }\n }\n\n /// Attempts to shrink the allocation referenced by `ptr` to fit `new_size`.\n ///\n /// If this returns `Ok`, then the allocator has asserted that the\n /// memory block referenced by `ptr` now fits `new_size`, and\n /// thus can only be used to carry data of that smaller\n /// layout. (The allocator is allowed to take advantage of this,\n /// carving off portions of the block for reuse elsewhere.) The\n /// truncated contents of the block within the smaller layout are\n /// unaltered, and ownership of block has not been transferred.\n ///\n /// If this returns `Err`, then the memory block is considered to\n /// still represent the original (larger) `layout`. None of the\n /// block has been carved off for reuse elsewhere, ownership of\n /// the memory block has not been transferred, and the contents of\n /// the memory block are unaltered.\n ///\n /// # Safety\n ///\n /// This function is unsafe because undefined behavior can result\n /// if the caller does not ensure all of the following:\n ///\n /// * `ptr` must be currently allocated via this allocator,\n ///\n /// * `layout` must *fit* the `ptr` (see above); note the\n /// `new_size` argument need not fit it,\n ///\n /// * `new_size` must not be greater than `layout.size()`\n /// (and must be greater than zero),\n ///\n /// # Errors\n ///\n /// Returns `Err(CannotReallocInPlace)` when the allocator is\n /// unable to assert that the memory block referenced by `ptr`\n /// could fit `layout`.\n ///\n /// Note that one cannot pass `CannotReallocInPlace` to the `handle_alloc_error`\n /// function; clients are expected either to be able to recover from\n /// `shrink_in_place` failures without aborting, or to fall back\n /// on another reallocation method before resorting to an abort.\n unsafe fn shrink_in_place(\n &mut self,\n ptr: NonNull,\n layout: Layout,\n new_size: usize,\n ) -> Result<(), CannotReallocInPlace> {\n let _ = ptr; // this default implementation doesn't care about the actual address.\n debug_assert!(new_size <= layout.size());\n let (l, _u) = self.usable_size(&layout);\n // layout.size() <= _u [guaranteed by usable_size()]\n // new_layout.size() <= layout.size() [required by this method]\n if l <= new_size {\n Ok(())\n } else {\n Err(CannotReallocInPlace)\n }\n }\n\n // == COMMON USAGE PATTERNS ==\n // alloc_one, dealloc_one, alloc_array, realloc_array. dealloc_array\n\n /// Allocates a block suitable for holding an instance of `T`.\n ///\n /// Captures a common usage pattern for allocators.\n ///\n /// The returned block is suitable for passing to the\n /// `alloc`/`realloc` methods of this allocator.\n ///\n /// Note to implementors: If this returns `Ok(ptr)`, then `ptr`\n /// must be considered \"currently allocated\" and must be\n /// acceptable input to methods such as `realloc` or `dealloc`,\n /// *even if* `T` is a zero-sized type. In other words, if your\n /// `Alloc` implementation overrides this method in a manner\n /// that can return a zero-sized `ptr`, then all reallocation and\n /// deallocation methods need to be similarly overridden to accept\n /// such values as input.\n ///\n /// # Errors\n ///\n /// Returning `Err` indicates that either memory is exhausted or\n /// `T` does not meet allocator's size or alignment constraints.\n ///\n /// For zero-sized `T`, may return either of `Ok` or `Err`, but\n /// will *not* yield undefined behavior.\n ///\n /// Clients wishing to abort computation in response to an\n /// allocation error are encouraged to call the [`handle_alloc_error`] function,\n /// rather than directly invoking `panic!` or similar.\n ///\n /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html\n fn alloc_one(&mut self) -> Result, AllocErr>\n where\n Self: Sized,\n {\n let k = Layout::new::();\n if k.size() > 0 {\n unsafe { self.alloc(k).map(|p| p.cast()) }\n } else {\n Err(AllocErr)\n }\n }\n\n /// Deallocates a block suitable for holding an instance of `T`.\n ///\n /// The given block must have been produced by this allocator,\n /// and must be suitable for storing a `T` (in terms of alignment\n /// as well as minimum and maximum size); otherwise yields\n /// undefined behavior.\n ///\n /// Captures a common usage pattern for allocators.\n ///\n /// # Safety\n ///\n /// This function is unsafe because undefined behavior can result\n /// if the caller does not ensure both:\n ///\n /// * `ptr` must denote a block of memory currently allocated via this allocator\n ///\n /// * the layout of `T` must *fit* that block of memory.\n unsafe fn dealloc_one(&mut self, ptr: NonNull)\n where\n Self: Sized,\n {\n let k = Layout::new::();\n if k.size() > 0 {\n self.dealloc(ptr.cast(), k);\n }\n }\n\n /// Allocates a block suitable for holding `n` instances of `T`.\n ///\n /// Captures a common usage pattern for allocators.\n ///\n /// The returned block is suitable for passing to the\n /// `alloc`/`realloc` methods of this allocator.\n ///\n /// Note to implementors: If this returns `Ok(ptr)`, then `ptr`\n /// must be considered \"currently allocated\" and must be\n /// acceptable input to methods such as `realloc` or `dealloc`,\n /// *even if* `T` is a zero-sized type. In other words, if your\n /// `Alloc` implementation overrides this method in a manner\n /// that can return a zero-sized `ptr`, then all reallocation and\n /// deallocation methods need to be similarly overridden to accept\n /// such values as input.\n ///\n /// # Errors\n ///\n /// Returning `Err` indicates that either memory is exhausted or\n /// `[T; n]` does not meet allocator's size or alignment\n /// constraints.\n ///\n /// For zero-sized `T` or `n == 0`, may return either of `Ok` or\n /// `Err`, but will *not* yield undefined behavior.\n ///\n /// Always returns `Err` on arithmetic overflow.\n ///\n /// Clients wishing to abort computation in response to an\n /// allocation error are encouraged to call the [`handle_alloc_error`] function,\n /// rather than directly invoking `panic!` or similar.\n ///\n /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html\n fn alloc_array(&mut self, n: usize) -> Result, AllocErr>\n where\n Self: Sized,\n {\n match Layout::array::(n) {\n Ok(layout) if layout.size() > 0 => unsafe { self.alloc(layout).map(|p| p.cast()) },\n _ => Err(AllocErr),\n }\n }\n\n /// Reallocates a block previously suitable for holding `n_old`\n /// instances of `T`, returning a block suitable for holding\n /// `n_new` instances of `T`.\n ///\n /// Captures a common usage pattern for allocators.\n ///\n /// The returned block is suitable for passing to the\n /// `alloc`/`realloc` methods of this allocator.\n ///\n /// # Safety\n ///\n /// This function is unsafe because undefined behavior can result\n /// if the caller does not ensure all of the following:\n ///\n /// * `ptr` must be currently allocated via this allocator,\n ///\n /// * the layout of `[T; n_old]` must *fit* that block of memory.\n ///\n /// # Errors\n ///\n /// Returning `Err` indicates that either memory is exhausted or\n /// `[T; n_new]` does not meet allocator's size or alignment\n /// constraints.\n ///\n /// For zero-sized `T` or `n_new == 0`, may return either of `Ok` or\n /// `Err`, but will *not* yield undefined behavior.\n ///\n /// Always returns `Err` on arithmetic overflow.\n ///\n /// Clients wishing to abort computation in response to a\n /// reallocation error are encouraged to call the [`handle_alloc_error`] function,\n /// rather than directly invoking `panic!` or similar.\n ///\n /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html\n unsafe fn realloc_array(\n &mut self,\n ptr: NonNull,\n n_old: usize,\n n_new: usize,\n ) -> Result, AllocErr>\n where\n Self: Sized,\n {\n match (Layout::array::(n_old), Layout::array::(n_new)) {\n (Ok(ref k_old), Ok(ref k_new)) if k_old.size() > 0 && k_new.size() > 0 => {\n debug_assert!(k_old.align() == k_new.align());\n self.realloc(ptr.cast(), *k_old, k_new.size())\n .map(NonNull::cast)\n }\n _ => Err(AllocErr),\n }\n }\n\n /// Deallocates a block suitable for holding `n` instances of `T`.\n ///\n /// Captures a common usage pattern for allocators.\n ///\n /// # Safety\n ///\n /// This function is unsafe because undefined behavior can result\n /// if the caller does not ensure both:\n ///\n /// * `ptr` must denote a block of memory currently allocated via this allocator\n ///\n /// * the layout of `[T; n]` must *fit* that block of memory.\n ///\n /// # Errors\n ///\n /// Returning `Err` indicates that either `[T; n]` or the given\n /// memory block does not meet allocator's size or alignment\n /// constraints.\n ///\n /// Always returns `Err` on arithmetic overflow.\n unsafe fn dealloc_array(&mut self, ptr: NonNull, n: usize) -> Result<(), AllocErr>\n where\n Self: Sized,\n {\n match Layout::array::(n) {\n Ok(k) if k.size() > 0 => {\n self.dealloc(ptr.cast(), k);\n Ok(())\n }\n _ => Err(AllocErr),\n }\n }\n}\n" }, { "package": "bumpalo@3.20.3", "sourcePath": "src/collections/mod.rs", "sha256": "d58dc46eb4f9fcdde574f09bc5b8646f53e42d49c169561d98e0c23e5b36848a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Collection types that allocate inside a [`Bump`] arena.\n//!\n//! [`Bump`]: ../struct.Bump.html\n\n#![allow(deprecated)]\n\nmod raw_vec;\n\npub mod vec;\npub use self::vec::Vec;\n\nmod str;\npub mod string;\npub use self::string::String;\n\nmod collect_in;\npub use collect_in::{CollectIn, FromIteratorIn};\n\n// pub mod binary_heap;\n// mod btree;\n// pub mod linked_list;\n// pub mod vec_deque;\n\n// pub mod btree_map {\n// //! A map based on a B-Tree.\n// pub use super::btree::map::*;\n// }\n\n// pub mod btree_set {\n// //! A set based on a B-Tree.\n// pub use super::btree::set::*;\n// }\n\n// #[doc(no_inline)]\n// pub use self::binary_heap::BinaryHeap;\n\n// #[doc(no_inline)]\n// pub use self::btree_map::BTreeMap;\n\n// #[doc(no_inline)]\n// pub use self::btree_set::BTreeSet;\n\n// #[doc(no_inline)]\n// pub use self::linked_list::LinkedList;\n\n// #[doc(no_inline)]\n// pub use self::vec_deque::VecDeque;\n\nuse crate::alloc::{AllocErr, LayoutErr};\n\n/// Augments `AllocErr` with a `CapacityOverflow` variant.\n#[derive(Clone, PartialEq, Eq, Debug)]\n// #[unstable(feature = \"try_reserve\", reason = \"new API\", issue=\"48043\")]\npub enum CollectionAllocErr {\n /// Error due to the computed capacity exceeding the collection's maximum\n /// (usually `isize::MAX` bytes).\n CapacityOverflow,\n /// Error due to the allocator (see the documentation for the [`AllocErr`] type).\n AllocErr,\n}\n\n// #[unstable(feature = \"try_reserve\", reason = \"new API\", issue=\"48043\")]\nimpl From for CollectionAllocErr {\n #[inline]\n fn from(AllocErr: AllocErr) -> Self {\n CollectionAllocErr::AllocErr\n }\n}\n\n// #[unstable(feature = \"try_reserve\", reason = \"new API\", issue=\"48043\")]\nimpl From for CollectionAllocErr {\n #[inline]\n fn from(_: LayoutErr) -> Self {\n CollectionAllocErr::CapacityOverflow\n }\n}\n\n// /// An intermediate trait for specialization of `Extend`.\n// #[doc(hidden)]\n// trait SpecExtend {\n// /// Extends `self` with the contents of the given iterator.\n// fn spec_extend(&mut self, iter: I);\n// }\n" }, { "package": "bumpalo@3.20.3", "sourcePath": "src/collections/raw_vec.rs", "sha256": "c045b13f3ac191f1c597f87a7385f59078a3d091f7888d8449d65b9e6f67e315", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n#![allow(unstable_name_collisions)]\n#![allow(dead_code)]\n\nuse crate::Bump;\n\nuse core::cmp;\nuse core::mem;\nuse core::ptr::{self, NonNull};\n\nuse crate::alloc::{handle_alloc_error, Alloc, Layout, UnstableLayoutMethods};\nuse crate::collections::CollectionAllocErr;\nuse crate::collections::CollectionAllocErr::*;\n// use boxed::Box;\n\n/// A low-level utility for more ergonomically allocating, reallocating, and deallocating\n/// a buffer of memory on the heap without having to worry about all the corner cases\n/// involved. This type is excellent for building your own data structures like Vec and VecDeque.\n/// In particular:\n///\n/// * Produces Unique::empty() on zero-sized types\n/// * Produces Unique::empty() on zero-length allocations\n/// * Catches all overflows in capacity computations (promotes them to \"capacity overflow\" panics)\n/// * Guards against 32-bit systems allocating more than isize::MAX bytes\n/// * Guards against overflowing your length\n/// * Aborts on OOM\n/// * Avoids freeing Unique::empty()\n/// * Contains a ptr::Unique and thus endows the user with all related benefits\n///\n/// This type does not in anyway inspect the memory that it manages. When dropped it *will*\n/// free its memory, but it *won't* try to Drop its contents. It is up to the user of RawVec\n/// to handle the actual things *stored* inside of a RawVec.\n///\n/// Note that a RawVec always forces its capacity to be usize::MAX for zero-sized types.\n/// This enables you to use capacity growing logic catch the overflows in your length\n/// that might occur with zero-sized types.\n///\n/// However this means that you need to be careful when round-tripping this type\n/// with a `Box<[T]>`: `cap()` won't yield the len. However `with_capacity`,\n/// `shrink_to_fit`, and `from_box` will actually set RawVec's private capacity\n/// field. This allows zero-sized types to not be special-cased by consumers of\n/// this type.\n#[allow(missing_debug_implementations)]\npub struct RawVec<'a, T> {\n ptr: NonNull,\n cap: usize,\n a: &'a Bump,\n}\n\nimpl<'a, T> RawVec<'a, T> {\n /// Like `new` but parameterized over the choice of allocator for\n /// the returned RawVec.\n pub fn new_in(a: &'a Bump) -> Self {\n // `cap: 0` means \"unallocated\". zero-sized types are ignored.\n RawVec {\n ptr: NonNull::dangling(),\n cap: 0,\n a,\n }\n }\n\n /// Like `with_capacity` but parameterized over the choice of\n /// allocator for the returned RawVec.\n #[inline]\n pub fn with_capacity_in(cap: usize, a: &'a Bump) -> Self {\n RawVec::allocate_in(cap, false, a)\n }\n\n /// Like `with_capacity_zeroed` but parameterized over the choice\n /// of allocator for the returned RawVec.\n #[inline]\n pub fn with_capacity_zeroed_in(cap: usize, a: &'a Bump) -> Self {\n RawVec::allocate_in(cap, true, a)\n }\n\n fn allocate_in(cap: usize, zeroed: bool, mut a: &'a Bump) -> Self {\n unsafe {\n let elem_size = mem::size_of::();\n\n let alloc_size = cap\n .checked_mul(elem_size)\n .unwrap_or_else(|| capacity_overflow());\n alloc_guard(alloc_size).unwrap_or_else(|_| capacity_overflow());\n\n // handles ZSTs and `cap = 0` alike\n let ptr = if alloc_size == 0 {\n NonNull::::dangling()\n } else {\n let align = mem::align_of::();\n let layout = Layout::from_size_align(alloc_size, align).unwrap();\n let result = if zeroed {\n a.alloc_zeroed(layout)\n } else {\n Alloc::alloc(&mut a, layout)\n };\n match result {\n Ok(ptr) => ptr.cast(),\n Err(_) => handle_alloc_error(layout),\n }\n };\n\n RawVec { ptr, cap, a }\n }\n }\n}\n\nimpl<'a, T> RawVec<'a, T> {\n /// Reconstitutes a RawVec from a pointer, capacity, and allocator.\n ///\n /// # Undefined Behavior\n ///\n /// The ptr must be allocated (via the given allocator `a`), and with the given capacity. The\n /// capacity cannot exceed `isize::MAX` (only a concern on 32-bit systems).\n /// If the ptr and capacity come from a RawVec created via `a`, then this is guaranteed.\n pub unsafe fn from_raw_parts_in(ptr: *mut T, cap: usize, a: &'a Bump) -> Self {\n RawVec {\n ptr: NonNull::new_unchecked(ptr),\n cap,\n a,\n }\n }\n}\n\nimpl<'a, T> RawVec<'a, T> {\n /// Gets a raw pointer to the start of the allocation. Note that this is\n /// Unique::empty() if `cap = 0` or T is zero-sized. In the former case, you must\n /// be careful.\n pub fn ptr(&self) -> *mut T {\n self.ptr.as_ptr()\n }\n\n /// Gets the capacity of the allocation.\n ///\n /// This will always be `usize::MAX` if `T` is zero-sized.\n #[inline(always)]\n pub fn cap(&self) -> usize {\n if mem::size_of::() == 0 {\n !0\n } else {\n self.cap\n }\n }\n\n /// Returns a shared reference to the allocator backing this RawVec.\n pub fn bump(&self) -> &'a Bump {\n self.a\n }\n\n fn current_layout(&self) -> Option {\n if self.cap == 0 {\n None\n } else {\n // We have an allocated chunk of memory, so we can bypass runtime\n // checks to get our current layout.\n unsafe {\n let align = mem::align_of::();\n let size = mem::size_of::() * self.cap;\n Some(Layout::from_size_align_unchecked(size, align))\n }\n }\n }\n\n /// Doubles the size of the type's backing allocation. This is common enough\n /// to want to do that it's easiest to just have a dedicated method. Slightly\n /// more efficient logic can be provided for this than the general case.\n ///\n /// This function is ideal for when pushing elements one-at-a-time because\n /// you don't need to incur the costs of the more general computations\n /// reserve needs to do to guard against overflow. You do however need to\n /// manually check if your `len == cap`.\n ///\n /// # Panics\n ///\n /// * Panics if T is zero-sized on the assumption that you managed to exhaust\n /// all `usize::MAX` slots in your imaginary buffer.\n /// * Panics on 32-bit platforms if the requested capacity exceeds\n /// `isize::MAX` bytes.\n ///\n /// # Aborts\n ///\n /// Aborts on OOM\n ///\n /// # Examples\n ///\n /// ```ignore\n /// # #![feature(alloc, raw_vec_internals)]\n /// # extern crate alloc;\n /// # use std::ptr;\n /// # use alloc::raw_vec::RawVec;\n /// struct MyVec {\n /// buf: RawVec,\n /// len: usize,\n /// }\n ///\n /// impl MyVec {\n /// pub fn push(&mut self, elem: T) {\n /// if self.len == self.buf.cap() { self.buf.double(); }\n /// // double would have aborted or panicked if the len exceeded\n /// // `isize::MAX` so this is safe to do unchecked now.\n /// unsafe {\n /// ptr::write(self.buf.ptr().add(self.len), elem);\n /// }\n /// self.len += 1;\n /// }\n /// }\n /// # fn main() {\n /// # let mut vec = MyVec { buf: RawVec::new(), len: 0 };\n /// # vec.push(1);\n /// # }\n /// ```\n #[inline(never)]\n #[cold]\n pub fn double(&mut self) {\n unsafe {\n let elem_size = mem::size_of::();\n\n // since we set the capacity to usize::MAX when elem_size is\n // 0, getting to here necessarily means the RawVec is overfull.\n assert!(elem_size != 0, \"capacity overflow\");\n\n let (new_cap, uniq) = match self.current_layout() {\n Some(cur) => {\n // Since we guarantee that we never allocate more than\n // isize::MAX bytes, `elem_size * self.cap <= isize::MAX` as\n // a precondition, so this can't overflow. Additionally the\n // alignment will never be too large as to \"not be\n // satisfiable\", so `Layout::from_size_align` will always\n // return `Some`.\n //\n // tl;dr; we bypass runtime checks due to dynamic assertions\n // in this module, allowing us to use\n // `from_size_align_unchecked`.\n let new_cap = 2 * self.cap;\n let new_size = new_cap * elem_size;\n alloc_guard(new_size).unwrap_or_else(|_| capacity_overflow());\n let ptr_res = self.a.realloc(self.ptr.cast(), cur, new_size);\n match ptr_res {\n Ok(ptr) => (new_cap, ptr.cast()),\n Err(_) => handle_alloc_error(Layout::from_size_align_unchecked(\n new_size,\n cur.align(),\n )),\n }\n }\n None => {\n // skip to 4 because tiny Vec's are dumb; but not if that\n // would cause overflow\n let new_cap = if elem_size > (!0) / 8 { 1 } else { 4 };\n match self.a.alloc_array::(new_cap) {\n Ok(ptr) => (new_cap, ptr),\n Err(_) => handle_alloc_error(Layout::array::(new_cap).unwrap()),\n }\n }\n };\n self.ptr = uniq;\n self.cap = new_cap;\n }\n }\n\n /// Attempts to double the size of the type's backing allocation in place. This is common\n /// enough to want to do that it's easiest to just have a dedicated method. Slightly\n /// more efficient logic can be provided for this than the general case.\n ///\n /// Returns true if the reallocation attempt has succeeded, or false otherwise.\n ///\n /// # Panics\n ///\n /// * Panics if T is zero-sized on the assumption that you managed to exhaust\n /// all `usize::MAX` slots in your imaginary buffer.\n /// * Panics on 32-bit platforms if the requested capacity exceeds\n /// `isize::MAX` bytes.\n #[inline(never)]\n #[cold]\n pub fn double_in_place(&mut self) -> bool {\n unsafe {\n let elem_size = mem::size_of::();\n let old_layout = match self.current_layout() {\n Some(layout) => layout,\n None => return false, // nothing to double\n };\n\n // since we set the capacity to usize::MAX when elem_size is\n // 0, getting to here necessarily means the RawVec is overfull.\n assert!(elem_size != 0, \"capacity overflow\");\n\n // Since we guarantee that we never allocate more than isize::MAX\n // bytes, `elem_size * self.cap <= isize::MAX` as a precondition, so\n // this can't overflow.\n //\n // Similarly like with `double` above we can go straight to\n // `Layout::from_size_align_unchecked` as we know this won't\n // overflow and the alignment is sufficiently small.\n let new_cap = 2 * self.cap;\n let new_size = new_cap * elem_size;\n alloc_guard(new_size).unwrap_or_else(|_| capacity_overflow());\n match self.a.grow_in_place(self.ptr.cast(), old_layout, new_size) {\n Ok(_) => {\n // We can't directly divide `size`.\n self.cap = new_cap;\n true\n }\n Err(_) => false,\n }\n }\n }\n\n /// The same as `reserve_exact`, but returns on errors instead of panicking or aborting.\n pub fn try_reserve_exact(\n &mut self,\n used_cap: usize,\n needed_extra_cap: usize,\n ) -> Result<(), CollectionAllocErr> {\n self.fallible_reserve_internal(used_cap, needed_extra_cap, Exact)\n }\n\n /// Ensures that the buffer contains at least enough space to hold\n /// `used_cap + needed_extra_cap` elements. If it doesn't already,\n /// will reallocate the minimum possible amount of memory necessary.\n /// Generally this will be exactly the amount of memory necessary,\n /// but in principle the allocator is free to give back more than\n /// we asked for.\n ///\n /// If `used_cap` exceeds `self.cap()`, this may fail to actually allocate\n /// the requested space. This is not really unsafe, but the unsafe\n /// code *you* write that relies on the behavior of this function may break.\n ///\n /// # Panics\n ///\n /// * Panics if the requested capacity exceeds `usize::MAX` bytes.\n /// * Panics on 32-bit platforms if the requested capacity exceeds\n /// `isize::MAX` bytes.\n ///\n /// # Aborts\n ///\n /// Aborts on OOM\n pub fn reserve_exact(&mut self, used_cap: usize, needed_extra_cap: usize) {\n self.infallible_reserve_internal(used_cap, needed_extra_cap, Exact)\n }\n\n /// Calculates the buffer's new size given that it'll hold `used_cap +\n /// needed_extra_cap` elements. This logic is used in amortized reserve methods.\n /// Returns `(new_capacity, new_alloc_size)`.\n fn amortized_new_size(\n &self,\n used_cap: usize,\n needed_extra_cap: usize,\n ) -> Result {\n // Nothing we can really do about these checks :(\n let required_cap = used_cap\n .checked_add(needed_extra_cap)\n .ok_or(CapacityOverflow)?;\n // Cannot overflow, because `cap <= isize::MAX`, and type of `cap` is `usize`.\n let double_cap = self.cap * 2;\n // `double_cap` guarantees exponential growth.\n Ok(cmp::max(double_cap, required_cap))\n }\n\n /// The same as `reserve`, but returns on errors instead of panicking or aborting.\n pub fn try_reserve(\n &mut self,\n used_cap: usize,\n needed_extra_cap: usize,\n ) -> Result<(), CollectionAllocErr> {\n self.fallible_reserve_internal(used_cap, needed_extra_cap, Amortized)\n }\n\n /// Ensures that the buffer contains at least enough space to hold\n /// `used_cap + needed_extra_cap` elements. If it doesn't already have\n /// enough capacity, will reallocate enough space plus comfortable slack\n /// space to get amortized `O(1)` behavior. Will limit this behavior\n /// if it would needlessly cause itself to panic.\n ///\n /// If `used_cap` exceeds `self.cap()`, this may fail to actually allocate\n /// the requested space. This is not really unsafe, but the unsafe\n /// code *you* write that relies on the behavior of this function may break.\n ///\n /// This is ideal for implementing a bulk-push operation like `extend`.\n ///\n /// # Panics\n ///\n /// * Panics if the requested capacity exceeds `usize::MAX` bytes.\n /// * Panics on 32-bit platforms if the requested capacity exceeds\n /// `isize::MAX` bytes.\n ///\n /// # Aborts\n ///\n /// Aborts on OOM\n ///\n /// # Examples\n ///\n /// ```ignore\n /// # #![feature(alloc, raw_vec_internals)]\n /// # extern crate alloc;\n /// # use std::ptr;\n /// # use alloc::raw_vec::RawVec;\n /// struct MyVec {\n /// buf: RawVec,\n /// len: usize,\n /// }\n ///\n /// impl MyVec {\n /// pub fn push_all(&mut self, elems: &[T]) {\n /// self.buf.reserve(self.len, elems.len());\n /// // reserve would have aborted or panicked if the len exceeded\n /// // `isize::MAX` so this is safe to do unchecked now.\n /// for x in elems {\n /// unsafe {\n /// ptr::write(self.buf.ptr().add(self.len), x.clone());\n /// }\n /// self.len += 1;\n /// }\n /// }\n /// }\n /// # fn main() {\n /// # let mut vector = MyVec { buf: RawVec::new(), len: 0 };\n /// # vector.push_all(&[1, 3, 5, 7, 9]);\n /// # }\n /// ```\n #[inline(always)]\n pub fn reserve(&mut self, used_cap: usize, needed_extra_cap: usize) {\n self.infallible_reserve_internal(used_cap, needed_extra_cap, Amortized)\n }\n\n /// Attempts to ensure that the buffer contains at least enough space to hold\n /// `used_cap + needed_extra_cap` elements. If it doesn't already have\n /// enough capacity, will reallocate in place enough space plus comfortable slack\n /// space to get amortized `O(1)` behavior. Will limit this behaviour\n /// if it would needlessly cause itself to panic.\n ///\n /// If `used_cap` exceeds `self.cap()`, this may fail to actually allocate\n /// the requested space. This is not really unsafe, but the unsafe\n /// code *you* write that relies on the behavior of this function may break.\n ///\n /// Returns true if the reallocation attempt has succeeded, or false otherwise.\n ///\n /// # Panics\n ///\n /// * Panics if the requested capacity exceeds `usize::MAX` bytes.\n /// * Panics on 32-bit platforms if the requested capacity exceeds\n /// `isize::MAX` bytes.\n pub fn reserve_in_place(&mut self, used_cap: usize, needed_extra_cap: usize) -> bool {\n unsafe {\n // NOTE: we don't early branch on ZSTs here because we want this\n // to actually catch \"asking for more than usize::MAX\" in that case.\n // If we make it past the first branch then we are guaranteed to\n // panic.\n\n // Don't actually need any more capacity. If the current `cap` is 0, we can't\n // reallocate in place.\n // Wrapping in case they give a bad `used_cap`\n let old_layout = match self.current_layout() {\n Some(layout) => layout,\n None => return false,\n };\n if self.cap().wrapping_sub(used_cap) >= needed_extra_cap {\n return false;\n }\n\n let new_cap = self\n .amortized_new_size(used_cap, needed_extra_cap)\n .unwrap_or_else(|_| capacity_overflow());\n\n // Here, `cap < used_cap + needed_extra_cap <= new_cap`\n // (regardless of whether `self.cap - used_cap` wrapped).\n // Therefore we can safely call grow_in_place.\n\n let new_layout = Layout::new::().repeat(new_cap).unwrap().0;\n // FIXME: may crash and burn on over-reserve\n alloc_guard(new_layout.size()).unwrap_or_else(|_| capacity_overflow());\n match self\n .a\n .grow_in_place(self.ptr.cast(), old_layout, new_layout.size())\n {\n Ok(_) => {\n self.cap = new_cap;\n true\n }\n Err(_) => false,\n }\n }\n }\n\n /// Shrinks the allocation down to the specified amount. If the given amount\n /// is 0, actually completely deallocates.\n ///\n /// # Panics\n ///\n /// Panics if the given amount is *larger* than the current capacity.\n ///\n /// # Aborts\n ///\n /// Aborts on OOM.\n pub fn shrink_to_fit(&mut self, amount: usize) {\n let elem_size = mem::size_of::();\n\n // Set the `cap` because they might be about to promote to a `Box<[T]>`\n if elem_size == 0 {\n self.cap = amount;\n return;\n }\n\n // This check is my waterloo; it's the only thing Vec wouldn't have to do.\n assert!(self.cap >= amount, \"Tried to shrink to a larger capacity\");\n\n if amount == 0 {\n // We want to create a new zero-length vector within the\n // same allocator. We use ptr::write to avoid an\n // erroneous attempt to drop the contents, and we use\n // ptr::read to sidestep condition against destructuring\n // types that implement Drop.\n\n unsafe {\n let a = self.a;\n self.dealloc_buffer();\n ptr::write(self, RawVec::new_in(a));\n }\n } else if self.cap != amount {\n unsafe {\n // We know here that our `amount` is greater than zero. This\n // implies, via the assert above, that capacity is also greater\n // than zero, which means that we've got a current layout that\n // \"fits\"\n //\n // We also know that `self.cap` is greater than `amount`, and\n // consequently we don't need runtime checks for creating either\n // layout\n let old_size = elem_size * self.cap;\n let new_size = elem_size * amount;\n let align = mem::align_of::();\n let old_layout = Layout::from_size_align_unchecked(old_size, align);\n match self.a.realloc(self.ptr.cast(), old_layout, new_size) {\n Ok(p) => self.ptr = p.cast(),\n Err(_) => {\n handle_alloc_error(Layout::from_size_align_unchecked(new_size, align))\n }\n }\n }\n self.cap = amount;\n }\n }\n}\n\n#[cfg(feature = \"boxed\")]\nimpl<'a, T> RawVec<'a, T> {\n /// Converts the entire buffer into `Box<[T]>`.\n ///\n /// Note that this will correctly reconstitute any `cap` changes\n /// that may have been performed. (See description of type for details.)\n ///\n /// # Undefined Behavior\n ///\n /// All elements of `RawVec` must be initialized. Notice that\n /// the rules around uninitialized boxed values are not finalized yet,\n /// but until they are, it is advisable to avoid them.\n pub unsafe fn into_box(self) -> crate::boxed::Box<'a, [T]> {\n use crate::boxed::Box;\n\n // NOTE: not calling `cap()` here; actually using the real `cap` field!\n let slice = core::slice::from_raw_parts_mut(self.ptr(), self.cap);\n let output: Box<'a, [T]> = Box::from_raw(slice);\n mem::forget(self);\n output\n }\n}\n\nenum Fallibility {\n Fallible,\n Infallible,\n}\n\nuse self::Fallibility::*;\n\nenum ReserveStrategy {\n Exact,\n Amortized,\n}\n\nuse self::ReserveStrategy::*;\n\nimpl<'a, T> RawVec<'a, T> {\n #[inline(always)]\n fn fallible_reserve_internal(\n &mut self,\n used_cap: usize,\n needed_extra_cap: usize,\n strategy: ReserveStrategy,\n ) -> Result<(), CollectionAllocErr> {\n // This portion of the method should always be inlined.\n if self.cap().wrapping_sub(used_cap) >= needed_extra_cap {\n return Ok(());\n }\n // This portion of the method should never be inlined, and will only be called when\n // the check above has confirmed that it is necessary.\n self.reserve_internal_or_error(used_cap, needed_extra_cap, Fallible, strategy)\n }\n\n #[inline(always)]\n fn infallible_reserve_internal(\n &mut self,\n used_cap: usize,\n needed_extra_cap: usize,\n strategy: ReserveStrategy,\n ) {\n // This portion of the method should always be inlined.\n if self.cap().wrapping_sub(used_cap) >= needed_extra_cap {\n return;\n }\n // This portion of the method should never be inlined, and will only be called when\n // the check above has confirmed that it is necessary.\n self.reserve_internal_or_panic(used_cap, needed_extra_cap, strategy)\n }\n\n #[inline(never)]\n fn reserve_internal_or_panic(\n &mut self,\n used_cap: usize,\n needed_extra_cap: usize,\n strategy: ReserveStrategy,\n ) {\n // Delegates the call to `reserve_internal_or_error` and panics in the event of an error.\n // This allows the method to have a return type of `()`, simplifying the assembly at the\n // call site.\n match self.reserve_internal(used_cap, needed_extra_cap, Infallible, strategy) {\n Err(CapacityOverflow) => capacity_overflow(),\n Err(AllocErr) => unreachable!(),\n Ok(()) => { /* yay */ }\n }\n }\n\n #[inline(never)]\n fn reserve_internal_or_error(\n &mut self,\n used_cap: usize,\n needed_extra_cap: usize,\n fallibility: Fallibility,\n strategy: ReserveStrategy,\n ) -> Result<(), CollectionAllocErr> {\n // Delegates the call to `reserve_internal`, which can be inlined.\n self.reserve_internal(used_cap, needed_extra_cap, fallibility, strategy)\n }\n\n /// Helper method to reserve additional space, reallocating the backing memory.\n /// The caller is responsible for confirming that there is not already enough space available.\n fn reserve_internal(\n &mut self,\n used_cap: usize,\n needed_extra_cap: usize,\n fallibility: Fallibility,\n strategy: ReserveStrategy,\n ) -> Result<(), CollectionAllocErr> {\n unsafe {\n use crate::AllocErr;\n\n // NOTE: we don't early branch on ZSTs here because we want this\n // to actually catch \"asking for more than usize::MAX\" in that case.\n // If we make it past the first branch then we are guaranteed to\n // panic.\n\n // Nothing we can really do about these checks :(\n let new_cap = match strategy {\n Exact => used_cap\n .checked_add(needed_extra_cap)\n .ok_or(CapacityOverflow)?,\n Amortized => self.amortized_new_size(used_cap, needed_extra_cap)?,\n };\n let new_layout = Layout::array::(new_cap).map_err(|_| CapacityOverflow)?;\n\n alloc_guard(new_layout.size())?;\n\n let res = match self.current_layout() {\n Some(layout) => {\n debug_assert!(new_layout.align() == layout.align());\n self.a.realloc(self.ptr.cast(), layout, new_layout.size())\n }\n None => Alloc::alloc(&mut self.a, new_layout),\n };\n\n if let (Err(AllocErr), Infallible) = (&res, fallibility) {\n handle_alloc_error(new_layout);\n }\n\n self.ptr = res?.cast();\n self.cap = new_cap;\n\n Ok(())\n }\n }\n}\n\nimpl<'a, T> RawVec<'a, T> {\n /// Frees the memory owned by the RawVec *without* trying to Drop its contents.\n pub unsafe fn dealloc_buffer(&mut self) {\n let elem_size = mem::size_of::();\n if elem_size != 0 {\n if let Some(layout) = self.current_layout() {\n self.a.dealloc(self.ptr.cast(), layout);\n }\n }\n }\n}\n\nimpl<'a, T> Drop for RawVec<'a, T> {\n /// Frees the memory owned by the RawVec *without* trying to Drop its contents.\n fn drop(&mut self) {\n unsafe {\n self.dealloc_buffer();\n }\n }\n}\n\n// We need to guarantee the following:\n// * We don't ever allocate `> isize::MAX` byte-size objects\n// * We don't overflow `usize::MAX` and actually allocate too little\n//\n// On 64-bit we just need to check for overflow since trying to allocate\n// `> isize::MAX` bytes will surely fail. On 32-bit and 16-bit we need to add\n// an extra guard for this in case we're running on a platform which can use\n// all 4GB in user-space. e.g. PAE or x32\n\n#[inline]\nfn alloc_guard(alloc_size: usize) -> Result<(), CollectionAllocErr> {\n if mem::size_of::() < 8 && alloc_size > ::core::isize::MAX as usize {\n Err(CapacityOverflow)\n } else {\n Ok(())\n }\n}\n\n// One central function responsible for reporting capacity overflows. This'll\n// ensure that the code generation related to these panics is minimal as there's\n// only one location which panics rather than a bunch throughout the module.\nfn capacity_overflow() -> ! {\n panic!(\"capacity overflow\")\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn reserve_does_not_overallocate() {\n let bump = Bump::new();\n {\n let mut v: RawVec = RawVec::new_in(&bump);\n // First `reserve` allocates like `reserve_exact`\n v.reserve(0, 9);\n assert_eq!(9, v.cap());\n }\n\n {\n let mut v: RawVec = RawVec::new_in(&bump);\n v.reserve(0, 7);\n assert_eq!(7, v.cap());\n // 97 if more than double of 7, so `reserve` should work\n // like `reserve_exact`.\n v.reserve(7, 90);\n assert_eq!(97, v.cap());\n }\n\n {\n let mut v: RawVec = RawVec::new_in(&bump);\n v.reserve(0, 12);\n assert_eq!(12, v.cap());\n v.reserve(12, 3);\n // 3 is less than half of 12, so `reserve` must grow\n // exponentially. At the time of writing this test grow\n // factor is 2, so new capacity is 24, however, grow factor\n // of 1.5 is OK too. Hence `>= 18` in assert.\n assert!(v.cap() >= 12 + 12 / 2);\n }\n }\n}\n" }, { "package": "bumpalo@3.20.3", "sourcePath": "src/collections/str/lossy.rs", "sha256": "c5d62b16e01071e2a574ae41ef6693ad12f1e6c786c5d38f7a13ebd6cb23c088", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::collections::str as core_str;\nuse core::char;\nuse core::fmt;\nuse core::fmt::Write;\nuse core::str;\n\n/// Lossy UTF-8 string.\npub struct Utf8Lossy<'a> {\n bytes: &'a [u8],\n}\n\nimpl<'a> Utf8Lossy<'a> {\n pub fn from_bytes(bytes: &'a [u8]) -> Utf8Lossy<'a> {\n Utf8Lossy { bytes }\n }\n\n pub fn chunks(&self) -> Utf8LossyChunksIter<'a> {\n Utf8LossyChunksIter {\n source: &self.bytes,\n }\n }\n}\n\n/// Iterator over lossy UTF-8 string\n#[allow(missing_debug_implementations)]\npub struct Utf8LossyChunksIter<'a> {\n source: &'a [u8],\n}\n\n#[derive(PartialEq, Eq, Debug)]\npub struct Utf8LossyChunk<'a> {\n /// Sequence of valid chars.\n /// Can be empty between broken UTF-8 chars.\n pub valid: &'a str,\n /// Single broken char, empty if none.\n /// Empty iff iterator item is last.\n pub broken: &'a [u8],\n}\n\nimpl<'a> Iterator for Utf8LossyChunksIter<'a> {\n type Item = Utf8LossyChunk<'a>;\n\n fn next(&mut self) -> Option> {\n if self.source.is_empty() {\n return None;\n }\n\n const TAG_CONT_U8: u8 = 128;\n fn unsafe_get(xs: &[u8], i: usize) -> u8 {\n unsafe { *xs.get_unchecked(i) }\n }\n fn safe_get(xs: &[u8], i: usize) -> u8 {\n if i >= xs.len() {\n 0\n } else {\n unsafe_get(xs, i)\n }\n }\n\n let mut i = 0;\n while i < self.source.len() {\n let i_ = i;\n\n let byte = unsafe_get(self.source, i);\n i += 1;\n\n if byte < 128 {\n } else {\n let w = core_str::utf8_char_width(byte);\n\n macro_rules! error {\n () => {{\n unsafe {\n let r = Utf8LossyChunk {\n valid: str::from_utf8_unchecked(&self.source[0..i_]),\n broken: &self.source[i_..i],\n };\n self.source = &self.source[i..];\n return Some(r);\n }\n }};\n }\n\n match w {\n 2 => {\n if safe_get(self.source, i) & 192 != TAG_CONT_U8 {\n error!();\n }\n i += 1;\n }\n 3 => {\n match (byte, safe_get(self.source, i)) {\n (0xE0, 0xA0..=0xBF) => (),\n (0xE1..=0xEC, 0x80..=0xBF) => (),\n (0xED, 0x80..=0x9F) => (),\n (0xEE..=0xEF, 0x80..=0xBF) => (),\n _ => {\n error!();\n }\n }\n i += 1;\n if safe_get(self.source, i) & 192 != TAG_CONT_U8 {\n error!();\n }\n i += 1;\n }\n 4 => {\n match (byte, safe_get(self.source, i)) {\n (0xF0, 0x90..=0xBF) => (),\n (0xF1..=0xF3, 0x80..=0xBF) => (),\n (0xF4, 0x80..=0x8F) => (),\n _ => {\n error!();\n }\n }\n i += 1;\n if safe_get(self.source, i) & 192 != TAG_CONT_U8 {\n error!();\n }\n i += 1;\n if safe_get(self.source, i) & 192 != TAG_CONT_U8 {\n error!();\n }\n i += 1;\n }\n _ => {\n error!();\n }\n }\n }\n }\n\n let r = Utf8LossyChunk {\n valid: unsafe { str::from_utf8_unchecked(self.source) },\n broken: &[],\n };\n self.source = &[];\n Some(r)\n }\n}\n\nimpl<'a> fmt::Display for Utf8Lossy<'a> {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n // If we're the empty string then our iterator won't actually yield\n // anything, so perform the formatting manually\n if self.bytes.is_empty() {\n return \"\".fmt(f);\n }\n\n for Utf8LossyChunk { valid, broken } in self.chunks() {\n // If we successfully decoded the whole chunk as a valid string then\n // we can return a direct formatting of the string which will also\n // respect various formatting flags if possible.\n if valid.len() == self.bytes.len() {\n assert!(broken.is_empty());\n return valid.fmt(f);\n }\n\n f.write_str(valid)?;\n if !broken.is_empty() {\n f.write_char(char::REPLACEMENT_CHARACTER)?;\n }\n }\n Ok(())\n }\n}\n\nimpl<'a> fmt::Debug for Utf8Lossy<'a> {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n f.write_char('\"')?;\n\n for Utf8LossyChunk { valid, broken } in self.chunks() {\n // Valid part.\n // Here we partially parse UTF-8 again which is suboptimal.\n {\n let mut from = 0;\n for (i, c) in valid.char_indices() {\n let esc = c.escape_debug();\n // If char needs escaping, flush backlog so far and write, else skip\n if esc.len() != 1 {\n f.write_str(&valid[from..i])?;\n for c in esc {\n f.write_char(c)?;\n }\n from = i + c.len_utf8();\n }\n }\n f.write_str(&valid[from..])?;\n }\n\n // Broken parts of string as hex escape.\n for &b in broken {\n write!(f, \"\\\\x{:02x}\", b)?;\n }\n }\n\n f.write_char('\"')\n }\n}\n" }, { "package": "bumpalo@3.20.3", "sourcePath": "src/collections/str/mod.rs", "sha256": "d82a8bd417fbf52a589d89a16ea2a0ac4f6ac920c3976ab1f5b6ac0c8493c4f2", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! String manipulation\n//!\n//! For more details, see std::str\n\n#[allow(missing_docs)]\npub mod lossy;\n\n// https://tools.ietf.org/html/rfc3629\n#[rustfmt::skip]\nstatic UTF8_CHAR_WIDTH: [u8; 256] = [\n1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,\n1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x1F\n1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,\n1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x3F\n1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,\n1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x5F\n1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,\n1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x7F\n0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,\n0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0x9F\n0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,\n0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0xBF\n0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,\n2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // 0xDF\n3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, // 0xEF\n4,4,4,4,4,0,0,0,0,0,0,0,0,0,0,0, // 0xFF\n];\n\n/// Given a first byte, determines how many bytes are in this UTF-8 character.\n#[inline]\npub fn utf8_char_width(b: u8) -> usize {\n UTF8_CHAR_WIDTH[b as usize] as usize\n}\n" }, { "package": "bumpalo@3.20.3", "sourcePath": "src/collections/string.rs", "sha256": "9cbca6a0c6ae8b9c961d7a140d5146ba4c1f2e7a317c33e2e693911f2848044d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! A UTF-8 encoded, growable string.\n//!\n//! This module contains the [`String`] type and several error types that may\n//! result from working with [`String`]s.\n//!\n//! This module is a fork of the [`std::string`] module, that uses a bump allocator.\n//!\n//! [`std::string`]: https://doc.rust-lang.org/std/string/index.html\n//!\n//! # Examples\n//!\n//! You can create a new [`String`] from a string literal with [`String::from_str_in`]:\n//!\n//! ```\n//! use bumpalo::{Bump, collections::String};\n//!\n//! let b = Bump::new();\n//!\n//! let s = String::from_str_in(\"world\", &b);\n//! ```\n//!\n//! [`String`]: struct.String.html\n//! [`String::from_str_in`]: struct.String.html#method.from_str_in\n//!\n//! If you have a vector of valid UTF-8 bytes, you can make a [`String`] out of\n//! it. You can do the reverse too.\n//!\n//! ```\n//! use bumpalo::{Bump, collections::String};\n//!\n//! let b = Bump::new();\n//!\n//! let sparkle_heart = bumpalo::vec![in &b; 240, 159, 146, 150];\n//!\n//! // We know these bytes are valid, so we'll use `unwrap()`.\n//! let sparkle_heart = String::from_utf8(sparkle_heart).unwrap();\n//!\n//! assert_eq!(\"💖\", sparkle_heart);\n//!\n//! let bytes = sparkle_heart.into_bytes();\n//!\n//! assert_eq!(bytes, [240, 159, 146, 150]);\n//! ```\n\nuse crate::collections::str::lossy;\nuse crate::collections::vec::Vec;\nuse crate::Bump;\nuse core::borrow::{Borrow, BorrowMut};\nuse core::char::decode_utf16;\nuse core::fmt;\nuse core::hash;\nuse core::iter::FusedIterator;\nuse core::mem;\nuse core::ops::Bound::{Excluded, Included, Unbounded};\nuse core::ops::{self, Add, AddAssign, Index, IndexMut, RangeBounds};\nuse core::ptr;\nuse core::str::{self, Chars, Utf8Error};\nuse core_alloc::borrow::Cow;\n\n/// Like the [`format!`] macro, but for creating [`bumpalo::collections::String`]s.\n///\n/// [`format!`]: https://doc.rust-lang.org/std/macro.format.html\n/// [`bumpalo::collections::String`]: collections/string/struct.String.html\n///\n/// # Examples\n///\n/// ```\n/// use bumpalo::Bump;\n///\n/// let b = Bump::new();\n///\n/// let who = \"World\";\n/// let s = bumpalo::format!(in &b, \"Hello, {}!\", who);\n/// assert_eq!(s, \"Hello, World!\")\n/// ```\n#[macro_export]\nmacro_rules! format {\n ( in $bump:expr, $fmt:expr, $($args:expr),* ) => {{\n use $crate::core_alloc::fmt::Write;\n let bump = $bump;\n let mut s = $crate::collections::String::new_in(bump);\n let _ = write!(&mut s, $fmt, $($args),*);\n s\n }};\n\n ( in $bump:expr, $fmt:expr, $($args:expr,)* ) => {\n $crate::format!(in $bump, $fmt, $($args),*)\n };\n}\n\n/// A UTF-8 encoded, growable string.\n///\n/// The `String` type is the most common string type that has ownership over the\n/// contents of the string. It has a close relationship with its borrowed\n/// counterpart, the primitive [`str`].\n///\n/// [`str`]: https://doc.rust-lang.org/std/primitive.str.html\n///\n/// # Examples\n///\n/// You can create a `String` from a literal string with [`String::from_str_in`]:\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n///\n/// let b = Bump::new();\n///\n/// let hello = String::from_str_in(\"Hello, world!\", &b);\n/// ```\n///\n/// You can append a [`char`] to a `String` with the [`push`] method, and\n/// append a [`&str`] with the [`push_str`] method:\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n///\n/// let b = Bump::new();\n///\n/// let mut hello = String::from_str_in(\"Hello, \", &b);\n///\n/// hello.push('w');\n/// hello.push_str(\"orld!\");\n/// ```\n///\n/// [`char`]: https://doc.rust-lang.org/std/primitive.char.html\n/// [`push`]: #method.push\n/// [`push_str`]: #method.push_str\n///\n/// If you have a vector of UTF-8 bytes, you can create a `String` from it with\n/// the [`from_utf8`] method:\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n///\n/// let b = Bump::new();\n///\n/// // some bytes, in a vector\n/// let sparkle_heart = bumpalo::vec![in &b; 240, 159, 146, 150];\n///\n/// // We know these bytes are valid, so we'll use `unwrap()`.\n/// let sparkle_heart = String::from_utf8(sparkle_heart).unwrap();\n///\n/// assert_eq!(\"💖\", sparkle_heart);\n/// ```\n///\n/// [`from_utf8`]: #method.from_utf8\n///\n/// # Deref\n///\n/// `String`s implement [`Deref`], and so inherit all of [`str`]'s\n/// methods. In addition, this means that you can pass a `String` to a\n/// function which takes a [`&str`] by using an ampersand (`&`):\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n///\n/// let b = Bump::new();\n///\n/// fn takes_str(s: &str) { }\n///\n/// let s = String::from_str_in(\"Hello\", &b);\n///\n/// takes_str(&s);\n/// ```\n///\n/// This will create a [`&str`] from the `String` and pass it in. This\n/// conversion is very inexpensive, and so generally, functions will accept\n/// [`&str`]s as arguments unless they need a `String` for some specific\n/// reason.\n///\n/// In certain cases Rust doesn't have enough information to make this\n/// conversion, known as [`Deref`] coercion. In the following example a string\n/// slice [`&'a str`][`&str`] implements the trait `TraitExample`, and the function\n/// `example_func` takes anything that implements the trait. In this case Rust\n/// would need to make two implicit conversions, which Rust doesn't have the\n/// means to do. For that reason, the following example will not compile.\n///\n/// ```compile_fail,E0277\n/// use bumpalo::{Bump, collections::String};\n///\n/// trait TraitExample {}\n///\n/// impl<'a> TraitExample for &'a str {}\n///\n/// fn example_func(example_arg: A) {}\n///\n/// let b = Bump::new();\n/// let example_string = String::from_str_in(\"example_string\", &b);\n/// example_func(&example_string);\n/// ```\n///\n/// There are two options that would work instead. The first would be to\n/// change the line `example_func(&example_string);` to\n/// `example_func(example_string.as_str());`, using the method [`as_str()`]\n/// to explicitly extract the string slice containing the string. The second\n/// way changes `example_func(&example_string);` to\n/// `example_func(&*example_string);`. In this case we are dereferencing a\n/// `String` to a [`str`][`&str`], then referencing the [`str`][`&str`] back to\n/// [`&str`]. The second way is more idiomatic, however both work to do the\n/// conversion explicitly rather than relying on the implicit conversion.\n///\n/// # Representation\n///\n/// A `String` is made up of three components: a pointer to some bytes, a\n/// length, and a capacity. The pointer points to an internal buffer `String`\n/// uses to store its data. The length is the number of bytes currently stored\n/// in the buffer, and the capacity is the size of the buffer in bytes. As such,\n/// the length will always be less than or equal to the capacity.\n///\n/// This buffer is always stored on the heap.\n///\n/// You can look at these with the [`as_ptr`], [`len`], and [`capacity`]\n/// methods:\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n/// use std::mem;\n///\n/// let b = Bump::new();\n///\n/// let mut story = String::from_str_in(\"Once upon a time...\", &b);\n///\n/// let ptr = story.as_mut_ptr();\n/// let len = story.len();\n/// let capacity = story.capacity();\n///\n/// // story has nineteen bytes\n/// assert_eq!(19, len);\n///\n/// // Now that we have our parts, we throw the story away.\n/// mem::forget(story);\n///\n/// // We can re-build a String out of ptr, len, and capacity. This is all\n/// // unsafe because we are responsible for making sure the components are\n/// // valid:\n/// let s = unsafe { String::from_raw_parts_in(ptr, len, capacity, &b) } ;\n///\n/// assert_eq!(String::from_str_in(\"Once upon a time...\", &b), s);\n/// ```\n///\n/// [`as_ptr`]: https://doc.rust-lang.org/std/primitive.str.html#method.as_ptr\n/// [`len`]: #method.len\n/// [`capacity`]: #method.capacity\n///\n/// If a `String` has enough capacity, adding elements to it will not\n/// re-allocate. For example, consider this program:\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n///\n/// let b = Bump::new();\n///\n/// let mut s = String::new_in(&b);\n///\n/// println!(\"{}\", s.capacity());\n///\n/// for _ in 0..5 {\n/// s.push_str(\"hello\");\n/// println!(\"{}\", s.capacity());\n/// }\n/// ```\n///\n/// This will output the following:\n///\n/// ```text\n/// 0\n/// 5\n/// 10\n/// 20\n/// 20\n/// 40\n/// ```\n///\n/// At first, we have no memory allocated at all, but as we append to the\n/// string, it increases its capacity appropriately. If we instead use the\n/// [`with_capacity_in`] method to allocate the correct capacity initially:\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n///\n/// let b = Bump::new();\n///\n/// let mut s = String::with_capacity_in(25, &b);\n///\n/// println!(\"{}\", s.capacity());\n///\n/// for _ in 0..5 {\n/// s.push_str(\"hello\");\n/// println!(\"{}\", s.capacity());\n/// }\n/// ```\n///\n/// [`with_capacity_in`]: #method.with_capacity_in\n///\n/// We end up with a different output:\n///\n/// ```text\n/// 25\n/// 25\n/// 25\n/// 25\n/// 25\n/// 25\n/// ```\n///\n/// Here, there's no need to allocate more memory inside the loop.\n///\n/// [`&str`]: https://doc.rust-lang.org/std/primitive.str.html\n/// [`Deref`]: https://doc.rust-lang.org/std/ops/trait.Deref.html\n/// [`as_str()`]: struct.String.html#method.as_str\n#[derive(PartialOrd, Eq, Ord)]\npub struct String<'bump> {\n vec: Vec<'bump, u8>,\n}\n\n/// A possible error value when converting a `String` from a UTF-8 byte vector.\n///\n/// This type is the error type for the [`from_utf8`] method on [`String`]. It\n/// is designed in such a way to carefully avoid reallocations: the\n/// [`into_bytes`] method will give back the byte vector that was used in the\n/// conversion attempt.\n///\n/// [`from_utf8`]: struct.String.html#method.from_utf8\n/// [`String`]: struct.String.html\n/// [`into_bytes`]: struct.FromUtf8Error.html#method.into_bytes\n///\n/// The [`Utf8Error`] type provided by [`std::str`] represents an error that may\n/// occur when converting a slice of [`u8`]s to a [`&str`]. In this sense, it's\n/// an analogue to `FromUtf8Error`, and you can get one from a `FromUtf8Error`\n/// through the [`utf8_error`] method.\n///\n/// [`Utf8Error`]: https://doc.rust-lang.org/std/str/struct.Utf8Error.html\n/// [`std::str`]: https://doc.rust-lang.org/std/str/index.html\n/// [`u8`]: https://doc.rust-lang.org/std/primitive.u8.html\n/// [`&str`]: https://doc.rust-lang.org/std/primitive.str.html\n/// [`utf8_error`]: #method.utf8_error\n///\n/// # Examples\n///\n/// Basic usage:\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n///\n/// let b = Bump::new();\n///\n/// // some invalid bytes, in a vector\n/// let bytes = bumpalo::vec![in &b; 0, 159];\n///\n/// let value = String::from_utf8(bytes);\n///\n/// assert!(value.is_err());\n/// assert_eq!(bumpalo::vec![in &b; 0, 159], value.unwrap_err().into_bytes());\n/// ```\n#[derive(Debug)]\npub struct FromUtf8Error<'bump> {\n bytes: Vec<'bump, u8>,\n error: Utf8Error,\n}\n\n/// A possible error value when converting a `String` from a UTF-16 byte slice.\n///\n/// This type is the error type for the [`from_utf16_in`] method on [`String`].\n///\n/// [`from_utf16_in`]: struct.String.html#method.from_utf16_in\n/// [`String`]: struct.String.html\n///\n/// # Examples\n///\n/// Basic usage:\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n///\n/// let b = Bump::new();\n///\n/// // 𝄞muic\n/// let v = &[0xD834, 0xDD1E, 0x006d, 0x0075, 0xD800, 0x0069, 0x0063];\n///\n/// assert!(String::from_utf16_in(v, &b).is_err());\n/// ```\n#[derive(Debug)]\npub struct FromUtf16Error(());\n\nimpl<'bump> String<'bump> {\n /// Creates a new empty `String`.\n ///\n /// Given that the `String` is empty, this will not allocate any initial\n /// buffer. While that means that this initial operation is very\n /// inexpensive, it may cause excessive allocation later when you add\n /// data. If you have an idea of how much data the `String` will hold,\n /// consider the [`with_capacity_in`] method to prevent excessive\n /// re-allocation.\n ///\n /// [`with_capacity_in`]: #method.with_capacity_in\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let s = String::new_in(&b);\n /// ```\n #[inline]\n pub fn new_in(bump: &'bump Bump) -> String<'bump> {\n String {\n vec: Vec::new_in(bump),\n }\n }\n\n /// Creates a new empty `String` with a particular capacity.\n ///\n /// `String`s have an internal buffer to hold their data. The capacity is\n /// the length of that buffer, and can be queried with the [`capacity`]\n /// method. This method creates an empty `String`, but one with an initial\n /// buffer that can hold `capacity` bytes. This is useful when you may be\n /// appending a bunch of data to the `String`, reducing the number of\n /// reallocations it needs to do.\n ///\n /// [`capacity`]: #method.capacity\n ///\n /// If the given capacity is `0`, no allocation will occur, and this method\n /// is identical to the [`new_in`] method.\n ///\n /// [`new_in`]: #method.new\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::with_capacity_in(10, &b);\n ///\n /// // The String contains no chars, even though it has capacity for more\n /// assert_eq!(s.len(), 0);\n ///\n /// // These are all done without reallocating...\n /// let cap = s.capacity();\n /// for _ in 0..10 {\n /// s.push('a');\n /// }\n ///\n /// assert_eq!(s.capacity(), cap);\n ///\n /// // ...but this may make the vector reallocate\n /// s.push('a');\n /// ```\n #[inline]\n pub fn with_capacity_in(capacity: usize, bump: &'bump Bump) -> String<'bump> {\n String {\n vec: Vec::with_capacity_in(capacity, bump),\n }\n }\n\n /// Converts a vector of bytes to a `String`.\n ///\n /// A string (`String`) is made of bytes ([`u8`]), and a vector of bytes\n /// ([`Vec`]) is made of bytes, so this function converts between the\n /// two. Not all byte slices are valid `String`s, however: `String`\n /// requires that it is valid UTF-8. `from_utf8()` checks to ensure that\n /// the bytes are valid UTF-8, and then does the conversion.\n ///\n /// If you are sure that the byte slice is valid UTF-8, and you don't want\n /// to incur the overhead of the validity check, there is an unsafe version\n /// of this function, [`from_utf8_unchecked`], which has the same behavior\n /// but skips the check.\n ///\n /// This method will take care to not copy the vector, for efficiency's\n /// sake.\n ///\n /// If you need a [`&str`] instead of a `String`, consider\n /// [`str::from_utf8`].\n ///\n /// The inverse of this method is [`into_bytes`].\n ///\n /// # Errors\n ///\n /// Returns [`Err`] if the slice is not UTF-8 with a description as to why the\n /// provided bytes are not UTF-8. The vector you moved in is also included.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// // some bytes, in a vector\n /// let sparkle_heart = bumpalo::vec![in &b; 240, 159, 146, 150];\n ///\n /// // We know these bytes are valid, so we'll use `unwrap()`.\n /// let sparkle_heart = String::from_utf8(sparkle_heart).unwrap();\n ///\n /// assert_eq!(\"💖\", sparkle_heart);\n /// ```\n ///\n /// Incorrect bytes:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// // some invalid bytes, in a vector\n /// let sparkle_heart = bumpalo::vec![in &b; 0, 159, 146, 150];\n ///\n /// assert!(String::from_utf8(sparkle_heart).is_err());\n /// ```\n ///\n /// See the docs for [`FromUtf8Error`] for more details on what you can do\n /// with this error.\n ///\n /// [`from_utf8_unchecked`]: struct.String.html#method.from_utf8_unchecked\n /// [`&str`]: https://doc.rust-lang.org/std/primitive.str.html\n /// [`u8`]: https://doc.rust-lang.org/std/primitive.u8.html\n /// [`Vec`]: ../vec/struct.Vec.html\n /// [`str::from_utf8`]: https://doc.rust-lang.org/std/str/fn.from_utf8.html\n /// [`into_bytes`]: struct.String.html#method.into_bytes\n /// [`FromUtf8Error`]: struct.FromUtf8Error.html\n /// [`Err`]: https://doc.rust-lang.org/std/result/enum.Result.html#variant.Err\n #[inline]\n pub fn from_utf8(vec: Vec<'bump, u8>) -> Result, FromUtf8Error<'bump>> {\n match str::from_utf8(&vec) {\n Ok(..) => Ok(String { vec }),\n Err(e) => Err(FromUtf8Error {\n bytes: vec,\n error: e,\n }),\n }\n }\n\n /// Converts a slice of bytes to a string, including invalid characters.\n ///\n /// Strings are made of bytes ([`u8`]), and a slice of bytes\n /// ([`&[u8]`][slice]) is made of bytes, so this function converts\n /// between the two. Not all byte slices are valid strings, however: strings\n /// are required to be valid UTF-8. During this conversion,\n /// `from_utf8_lossy_in()` will replace any invalid UTF-8 sequences with\n /// [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD], which looks like this: �\n ///\n /// [`u8`]: https://doc.rust-lang.org/std/primitive.u8.html\n /// [slice]: https://doc.rust-lang.org/std/primitive.slice.html\n /// [U+FFFD]: https://doc.rust-lang.org/std/char/constant.REPLACEMENT_CHARACTER.html\n ///\n /// If you are sure that the byte slice is valid UTF-8, and you don't want\n /// to incur the overhead of the conversion, there is an unsafe version\n /// of this function, [`from_utf8_unchecked`], which has the same behavior\n /// but skips the checks.\n ///\n /// [`from_utf8_unchecked`]: struct.String.html#method.from_utf8_unchecked\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{collections::String, Bump, vec};\n ///\n /// let b = Bump::new();\n ///\n /// // some bytes, in a vector\n /// let sparkle_heart = bumpalo::vec![in &b; 240, 159, 146, 150];\n ///\n /// let sparkle_heart = String::from_utf8_lossy_in(&sparkle_heart, &b);\n ///\n /// assert_eq!(\"💖\", sparkle_heart);\n /// ```\n ///\n /// Incorrect bytes:\n ///\n /// ```\n /// use bumpalo::{collections::String, Bump, vec};\n ///\n /// let b = Bump::new();\n ///\n /// // some invalid bytes\n /// let input = b\"Hello \\xF0\\x90\\x80World\";\n /// let output = String::from_utf8_lossy_in(input, &b);\n ///\n /// assert_eq!(\"Hello �World\", output);\n /// ```\n pub fn from_utf8_lossy_in(v: &[u8], bump: &'bump Bump) -> String<'bump> {\n let mut iter = lossy::Utf8Lossy::from_bytes(v).chunks();\n\n let (first_valid, first_broken) = if let Some(chunk) = iter.next() {\n let lossy::Utf8LossyChunk { valid, broken } = chunk;\n if valid.len() == v.len() {\n debug_assert!(broken.is_empty());\n unsafe {\n return String::from_utf8_unchecked(Vec::from_iter_in(v.iter().cloned(), bump));\n }\n }\n (valid, broken)\n } else {\n return String::from_str_in(\"\", bump);\n };\n\n const REPLACEMENT: &str = \"\\u{FFFD}\";\n\n let mut res = String::with_capacity_in(v.len(), bump);\n res.push_str(first_valid);\n if !first_broken.is_empty() {\n res.push_str(REPLACEMENT);\n }\n\n for lossy::Utf8LossyChunk { valid, broken } in iter {\n res.push_str(valid);\n if !broken.is_empty() {\n res.push_str(REPLACEMENT);\n }\n }\n\n res\n }\n\n /// Decode a UTF-16 encoded slice `v` into a `String`, returning [`Err`]\n /// if `v` contains any invalid data.\n ///\n /// [`Err`]: https://doc.rust-lang.org/std/result/enum.Result.html#variant.Err\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// // 𝄞music\n /// let v = &[0xD834, 0xDD1E, 0x006d, 0x0075, 0x0073, 0x0069, 0x0063];\n /// assert_eq!(String::from_str_in(\"𝄞music\", &b), String::from_utf16_in(v, &b).unwrap());\n ///\n /// // 𝄞muic\n /// let v = &[0xD834, 0xDD1E, 0x006d, 0x0075, 0xD800, 0x0069, 0x0063];\n /// assert!(String::from_utf16_in(v, &b).is_err());\n /// ```\n pub fn from_utf16_in(v: &[u16], bump: &'bump Bump) -> Result, FromUtf16Error> {\n let mut ret = String::with_capacity_in(v.len(), bump);\n for c in decode_utf16(v.iter().cloned()) {\n if let Ok(c) = c {\n ret.push(c);\n } else {\n return Err(FromUtf16Error(()));\n }\n }\n Ok(ret)\n }\n\n /// Construct a new `String<'bump>` from a string slice.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let s = String::from_str_in(\"hello\", &b);\n /// assert_eq!(s, \"hello\");\n /// ```\n #[inline]\n pub fn from_str_in(s: &str, bump: &'bump Bump) -> String<'bump> {\n let len = s.len();\n let mut t = String::with_capacity_in(len, bump);\n // SAFETY:\n // * `src` is valid for reads of `s.len()` bytes by virtue of being an allocated `&str`.\n // * `dst` is valid for writes of `s.len()` bytes as `String::with_capacity_in(s.len(), bump)`\n // above guarantees that.\n // * Alignment is not relevant as `u8` has no alignment requirements.\n // * Source and destination ranges cannot overlap as we just reserved the destination\n // range from the bump.\n unsafe { ptr::copy_nonoverlapping(s.as_ptr(), t.vec.as_mut_ptr(), len) };\n // SAFETY: We reserved sufficent capacity for the string above.\n // The elements at `0..len` were initialized by `copy_nonoverlapping` above.\n unsafe { t.vec.set_len(len) };\n t\n }\n\n /// Construct a new `String<'bump>` from an iterator of `char`s.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let s = String::from_iter_in(['h', 'e', 'l', 'l', 'o'].iter().cloned(), &b);\n /// assert_eq!(s, \"hello\");\n /// ```\n pub fn from_iter_in>(iter: I, bump: &'bump Bump) -> String<'bump> {\n let mut s = String::new_in(bump);\n for c in iter {\n s.push(c);\n }\n s\n }\n\n /// Creates a new `String` from a length, capacity, and pointer.\n ///\n /// # Safety\n ///\n /// This is highly unsafe, due to the number of invariants that aren't\n /// checked:\n ///\n /// * The memory at `ptr` needs to have been previously allocated by the\n /// same allocator the standard library uses.\n /// * `length` needs to be less than or equal to `capacity`.\n /// * `capacity` needs to be the correct value.\n ///\n /// Violating these may cause problems like corrupting the allocator's\n /// internal data structures.\n ///\n /// The ownership of `ptr` is effectively transferred to the\n /// `String` which may then deallocate, reallocate or change the\n /// contents of memory pointed to by the pointer at will. Ensure\n /// that nothing else uses the pointer after calling this\n /// function.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n /// use std::mem;\n ///\n /// let b = Bump::new();\n ///\n /// unsafe {\n /// let mut s = String::from_str_in(\"hello\", &b);\n /// let ptr = s.as_mut_ptr();\n /// let len = s.len();\n /// let capacity = s.capacity();\n ///\n /// mem::forget(s);\n ///\n /// let s = String::from_raw_parts_in(ptr, len, capacity, &b);\n ///\n /// assert_eq!(s, \"hello\");\n /// }\n /// ```\n #[inline]\n pub unsafe fn from_raw_parts_in(\n buf: *mut u8,\n length: usize,\n capacity: usize,\n bump: &'bump Bump,\n ) -> String<'bump> {\n String {\n vec: Vec::from_raw_parts_in(buf, length, capacity, bump),\n }\n }\n\n /// Converts a vector of bytes to a `String` without checking that the\n /// string contains valid UTF-8.\n ///\n /// See the safe version, [`from_utf8`], for more details.\n ///\n /// [`from_utf8`]: struct.String.html#method.from_utf8\n ///\n /// # Safety\n ///\n /// This function is unsafe because it does not check that the bytes passed\n /// to it are valid UTF-8. If this constraint is violated, it may cause\n /// memory unsafety issues with future users of the `String`,\n /// as it is assumed that `String`s are valid UTF-8.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// // some bytes, in a vector\n /// let sparkle_heart = bumpalo::vec![in &b; 240, 159, 146, 150];\n ///\n /// let sparkle_heart = unsafe {\n /// String::from_utf8_unchecked(sparkle_heart)\n /// };\n ///\n /// assert_eq!(\"💖\", sparkle_heart);\n /// ```\n #[inline]\n pub unsafe fn from_utf8_unchecked(bytes: Vec<'bump, u8>) -> String<'bump> {\n String { vec: bytes }\n }\n\n /// Returns a shared reference to the allocator backing this `String`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// // uses the same allocator as the provided `String`\n /// fn copy_string<'bump>(s: &String<'bump>) -> &'bump str {\n /// s.bump().alloc_str(s.as_str())\n /// }\n /// ```\n #[inline]\n #[must_use]\n pub fn bump(&self) -> &'bump Bump {\n self.vec.bump()\n }\n\n /// Converts a `String` into a byte vector.\n ///\n /// This consumes the `String`, so we do not need to copy its contents.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let s = String::from_str_in(\"hello\", &b);\n ///\n /// assert_eq!(s.into_bytes(), [104, 101, 108, 108, 111]);\n /// ```\n #[inline]\n pub fn into_bytes(self) -> Vec<'bump, u8> {\n self.vec\n }\n\n /// Convert this `String<'bump>` into a `&'bump str`. This is analogous to\n /// [`std::string::String::into_boxed_str`][into_boxed_str].\n ///\n /// [into_boxed_str]: https://doc.rust-lang.org/std/string/struct.String.html#method.into_boxed_str\n ///\n /// # Example\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let s = String::from_str_in(\"foo\", &b);\n ///\n /// assert_eq!(s.into_bump_str(), \"foo\");\n /// ```\n pub fn into_bump_str(self) -> &'bump str {\n let s = unsafe {\n let s = self.as_str();\n mem::transmute(s)\n };\n mem::forget(self);\n s\n }\n\n /// Extracts a string slice containing the entire `String`.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let s = String::from_str_in(\"foo\", &b);\n ///\n /// assert_eq!(\"foo\", s.as_str());\n /// ```\n #[inline]\n pub fn as_str(&self) -> &str {\n self\n }\n\n /// Converts a `String` into a mutable string slice.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"foobar\", &b);\n /// let s_mut_str = s.as_mut_str();\n ///\n /// s_mut_str.make_ascii_uppercase();\n ///\n /// assert_eq!(\"FOOBAR\", s_mut_str);\n /// ```\n #[inline]\n pub fn as_mut_str(&mut self) -> &mut str {\n self\n }\n\n /// Appends a given string slice onto the end of this `String`.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"foo\", &b);\n ///\n /// s.push_str(\"bar\");\n ///\n /// assert_eq!(\"foobar\", s);\n /// ```\n #[inline]\n pub fn push_str(&mut self, string: &str) {\n self.vec.extend_from_slice_copy(string.as_bytes())\n }\n\n /// Returns this `String`'s capacity, in bytes.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let s = String::with_capacity_in(10, &b);\n ///\n /// assert!(s.capacity() >= 10);\n /// ```\n #[inline]\n pub fn capacity(&self) -> usize {\n self.vec.capacity()\n }\n\n /// Ensures that this `String`'s capacity is at least `additional` bytes\n /// larger than its length.\n ///\n /// The capacity may be increased by more than `additional` bytes if it\n /// chooses, to prevent frequent reallocations.\n ///\n /// If you do not want this \"at least\" behavior, see the [`reserve_exact`]\n /// method.\n ///\n /// # Panics\n ///\n /// Panics if the new capacity overflows [`usize`].\n ///\n /// [`reserve_exact`]: struct.String.html#method.reserve_exact\n /// [`usize`]: https://doc.rust-lang.org/std/primitive.usize.html\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::new_in(&b);\n ///\n /// s.reserve(10);\n ///\n /// assert!(s.capacity() >= 10);\n /// ```\n ///\n /// This may not actually increase the capacity:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::with_capacity_in(10, &b);\n /// s.push('a');\n /// s.push('b');\n ///\n /// // s now has a length of 2 and a capacity of 10\n /// assert_eq!(2, s.len());\n /// assert_eq!(10, s.capacity());\n ///\n /// // Since we already have an extra 8 capacity, calling this...\n /// s.reserve(8);\n ///\n /// // ... doesn't actually increase.\n /// assert_eq!(10, s.capacity());\n /// ```\n #[inline]\n pub fn reserve(&mut self, additional: usize) {\n self.vec.reserve(additional)\n }\n\n /// Ensures that this `String`'s capacity is `additional` bytes\n /// larger than its length.\n ///\n /// Consider using the [`reserve`] method unless you absolutely know\n /// better than the allocator.\n ///\n /// [`reserve`]: #method.reserve\n ///\n /// # Panics\n ///\n /// Panics if the new capacity overflows `usize`.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::new_in(&b);\n ///\n /// s.reserve_exact(10);\n ///\n /// assert!(s.capacity() >= 10);\n /// ```\n ///\n /// This may not actually increase the capacity:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::with_capacity_in(10, &b);\n /// s.push('a');\n /// s.push('b');\n ///\n /// // s now has a length of 2 and a capacity of 10\n /// assert_eq!(2, s.len());\n /// assert_eq!(10, s.capacity());\n ///\n /// // Since we already have an extra 8 capacity, calling this...\n /// s.reserve_exact(8);\n ///\n /// // ... doesn't actually increase.\n /// assert_eq!(10, s.capacity());\n /// ```\n #[inline]\n pub fn reserve_exact(&mut self, additional: usize) {\n self.vec.reserve_exact(additional)\n }\n\n /// Shrinks the capacity of this `String` to match its length.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"foo\", &b);\n ///\n /// s.reserve(100);\n /// assert!(s.capacity() >= 100);\n ///\n /// s.shrink_to_fit();\n /// assert_eq!(3, s.capacity());\n /// ```\n #[inline]\n pub fn shrink_to_fit(&mut self) {\n self.vec.shrink_to_fit()\n }\n\n /// Appends the given [`char`] to the end of this `String`.\n ///\n /// [`char`]: https://doc.rust-lang.org/std/primitive.char.html\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"abc\", &b);\n ///\n /// s.push('1');\n /// s.push('2');\n /// s.push('3');\n ///\n /// assert_eq!(\"abc123\", s);\n /// ```\n #[inline]\n pub fn push(&mut self, ch: char) {\n match ch.len_utf8() {\n 1 => self.vec.push(ch as u8),\n _ => self\n .vec\n .extend_from_slice(ch.encode_utf8(&mut [0; 4]).as_bytes()),\n }\n }\n\n /// Returns a byte slice of this `String`'s contents.\n ///\n /// The inverse of this method is [`from_utf8`].\n ///\n /// [`from_utf8`]: #method.from_utf8\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let s = String::from_str_in(\"hello\", &b);\n ///\n /// assert_eq!(&[104, 101, 108, 108, 111], s.as_bytes());\n /// ```\n #[inline]\n pub fn as_bytes(&self) -> &[u8] {\n &self.vec\n }\n\n /// Shortens this `String` to the specified length.\n ///\n /// If `new_len` is greater than the string's current length, this has no\n /// effect.\n ///\n /// Note that this method has no effect on the allocated capacity\n /// of the string.\n ///\n /// # Panics\n ///\n /// Panics if `new_len` does not lie on a [`char`] boundary.\n ///\n /// [`char`]: https://doc.rust-lang.org/std/primitive.char.html\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"hello\", &b);\n ///\n /// s.truncate(2);\n ///\n /// assert_eq!(\"he\", s);\n /// ```\n #[inline]\n pub fn truncate(&mut self, new_len: usize) {\n if new_len <= self.len() {\n assert!(self.is_char_boundary(new_len));\n self.vec.truncate(new_len)\n }\n }\n\n /// Removes the last character from the string buffer and returns it.\n ///\n /// Returns [`None`] if this `String` is empty.\n ///\n /// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"foo\", &b);\n ///\n /// assert_eq!(s.pop(), Some('o'));\n /// assert_eq!(s.pop(), Some('o'));\n /// assert_eq!(s.pop(), Some('f'));\n ///\n /// assert_eq!(s.pop(), None);\n /// ```\n #[inline]\n pub fn pop(&mut self) -> Option {\n let ch = self.chars().rev().next()?;\n let newlen = self.len() - ch.len_utf8();\n unsafe {\n self.vec.set_len(newlen);\n }\n Some(ch)\n }\n\n /// Removes a [`char`] from this `String` at a byte position and returns it.\n ///\n /// This is an `O(n)` operation, as it requires copying every element in the\n /// buffer.\n ///\n /// # Panics\n ///\n /// Panics if `idx` is larger than or equal to the `String`'s length,\n /// or if it does not lie on a [`char`] boundary.\n ///\n /// [`char`]: https://doc.rust-lang.org/std/primitive.char.html\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"foo\", &b);\n ///\n /// assert_eq!(s.remove(0), 'f');\n /// assert_eq!(s.remove(1), 'o');\n /// assert_eq!(s.remove(0), 'o');\n /// ```\n #[inline]\n pub fn remove(&mut self, idx: usize) -> char {\n let ch = match self[idx..].chars().next() {\n Some(ch) => ch,\n None => panic!(\"cannot remove a char from the end of a string\"),\n };\n\n let next = idx + ch.len_utf8();\n let len = self.len();\n unsafe {\n ptr::copy(\n self.vec.as_ptr().add(next),\n self.vec.as_mut_ptr().add(idx),\n len - next,\n );\n self.vec.set_len(len - (next - idx));\n }\n ch\n }\n\n /// Retains only the characters specified by the predicate.\n ///\n /// In other words, remove all characters `c` such that `f(c)` returns `false`.\n /// This method operates in place and preserves the order of the retained\n /// characters.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"f_o_ob_ar\", &b);\n ///\n /// s.retain(|c| c != '_');\n ///\n /// assert_eq!(s, \"foobar\");\n /// ```\n #[inline]\n pub fn retain(&mut self, mut f: F)\n where\n F: FnMut(char) -> bool,\n {\n struct SetLenOnDrop<'a, 'bump> {\n s: &'a mut String<'bump>,\n idx: usize,\n del_bytes: usize,\n }\n\n impl<'a, 'bump> Drop for SetLenOnDrop<'a, 'bump> {\n fn drop(&mut self) {\n let new_len = self.idx - self.del_bytes;\n debug_assert!(new_len <= self.s.len());\n unsafe { self.s.vec.set_len(new_len) };\n }\n }\n\n let len = self.len();\n let mut guard = SetLenOnDrop {\n s: self,\n idx: 0,\n del_bytes: 0,\n };\n\n while guard.idx < len {\n let ch =\n // SAFETY: `guard.idx` is positive-or-zero and less that len so the `get_unchecked`\n // is in bound. `self` is valid UTF-8 like string and the returned slice starts at\n // a unicode code point so the `Chars` always return one character.\n unsafe { guard.s.get_unchecked(guard.idx..len).chars().next().unwrap_unchecked() };\n let ch_len = ch.len_utf8();\n\n if !f(ch) {\n guard.del_bytes += ch_len;\n } else if guard.del_bytes > 0 {\n // SAFETY: `guard.idx` is in bound and `guard.del_bytes` represent the number of\n // bytes that are erased from the string so the resulting `guard.idx -\n // guard.del_bytes` always represent a valid unicode code point.\n //\n // `guard.del_bytes` >= `ch.len_utf8()`, so taking a slice with `ch.len_utf8()` len\n // is safe.\n ch.encode_utf8(unsafe {\n core::slice::from_raw_parts_mut(\n guard.s.as_mut_ptr().add(guard.idx - guard.del_bytes),\n ch.len_utf8(),\n )\n });\n }\n\n // Point idx to the next char\n guard.idx += ch_len;\n }\n\n drop(guard);\n }\n\n /// Inserts a character into this `String` at a byte position.\n ///\n /// This is an `O(n)` operation as it requires copying every element in the\n /// buffer.\n ///\n /// # Panics\n ///\n /// Panics if `idx` is larger than the `String`'s length, or if it does not\n /// lie on a [`char`] boundary.\n ///\n /// [`char`]: https://doc.rust-lang.org/std/primitive.char.html\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::with_capacity_in(3, &b);\n ///\n /// s.insert(0, 'f');\n /// s.insert(1, 'o');\n /// s.insert(2, 'o');\n ///\n /// assert_eq!(\"foo\", s);\n /// ```\n #[inline]\n pub fn insert(&mut self, idx: usize, ch: char) {\n assert!(self.is_char_boundary(idx));\n let mut bits = [0; 4];\n let bits = ch.encode_utf8(&mut bits).as_bytes();\n\n unsafe {\n self.insert_bytes(idx, bits);\n }\n }\n\n unsafe fn insert_bytes(&mut self, idx: usize, bytes: &[u8]) {\n let len = self.len();\n let amt = bytes.len();\n self.vec.reserve(amt);\n\n ptr::copy(\n self.vec.as_ptr().add(idx),\n self.vec.as_mut_ptr().add(idx + amt),\n len - idx,\n );\n ptr::copy(bytes.as_ptr(), self.vec.as_mut_ptr().add(idx), amt);\n self.vec.set_len(len + amt);\n }\n\n /// Inserts a string slice into this `String` at a byte position.\n ///\n /// This is an `O(n)` operation as it requires copying every element in the\n /// buffer.\n ///\n /// # Panics\n ///\n /// Panics if `idx` is larger than the `String`'s length, or if it does not\n /// lie on a [`char`] boundary.\n ///\n /// [`char`]: https://doc.rust-lang.org/std/primitive.char.html\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"bar\", &b);\n ///\n /// s.insert_str(0, \"foo\");\n ///\n /// assert_eq!(\"foobar\", s);\n /// ```\n #[inline]\n pub fn insert_str(&mut self, idx: usize, string: &str) {\n assert!(self.is_char_boundary(idx));\n\n unsafe {\n self.insert_bytes(idx, string.as_bytes());\n }\n }\n\n /// Returns a mutable reference to the contents of this `String`.\n ///\n /// # Safety\n ///\n /// This function is unsafe because the returned `&mut Vec` allows writing\n /// bytes which are not valid UTF-8. If this constraint is violated, using\n /// the original `String` after dropping the `&mut Vec` may violate memory\n /// safety, as it is assumed that `String`s are valid UTF-8.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"hello\", &b);\n ///\n /// unsafe {\n /// let vec = s.as_mut_vec();\n /// assert_eq!(vec, &[104, 101, 108, 108, 111]);\n ///\n /// vec.reverse();\n /// }\n /// assert_eq!(s, \"olleh\");\n /// ```\n #[inline]\n pub unsafe fn as_mut_vec(&mut self) -> &mut Vec<'bump, u8> {\n &mut self.vec\n }\n\n /// Returns the length of this `String`, in bytes.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let a = String::from_str_in(\"foo\", &b);\n ///\n /// assert_eq!(a.len(), 3);\n /// ```\n #[inline]\n pub fn len(&self) -> usize {\n self.vec.len()\n }\n\n /// Returns `true` if this `String` has a length of zero.\n ///\n /// Returns `false` otherwise.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut v = String::new_in(&b);\n /// assert!(v.is_empty());\n ///\n /// v.push('a');\n /// assert!(!v.is_empty());\n /// ```\n #[inline]\n pub fn is_empty(&self) -> bool {\n self.len() == 0\n }\n\n /// Splits the string into two at the given index.\n ///\n /// Returns a newly allocated `String`. `self` contains bytes `[0, at)`, and\n /// the returned `String` contains bytes `[at, len)`. `at` must be on the\n /// boundary of a UTF-8 code point.\n ///\n /// Note that the capacity of `self` does not change.\n ///\n /// # Panics\n ///\n /// Panics if `at` is not on a UTF-8 code point boundary, or if it is beyond the last\n /// code point of the string.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut hello = String::from_str_in(\"Hello, World!\", &b);\n /// let world = hello.split_off(7);\n /// assert_eq!(hello, \"Hello, \");\n /// assert_eq!(world, \"World!\");\n /// ```\n #[inline]\n pub fn split_off(&mut self, at: usize) -> String<'bump> {\n assert!(self.is_char_boundary(at));\n let other = self.vec.split_off(at);\n unsafe { String::from_utf8_unchecked(other) }\n }\n\n /// Truncates this `String`, removing all contents.\n ///\n /// While this means the `String` will have a length of zero, it does not\n /// touch its capacity.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"foo\", &b);\n ///\n /// s.clear();\n ///\n /// assert!(s.is_empty());\n /// assert_eq!(0, s.len());\n /// assert_eq!(3, s.capacity());\n /// ```\n #[inline]\n pub fn clear(&mut self) {\n self.vec.clear()\n }\n\n /// Creates a draining iterator that removes the specified range in the `String`\n /// and yields the removed `chars`.\n ///\n /// Note: The element range is removed even if the iterator is not\n /// consumed until the end.\n ///\n /// # Panics\n ///\n /// Panics if the starting point or end point do not lie on a [`char`]\n /// boundary, or if they're out of bounds.\n ///\n /// [`char`]: https://doc.rust-lang.org/std/primitive.char.html\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"α is alpha, β is beta\", &b);\n /// let beta_offset = s.find('β').unwrap_or(s.len());\n ///\n /// // Remove the range up until the β from the string\n /// let t = String::from_iter_in(s.drain(..beta_offset), &b);\n /// assert_eq!(t, \"α is alpha, \");\n /// assert_eq!(s, \"β is beta\");\n ///\n /// // A full range clears the string\n /// drop(s.drain(..));\n /// assert_eq!(s, \"\");\n /// ```\n pub fn drain<'a, R>(&'a mut self, range: R) -> Drain<'a, 'bump>\n where\n R: RangeBounds,\n {\n // Memory safety\n //\n // The String version of Drain does not have the memory safety issues\n // of the vector version. The data is just plain bytes.\n // Because the range removal happens in Drop, if the Drain iterator is leaked,\n // the removal will not happen.\n let len = self.len();\n let start = match range.start_bound() {\n Included(&n) => n,\n Excluded(&n) => n + 1,\n Unbounded => 0,\n };\n let end = match range.end_bound() {\n Included(&n) => n + 1,\n Excluded(&n) => n,\n Unbounded => len,\n };\n\n // Take out two simultaneous borrows. The &mut String won't be accessed\n // until iteration is over, in Drop.\n let self_ptr = self as *mut _;\n // slicing does the appropriate bounds checks\n let chars_iter = self[start..end].chars();\n\n Drain {\n start,\n end,\n iter: chars_iter,\n string: self_ptr,\n }\n }\n\n /// Removes the specified range in the string,\n /// and replaces it with the given string.\n /// The given string doesn't need to be the same length as the range.\n ///\n /// # Panics\n ///\n /// Panics if the starting point or end point do not lie on a [`char`]\n /// boundary, or if they're out of bounds.\n ///\n /// [`char`]: https://doc.rust-lang.org/std/primitive.char.html\n /// [`Vec::splice`]: ../vec/struct.Vec.html#method.splice\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// let mut s = String::from_str_in(\"α is alpha, β is beta\", &b);\n /// let beta_offset = s.find('β').unwrap_or(s.len());\n ///\n /// // Replace the range up until the β from the string\n /// s.replace_range(..beta_offset, \"Α is capital alpha; \");\n /// assert_eq!(s, \"Α is capital alpha; β is beta\");\n /// ```\n pub fn replace_range(&mut self, range: R, replace_with: &str)\n where\n R: RangeBounds,\n {\n // Memory safety\n //\n // Replace_range does not have the memory safety issues of a vector Splice.\n // of the vector version. The data is just plain bytes.\n\n match range.start_bound() {\n Included(&n) => assert!(self.is_char_boundary(n)),\n Excluded(&n) => assert!(self.is_char_boundary(n + 1)),\n Unbounded => {}\n };\n match range.end_bound() {\n Included(&n) => assert!(self.is_char_boundary(n + 1)),\n Excluded(&n) => assert!(self.is_char_boundary(n)),\n Unbounded => {}\n };\n\n unsafe { self.as_mut_vec() }.splice(range, replace_with.bytes());\n }\n}\n\nimpl<'bump> FromUtf8Error<'bump> {\n /// Returns a slice of bytes that were attempted to convert to a `String`.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// // some invalid bytes, in a vector\n /// let bytes = bumpalo::vec![in &b; 0, 159];\n ///\n /// let value = String::from_utf8(bytes);\n ///\n /// assert_eq!(&[0, 159], value.unwrap_err().as_bytes());\n /// ```\n pub fn as_bytes(&self) -> &[u8] {\n &self.bytes[..]\n }\n\n /// Returns the bytes that were attempted to convert to a `String`.\n ///\n /// This method is carefully constructed to avoid allocation. It will\n /// consume the error, moving out the bytes, so that a copy of the bytes\n /// does not need to be made.\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// // some invalid bytes, in a vector\n /// let bytes = bumpalo::vec![in &b; 0, 159];\n ///\n /// let value = String::from_utf8(bytes);\n ///\n /// assert_eq!(bumpalo::vec![in &b; 0, 159], value.unwrap_err().into_bytes());\n /// ```\n pub fn into_bytes(self) -> Vec<'bump, u8> {\n self.bytes\n }\n\n /// Fetch a `Utf8Error` to get more details about the conversion failure.\n ///\n /// The [`Utf8Error`] type provided by [`std::str`] represents an error that may\n /// occur when converting a slice of [`u8`]s to a [`&str`]. In this sense, it's\n /// an analogue to `FromUtf8Error`. See its documentation for more details\n /// on using it.\n ///\n /// [`Utf8Error`]: https://doc.rust-lang.org/std/str/struct.Utf8Error.html\n /// [`std::str`]: https://doc.rust-lang.org/std/str/index.html\n /// [`u8`]: https://doc.rust-lang.org/std/primitive.u8.html\n /// [`&str`]: https://doc.rust-lang.org/std/primitive.str.html\n ///\n /// # Examples\n ///\n /// Basic usage:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::String};\n ///\n /// let b = Bump::new();\n ///\n /// // some invalid bytes, in a vector\n /// let bytes = bumpalo::vec![in &b; 0, 159];\n ///\n /// let error = String::from_utf8(bytes).unwrap_err().utf8_error();\n ///\n /// // the first byte is invalid here\n /// assert_eq!(1, error.valid_up_to());\n /// ```\n pub fn utf8_error(&self) -> Utf8Error {\n self.error\n }\n}\n\nimpl<'bump> fmt::Display for FromUtf8Error<'bump> {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n fmt::Display::fmt(&self.error, f)\n }\n}\n\nimpl fmt::Display for FromUtf16Error {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n fmt::Display::fmt(\"invalid utf-16: lone surrogate found\", f)\n }\n}\n\nimpl<'bump> Clone for String<'bump> {\n fn clone(&self) -> Self {\n String {\n vec: self.vec.clone(),\n }\n }\n\n fn clone_from(&mut self, source: &Self) {\n self.vec.clone_from(&source.vec);\n }\n}\n\nimpl<'bump> Extend for String<'bump> {\n fn extend>(&mut self, iter: I) {\n let iterator = iter.into_iter();\n let (lower_bound, _) = iterator.size_hint();\n self.reserve(lower_bound);\n for ch in iterator {\n self.push(ch)\n }\n }\n}\n\nimpl<'a, 'bump> Extend<&'a char> for String<'bump> {\n fn extend>(&mut self, iter: I) {\n self.extend(iter.into_iter().cloned());\n }\n}\n\nimpl<'a, 'bump> Extend<&'a str> for String<'bump> {\n fn extend>(&mut self, iter: I) {\n for s in iter {\n self.push_str(s)\n }\n }\n}\n\nimpl<'bump> Extend> for String<'bump> {\n fn extend>>(&mut self, iter: I) {\n for s in iter {\n self.push_str(&s)\n }\n }\n}\n\nimpl<'bump> Extend for String<'bump> {\n fn extend>(&mut self, iter: I) {\n for s in iter {\n self.push_str(&s)\n }\n }\n}\n\nimpl<'a, 'bump> Extend> for String<'bump> {\n fn extend>>(&mut self, iter: I) {\n for s in iter {\n self.push_str(&s)\n }\n }\n}\n\nimpl<'bump> PartialEq for String<'bump> {\n #[inline]\n fn eq(&self, other: &String) -> bool {\n PartialEq::eq(&self[..], &other[..])\n }\n}\n\nmacro_rules! impl_eq {\n ($lhs:ty, $rhs: ty) => {\n impl<'a, 'bump> PartialEq<$rhs> for $lhs {\n #[inline]\n fn eq(&self, other: &$rhs) -> bool {\n PartialEq::eq(&self[..], &other[..])\n }\n }\n\n impl<'a, 'b, 'bump> PartialEq<$lhs> for $rhs {\n #[inline]\n fn eq(&self, other: &$lhs) -> bool {\n PartialEq::eq(&self[..], &other[..])\n }\n }\n };\n}\n\nimpl_eq! { String<'bump>, str }\nimpl_eq! { String<'bump>, &'a str }\nimpl_eq! { Cow<'a, str>, String<'bump> }\nimpl_eq! { core_alloc::string::String, String<'bump> }\n\nimpl<'bump> fmt::Display for String<'bump> {\n #[inline]\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n fmt::Display::fmt(&**self, f)\n }\n}\n\nimpl<'bump> fmt::Debug for String<'bump> {\n #[inline]\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n fmt::Debug::fmt(&**self, f)\n }\n}\n\nimpl<'bump> hash::Hash for String<'bump> {\n #[inline]\n fn hash(&self, hasher: &mut H) {\n (**self).hash(hasher)\n }\n}\n\n/// Implements the `+` operator for concatenating two strings.\n///\n/// This consumes the `String<'bump>` on the left-hand side and re-uses its buffer (growing it if\n/// necessary). This is done to avoid allocating a new `String<'bump>` and copying the entire contents on\n/// every operation, which would lead to `O(n^2)` running time when building an `n`-byte string by\n/// repeated concatenation.\n///\n/// The string on the right-hand side is only borrowed; its contents are copied into the returned\n/// `String<'bump>`.\n///\n/// # Examples\n///\n/// Concatenating two `String<'bump>`s takes the first by value and borrows the second:\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n///\n/// let bump = Bump::new();\n///\n/// let a = String::from_str_in(\"hello\", &bump);\n/// let b = String::from_str_in(\" world\", &bump);\n/// let c = a + &b;\n/// // `a` is moved and can no longer be used here.\n/// ```\n///\n/// If you want to keep using the first `String`, you can clone it and append to the clone instead:\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n///\n/// let bump = Bump::new();\n///\n/// let a = String::from_str_in(\"hello\", &bump);\n/// let b = String::from_str_in(\" world\", &bump);\n/// let c = a.clone() + &b;\n/// // `a` is still valid here.\n/// ```\n///\n/// Concatenating `&str` slices can be done by converting the first to a `String`:\n///\n/// ```\n/// use bumpalo::{Bump, collections::String};\n///\n/// let bump = Bump::new();\n///\n/// let a = \"hello\";\n/// let b = \" world\";\n/// let c = String::from_str_in(a, &bump) + b;\n/// ```\nimpl<'a, 'bump> Add<&'a str> for String<'bump> {\n type Output = String<'bump>;\n\n #[inline]\n fn add(mut self, other: &str) -> String<'bump> {\n self.push_str(other);\n self\n }\n}\n\n/// Implements the `+=` operator for appending to a `String<'bump>`.\n///\n/// This has the same behavior as the [`push_str`][String::push_str] method.\nimpl<'a, 'bump> AddAssign<&'a str> for String<'bump> {\n #[inline]\n fn add_assign(&mut self, other: &str) {\n self.push_str(other);\n }\n}\n\nimpl<'bump> ops::Index> for String<'bump> {\n type Output = str;\n\n #[inline]\n fn index(&self, index: ops::Range) -> &str {\n &self[..][index]\n }\n}\nimpl<'bump> ops::Index> for String<'bump> {\n type Output = str;\n\n #[inline]\n fn index(&self, index: ops::RangeTo) -> &str {\n &self[..][index]\n }\n}\nimpl<'bump> ops::Index> for String<'bump> {\n type Output = str;\n\n #[inline]\n fn index(&self, index: ops::RangeFrom) -> &str {\n &self[..][index]\n }\n}\nimpl<'bump> ops::Index for String<'bump> {\n type Output = str;\n\n #[inline]\n fn index(&self, _index: ops::RangeFull) -> &str {\n unsafe { str::from_utf8_unchecked(&self.vec) }\n }\n}\nimpl<'bump> ops::Index> for String<'bump> {\n type Output = str;\n\n #[inline]\n fn index(&self, index: ops::RangeInclusive) -> &str {\n Index::index(&**self, index)\n }\n}\nimpl<'bump> ops::Index> for String<'bump> {\n type Output = str;\n\n #[inline]\n fn index(&self, index: ops::RangeToInclusive) -> &str {\n Index::index(&**self, index)\n }\n}\n\nimpl<'bump> ops::IndexMut> for String<'bump> {\n #[inline]\n fn index_mut(&mut self, index: ops::Range) -> &mut str {\n &mut self[..][index]\n }\n}\nimpl<'bump> ops::IndexMut> for String<'bump> {\n #[inline]\n fn index_mut(&mut self, index: ops::RangeTo) -> &mut str {\n &mut self[..][index]\n }\n}\nimpl<'bump> ops::IndexMut> for String<'bump> {\n #[inline]\n fn index_mut(&mut self, index: ops::RangeFrom) -> &mut str {\n &mut self[..][index]\n }\n}\nimpl<'bump> ops::IndexMut for String<'bump> {\n #[inline]\n fn index_mut(&mut self, _index: ops::RangeFull) -> &mut str {\n unsafe { str::from_utf8_unchecked_mut(&mut *self.vec) }\n }\n}\nimpl<'bump> ops::IndexMut> for String<'bump> {\n #[inline]\n fn index_mut(&mut self, index: ops::RangeInclusive) -> &mut str {\n IndexMut::index_mut(&mut **self, index)\n }\n}\nimpl<'bump> ops::IndexMut> for String<'bump> {\n #[inline]\n fn index_mut(&mut self, index: ops::RangeToInclusive) -> &mut str {\n IndexMut::index_mut(&mut **self, index)\n }\n}\n\nimpl<'bump> ops::Deref for String<'bump> {\n type Target = str;\n\n #[inline]\n fn deref(&self) -> &str {\n unsafe { str::from_utf8_unchecked(&self.vec) }\n }\n}\n\nimpl<'bump> ops::DerefMut for String<'bump> {\n #[inline]\n fn deref_mut(&mut self) -> &mut str {\n unsafe { str::from_utf8_unchecked_mut(&mut *self.vec) }\n }\n}\n\nimpl<'bump> AsRef for String<'bump> {\n #[inline]\n fn as_ref(&self) -> &str {\n self\n }\n}\n\nimpl<'bump> AsRef<[u8]> for String<'bump> {\n #[inline]\n fn as_ref(&self) -> &[u8] {\n self.as_bytes()\n }\n}\n\nimpl<'bump> fmt::Write for String<'bump> {\n #[inline]\n fn write_str(&mut self, s: &str) -> fmt::Result {\n self.push_str(s);\n Ok(())\n }\n\n #[inline]\n fn write_char(&mut self, c: char) -> fmt::Result {\n self.push(c);\n Ok(())\n }\n}\n\nimpl<'bump> Borrow for String<'bump> {\n #[inline]\n fn borrow(&self) -> &str {\n &self[..]\n }\n}\n\nimpl<'bump> BorrowMut for String<'bump> {\n #[inline]\n fn borrow_mut(&mut self) -> &mut str {\n &mut self[..]\n }\n}\n\n/// A draining iterator for `String`.\n///\n/// This struct is created by the [`String::drain`] method. See its\n/// documentation for more information.\npub struct Drain<'a, 'bump> {\n /// Will be used as &'a mut String in the destructor\n string: *mut String<'bump>,\n /// Start of part to remove\n start: usize,\n /// End of part to remove\n end: usize,\n /// Current remaining range to remove\n iter: Chars<'a>,\n}\n\nimpl<'a, 'bump> fmt::Debug for Drain<'a, 'bump> {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n f.pad(\"Drain { .. }\")\n }\n}\n\nunsafe impl<'a, 'bump> Sync for Drain<'a, 'bump> {}\nunsafe impl<'a, 'bump> Send for Drain<'a, 'bump> {}\n\nimpl<'a, 'bump> Drop for Drain<'a, 'bump> {\n fn drop(&mut self) {\n unsafe {\n // Use Vec::drain. \"Reaffirm\" the bounds checks to avoid\n // panic code being inserted again.\n let self_vec = (*self.string).as_mut_vec();\n if self.start <= self.end && self.end <= self_vec.len() {\n self_vec.drain(self.start..self.end);\n }\n }\n }\n}\n\n// TODO: implement `AsRef` and `as_str`\n\nimpl<'a, 'bump> Iterator for Drain<'a, 'bump> {\n type Item = char;\n\n #[inline]\n fn next(&mut self) -> Option {\n self.iter.next()\n }\n\n fn size_hint(&self) -> (usize, Option) {\n self.iter.size_hint()\n }\n}\n\nimpl<'a, 'bump> DoubleEndedIterator for Drain<'a, 'bump> {\n #[inline]\n fn next_back(&mut self) -> Option {\n self.iter.next_back()\n }\n}\n\nimpl<'a, 'bump> FusedIterator for Drain<'a, 'bump> {}\n\n#[cfg(feature = \"serde\")]\nmod serialize {\n use super::*;\n\n use serde::{Serialize, Serializer};\n\n impl<'bump> Serialize for String<'bump> {\n fn serialize(&self, serializer: S) -> Result\n where\n S: Serializer,\n {\n serializer.serialize_str(&self)\n }\n }\n}\n" }, { "package": "bumpalo@3.20.3", "sourcePath": "src/collections/vec.rs", "sha256": "ebb91c051e92c00c192c7b25eea9a5961ff9895a4fe195b29a2d71d0422b4a8c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n//! A contiguous growable array type with heap-allocated contents, written\n//! [`Vec<'bump, T>`].\n//!\n//! Vectors have `O(1)` indexing, amortized `O(1)` push (to the end) and\n//! `O(1)` pop (from the end).\n//!\n//! This module is a fork of the [`std::vec`] module, that uses a bump allocator.\n//!\n//! [`std::vec`]: https://doc.rust-lang.org/std/vec/index.html\n//!\n//! # Examples\n//!\n//! You can explicitly create a [`Vec<'bump, T>`] with [`new_in`]:\n//!\n//! ```\n//! use bumpalo::{Bump, collections::Vec};\n//!\n//! let b = Bump::new();\n//! let v: Vec = Vec::new_in(&b);\n//! ```\n//!\n//! ... or by using the [`vec!`] macro:\n//!\n//! ```\n//! use bumpalo::{Bump, collections::Vec};\n//!\n//! let b = Bump::new();\n//!\n//! let v: Vec = bumpalo::vec![in &b];\n//!\n//! let v = bumpalo::vec![in &b; 1, 2, 3, 4, 5];\n//!\n//! let v = bumpalo::vec![in &b; 0; 10]; // ten zeroes\n//! ```\n//!\n//! You can [`push`] values onto the end of a vector (which will grow the vector\n//! as needed):\n//!\n//! ```\n//! use bumpalo::{Bump, collections::Vec};\n//!\n//! let b = Bump::new();\n//!\n//! let mut v = bumpalo::vec![in &b; 1, 2];\n//!\n//! v.push(3);\n//! ```\n//!\n//! Popping values works in much the same way:\n//!\n//! ```\n//! use bumpalo::{Bump, collections::Vec};\n//!\n//! let b = Bump::new();\n//!\n//! let mut v = bumpalo::vec![in &b; 1, 2];\n//!\n//! assert_eq!(v.pop(), Some(2));\n//! ```\n//!\n//! Vectors also support indexing (through the [`Index`] and [`IndexMut`] traits):\n//!\n//! ```\n//! use bumpalo::{Bump, collections::Vec};\n//!\n//! let b = Bump::new();\n//!\n//! let mut v = bumpalo::vec![in &b; 1, 2, 3];\n//! assert_eq!(v[2], 3);\n//! v[1] += 5;\n//! assert_eq!(v, [1, 7, 3]);\n//! ```\n//!\n//! [`Vec<'bump, T>`]: struct.Vec.html\n//! [`new_in`]: struct.Vec.html#method.new_in\n//! [`push`]: struct.Vec.html#method.push\n//! [`Index`]: https://doc.rust-lang.org/std/ops/trait.Index.html\n//! [`IndexMut`]: https://doc.rust-lang.org/std/ops/trait.IndexMut.html\n//! [`vec!`]: ../../macro.vec.html\n\nuse super::raw_vec::RawVec;\nuse crate::collections::CollectionAllocErr;\nuse crate::Bump;\nuse core::borrow::{Borrow, BorrowMut};\nuse core::cmp::Ordering;\nuse core::fmt;\nuse core::hash::{self, Hash};\nuse core::iter::FusedIterator;\nuse core::marker::PhantomData;\nuse core::mem;\nuse core::ops;\nuse core::ops::Bound::{Excluded, Included, Unbounded};\nuse core::ops::{Index, IndexMut, RangeBounds};\nuse core::ptr;\nuse core::ptr::NonNull;\nuse core::slice;\n#[cfg(feature = \"std\")]\nuse std::io;\n\nunsafe fn arith_offset(p: *const T, offset: isize) -> *const T {\n p.offset(offset)\n}\n\nfn partition_dedup_by(s: &mut [T], mut same_bucket: F) -> (&mut [T], &mut [T])\nwhere\n F: FnMut(&mut T, &mut T) -> bool,\n{\n // Although we have a mutable reference to `s`, we cannot make\n // *arbitrary* changes. The `same_bucket` calls could panic, so we\n // must ensure that the slice is in a valid state at all times.\n //\n // The way that we handle this is by using swaps; we iterate\n // over all the elements, swapping as we go so that at the end\n // the elements we wish to keep are in the front, and those we\n // wish to reject are at the back. We can then split the slice.\n // This operation is still O(n).\n //\n // Example: We start in this state, where `r` represents \"next\n // read\" and `w` represents \"next_write`.\n //\n // r\n // +---+---+---+---+---+---+\n // | 0 | 1 | 1 | 2 | 3 | 3 |\n // +---+---+---+---+---+---+\n // w\n //\n // Comparing s[r] against s[w-1], this is not a duplicate, so\n // we swap s[r] and s[w] (no effect as r==w) and then increment both\n // r and w, leaving us with:\n //\n // r\n // +---+---+---+---+---+---+\n // | 0 | 1 | 1 | 2 | 3 | 3 |\n // +---+---+---+---+---+---+\n // w\n //\n // Comparing s[r] against s[w-1], this value is a duplicate,\n // so we increment `r` but leave everything else unchanged:\n //\n // r\n // +---+---+---+---+---+---+\n // | 0 | 1 | 1 | 2 | 3 | 3 |\n // +---+---+---+---+---+---+\n // w\n //\n // Comparing s[r] against s[w-1], this is not a duplicate,\n // so swap s[r] and s[w] and advance r and w:\n //\n // r\n // +---+---+---+---+---+---+\n // | 0 | 1 | 2 | 1 | 3 | 3 |\n // +---+---+---+---+---+---+\n // w\n //\n // Not a duplicate, repeat:\n //\n // r\n // +---+---+---+---+---+---+\n // | 0 | 1 | 2 | 3 | 1 | 3 |\n // +---+---+---+---+---+---+\n // w\n //\n // Duplicate, advance r. End of slice. Split at w.\n\n let len = s.len();\n if len <= 1 {\n return (s, &mut []);\n }\n\n let ptr = s.as_mut_ptr();\n let mut next_read: usize = 1;\n let mut next_write: usize = 1;\n\n unsafe {\n // Avoid bounds checks by using raw pointers.\n while next_read < len {\n let ptr_read = ptr.add(next_read);\n let prev_ptr_write = ptr.add(next_write - 1);\n if !same_bucket(&mut *ptr_read, &mut *prev_ptr_write) {\n if next_read != next_write {\n let ptr_write = prev_ptr_write.offset(1);\n mem::swap(&mut *ptr_read, &mut *ptr_write);\n }\n next_write += 1;\n }\n next_read += 1;\n }\n }\n\n s.split_at_mut(next_write)\n}\n\nunsafe fn offset_from(p: *const T, origin: *const T) -> isize\nwhere\n T: Sized,\n{\n let pointee_size = mem::size_of::();\n assert!(0 < pointee_size && pointee_size <= isize::max_value() as usize);\n\n // This is the same sequence that Clang emits for pointer subtraction.\n // It can be neither `nsw` nor `nuw` because the input is treated as\n // unsigned but then the output is treated as signed, so neither works.\n let d = isize::wrapping_sub(p as _, origin as _);\n d / (pointee_size as isize)\n}\n\n/// Creates a [`Vec`] containing the arguments.\n///\n/// `vec!` allows `Vec`s to be defined with the same syntax as array expressions.\n/// There are two forms of this macro:\n///\n/// - Create a [`Vec`] containing a given list of elements:\n///\n/// ```\n/// use bumpalo::Bump;\n///\n/// let b = Bump::new();\n/// let v = bumpalo::vec![in &b; 1, 2, 3];\n/// assert_eq!(v, [1, 2, 3]);\n/// ```\n///\n/// - Create a [`Vec`] from a given element and size:\n///\n/// ```\n/// use bumpalo::Bump;\n///\n/// let b = Bump::new();\n/// let v = bumpalo::vec![in &b; 1; 3];\n/// assert_eq!(v, [1, 1, 1]);\n/// ```\n///\n/// Note that unlike array expressions, this syntax supports all elements\n/// which implement [`Clone`] and the number of elements doesn't have to be\n/// a constant.\n///\n/// This will use `clone` to duplicate an expression, so one should be careful\n/// using this with types having a non-standard `Clone` implementation. For\n/// example, `bumpalo::vec![in ≎ Rc::new(1); 5]` will create a vector of five references\n/// to the same boxed integer value, not five references pointing to independently\n/// boxed integers.\n///\n/// [`Vec`]: collections/vec/struct.Vec.html\n/// [`Clone`]: https://doc.rust-lang.org/std/clone/trait.Clone.html\n#[macro_export]\nmacro_rules! vec {\n (in $bump:expr; $elem:expr; $n:expr) => {{\n let n = $n;\n let mut v = $crate::collections::Vec::with_capacity_in(n, $bump);\n if n > 0 {\n let elem = $elem;\n for _ in 0..n - 1 {\n v.push(elem.clone());\n }\n v.push(elem);\n }\n v\n }};\n (in $bump:expr) => { $crate::collections::Vec::new_in($bump) };\n (in $bump:expr; $($x:expr),*) => {{\n let mut v = $crate::collections::Vec::new_in($bump);\n $( v.push($x); )*\n v\n }};\n (in $bump:expr; $($x:expr,)*) => (bumpalo::vec![in $bump; $($x),*])\n}\n\n/// A contiguous growable array type, written `Vec<'bump, T>` but pronounced 'vector'.\n///\n/// # Examples\n///\n/// ```\n/// use bumpalo::{Bump, collections::Vec};\n///\n/// let b = Bump::new();\n///\n/// let mut vec = Vec::new_in(&b);\n/// vec.push(1);\n/// vec.push(2);\n///\n/// assert_eq!(vec.len(), 2);\n/// assert_eq!(vec[0], 1);\n///\n/// assert_eq!(vec.pop(), Some(2));\n/// assert_eq!(vec.len(), 1);\n///\n/// vec[0] = 7;\n/// assert_eq!(vec[0], 7);\n///\n/// vec.extend([1, 2, 3].iter().cloned());\n///\n/// for x in &vec {\n/// println!(\"{}\", x);\n/// }\n/// assert_eq!(vec, [7, 1, 2, 3]);\n/// ```\n///\n/// The [`vec!`] macro is provided to make initialization more convenient:\n///\n/// ```\n/// use bumpalo::{Bump, collections::Vec};\n///\n/// let b = Bump::new();\n///\n/// let mut vec = bumpalo::vec![in &b; 1, 2, 3];\n/// vec.push(4);\n/// assert_eq!(vec, [1, 2, 3, 4]);\n/// ```\n///\n/// It can also initialize each element of a `Vec<'bump, T>` with a given value.\n/// This may be more efficient than performing allocation and initialization\n/// in separate steps, especially when initializing a vector of zeros:\n///\n/// ```\n/// use bumpalo::{Bump, collections::Vec};\n///\n/// let b = Bump::new();\n///\n/// let vec = bumpalo::vec![in &b; 0; 5];\n/// assert_eq!(vec, [0, 0, 0, 0, 0]);\n///\n/// // The following is equivalent, but potentially slower:\n/// let mut vec1 = Vec::with_capacity_in(5, &b);\n/// vec1.resize(5, 0);\n/// ```\n///\n/// Use a `Vec<'bump, T>` as an efficient stack:\n///\n/// ```\n/// use bumpalo::{Bump, collections::Vec};\n///\n/// let b = Bump::new();\n///\n/// let mut stack = Vec::new_in(&b);\n///\n/// stack.push(1);\n/// stack.push(2);\n/// stack.push(3);\n///\n/// while let Some(top) = stack.pop() {\n/// // Prints 3, 2, 1\n/// println!(\"{}\", top);\n/// }\n/// ```\n///\n/// # Indexing\n///\n/// The `Vec` type allows to access values by index, because it implements the\n/// [`Index`] trait. An example will be more explicit:\n///\n/// ```\n/// use bumpalo::{Bump, collections::Vec};\n///\n/// let b = Bump::new();\n///\n/// let v = bumpalo::vec![in &b; 0, 2, 4, 6];\n/// println!(\"{}\", v[1]); // it will display '2'\n/// ```\n///\n/// However be careful: if you try to access an index which isn't in the `Vec`,\n/// your software will panic! You cannot do this:\n///\n/// ```should_panic\n/// use bumpalo::{Bump, collections::Vec};\n///\n/// let b = Bump::new();\n///\n/// let v = bumpalo::vec![in &b; 0, 2, 4, 6];\n/// println!(\"{}\", v[6]); // it will panic!\n/// ```\n///\n/// In conclusion: always check if the index you want to get really exists\n/// before doing it.\n///\n/// # Slicing\n///\n/// A `Vec` can be mutable. Slices, on the other hand, are read-only objects.\n/// To get a slice, use `&`. Example:\n///\n/// ```\n/// use bumpalo::{Bump, collections::Vec};\n///\n/// fn read_slice(slice: &[usize]) {\n/// // ...\n/// }\n///\n/// let b = Bump::new();\n///\n/// let v = bumpalo::vec![in &b; 0, 1];\n/// read_slice(&v);\n///\n/// // ... and that's all!\n/// // you can also do it like this:\n/// let x : &[usize] = &v;\n/// ```\n///\n/// In Rust, it's more common to pass slices as arguments rather than vectors\n/// when you just want to provide a read access. The same goes for [`String`] and\n/// [`&str`].\n///\n/// # Capacity and reallocation\n///\n/// The capacity of a vector is the amount of space allocated for any future\n/// elements that will be added onto the vector. This is not to be confused with\n/// the *length* of a vector, which specifies the number of actual elements\n/// within the vector. If a vector's length exceeds its capacity, its capacity\n/// will automatically be increased, but its elements will have to be\n/// reallocated.\n///\n/// For example, a vector with capacity 10 and length 0 would be an empty vector\n/// with space for 10 more elements. Pushing 10 or fewer elements onto the\n/// vector will not change its capacity or cause reallocation to occur. However,\n/// if the vector's length is increased to 11, it will have to reallocate, which\n/// can be slow. For this reason, it is recommended to use [`Vec::with_capacity_in`]\n/// whenever possible to specify how big the vector is expected to get.\n///\n/// # Guarantees\n///\n/// Due to its incredibly fundamental nature, `Vec` makes a lot of guarantees\n/// about its design. This ensures that it's as low-overhead as possible in\n/// the general case, and can be correctly manipulated in primitive ways\n/// by unsafe code. Note that these guarantees refer to an unqualified `Vec<'bump, T>`.\n/// If additional type parameters are added (e.g. to support custom allocators),\n/// overriding their defaults may change the behavior.\n///\n/// Most fundamentally, `Vec` is and always will be a (pointer, capacity, length)\n/// triplet. No more, no less. The order of these fields is completely\n/// unspecified, and you should use the appropriate methods to modify these.\n/// The pointer will never be null, so this type is null-pointer-optimized.\n///\n/// However, the pointer may not actually point to allocated memory. In particular,\n/// if you construct a `Vec` with capacity 0 via [`Vec::new_in`], [`bumpalo::vec![in bump]`][`vec!`],\n/// [`Vec::with_capacity_in(0)`][`Vec::with_capacity_in`], or by calling [`shrink_to_fit`]\n/// on an empty Vec, it will not allocate memory. Similarly, if you store zero-sized\n/// types inside a `Vec`, it will not allocate space for them. *Note that in this case\n/// the `Vec` may not report a [`capacity`] of 0*. `Vec` will allocate if and only\n/// if [`mem::size_of::`]\\() * capacity() > 0. In general, `Vec`'s allocation\n/// details are very subtle — if you intend to allocate memory using a `Vec`\n/// and use it for something else (either to pass to unsafe code, or to build your\n/// own memory-backed collection), be sure to deallocate this memory by using\n/// `from_raw_parts` to recover the `Vec` and then dropping it.\n///\n/// If a `Vec` *has* allocated memory, then the memory it points to is\n/// in the [`Bump`] arena used to construct it, and its\n/// pointer points to [`len`] initialized, contiguous elements in order (what\n/// you would see if you coerced it to a slice), followed by [`capacity`] -\n/// [`len`] logically uninitialized, contiguous elements.\n///\n/// `Vec` will never perform a \"small optimization\" where elements are actually\n/// stored on the stack for two reasons:\n///\n/// * It would make it more difficult for unsafe code to correctly manipulate\n/// a `Vec`. The contents of a `Vec` wouldn't have a stable address if it were\n/// only moved, and it would be more difficult to determine if a `Vec` had\n/// actually allocated memory.\n///\n/// * It would penalize the general case, incurring an additional branch\n/// on every access.\n///\n/// `Vec` will never automatically shrink itself, even if completely empty. This\n/// ensures no unnecessary allocations or deallocations occur. Emptying a `Vec`\n/// and then filling it back up to the same [`len`] should incur no calls to\n/// the allocator. If you wish to free up unused memory, use\n/// [`shrink_to_fit`][`shrink_to_fit`].\n///\n/// [`push`] and [`insert`] will never (re)allocate if the reported capacity is\n/// sufficient. [`push`] and [`insert`] *will* (re)allocate if\n/// [`len`] == [`capacity`]. That is, the reported capacity is completely\n/// accurate, and can be relied on. It can even be used to manually free the memory\n/// allocated by a `Vec` if desired. Bulk insertion methods *may* reallocate, even\n/// when not necessary.\n///\n/// `Vec` does not guarantee any particular growth strategy when reallocating\n/// when full, nor when [`reserve`] is called. The current strategy is basic\n/// and it may prove desirable to use a non-constant growth factor. Whatever\n/// strategy is used will of course guarantee `O(1)` amortized [`push`].\n///\n/// `bumpalo::vec![in bump; x; n]`, `bumpalo::vec![in bump; a, b, c, d]`, and\n/// [`Vec::with_capacity_in(n)`][`Vec::with_capacity_in`], will all produce a\n/// `Vec` with exactly the requested capacity. If [`len`] == [`capacity`], (as\n/// is the case for the [`vec!`] macro), then a `Vec<'bump, T>` can be converted\n/// to and from a [`Box<[T]>`][owned slice] without reallocating or moving the\n/// elements.\n///\n/// `Vec` will not specifically overwrite any data that is removed from it,\n/// but also won't specifically preserve it. Its uninitialized memory is\n/// scratch space that it may use however it wants. It will generally just do\n/// whatever is most efficient or otherwise easy to implement. Do not rely on\n/// removed data to be erased for security purposes. Even if you drop a `Vec`, its\n/// buffer may simply be reused by another `Vec`. Even if you zero a `Vec`'s memory\n/// first, that may not actually happen because the optimizer does not consider\n/// this a side-effect that must be preserved. There is one case which we will\n/// not break, however: using `unsafe` code to write to the excess capacity,\n/// and then increasing the length to match, is always valid.\n///\n/// `Vec` does not currently guarantee the order in which elements are dropped.\n/// The order has changed in the past and may change again.\n///\n/// [`vec!`]: ../../macro.vec.html\n/// [`Index`]: https://doc.rust-lang.org/std/ops/trait.Index.html\n/// [`String`]: ../string/struct.String.html\n/// [`&str`]: https://doc.rust-lang.org/std/primitive.str.html\n/// [`Vec::with_capacity_in`]: struct.Vec.html#method.with_capacity_in\n/// [`Vec::new_in`]: struct.Vec.html#method.new_in\n/// [`shrink_to_fit`]: struct.Vec.html#method.shrink_to_fit\n/// [`capacity`]: struct.Vec.html#method.capacity\n/// [`mem::size_of::`]: https://doc.rust-lang.org/std/mem/fn.size_of.html\n/// [`len`]: struct.Vec.html#method.len\n/// [`push`]: struct.Vec.html#method.push\n/// [`insert`]: struct.Vec.html#method.insert\n/// [`reserve`]: struct.Vec.html#method.reserve\n/// [owned slice]: https://doc.rust-lang.org/std/boxed/struct.Box.html\npub struct Vec<'bump, T: 'bump> {\n buf: RawVec<'bump, T>,\n len: usize,\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// Inherent methods\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<'bump, T: 'bump> Vec<'bump, T> {\n /// Constructs a new, empty `Vec<'bump, T>`.\n ///\n /// The vector will not allocate until elements are pushed onto it.\n ///\n /// # Examples\n ///\n /// ```\n /// # #![allow(unused_mut)]\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n /// let mut vec: Vec = Vec::new_in(&b);\n /// ```\n #[inline]\n pub fn new_in(bump: &'bump Bump) -> Vec<'bump, T> {\n Vec {\n buf: RawVec::new_in(bump),\n len: 0,\n }\n }\n\n /// Constructs a new, empty `Vec<'bump, T>` with the specified capacity.\n ///\n /// The vector will be able to hold exactly `capacity` elements without\n /// reallocating. If `capacity` is 0, the vector will not allocate.\n ///\n /// It is important to note that although the returned vector has the\n /// *capacity* specified, the vector will have a zero *length*. For an\n /// explanation of the difference between length and capacity, see\n /// *[Capacity and reallocation]*.\n ///\n /// [Capacity and reallocation]: #capacity-and-reallocation\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = Vec::with_capacity_in(10, &b);\n ///\n /// // The vector contains no items, even though it has capacity for more\n /// assert_eq!(vec.len(), 0);\n ///\n /// // These are all done without reallocating...\n /// for i in 0..10 {\n /// vec.push(i);\n /// }\n ///\n /// // ...but this may make the vector reallocate\n /// vec.push(11);\n /// ```\n #[inline]\n pub fn with_capacity_in(capacity: usize, bump: &'bump Bump) -> Vec<'bump, T> {\n Vec {\n buf: RawVec::with_capacity_in(capacity, bump),\n len: 0,\n }\n }\n\n /// Construct a new `Vec` from the given iterator's items.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n /// use std::iter;\n ///\n /// let b = Bump::new();\n /// let v = Vec::from_iter_in(iter::repeat(7).take(3), &b);\n /// assert_eq!(v, [7, 7, 7]);\n /// ```\n pub fn from_iter_in>(iter: I, bump: &'bump Bump) -> Vec<'bump, T> {\n let mut v = Vec::new_in(bump);\n v.extend(iter);\n v\n }\n\n /// Creates a `Vec<'bump, T>` directly from the raw components of another vector.\n ///\n /// # Safety\n ///\n /// This is highly unsafe, due to the number of invariants that aren't\n /// checked:\n ///\n /// * `ptr` needs to have been previously allocated via [`String`]/`Vec<'bump, T>`\n /// (at least, it's highly likely to be incorrect if it wasn't).\n /// * `ptr`'s `T` needs to have the same size and alignment as it was allocated with.\n /// * `length` needs to be less than or equal to `capacity`.\n /// * `capacity` needs to be the capacity that the pointer was allocated with.\n ///\n /// Violating these may cause problems like corrupting the allocator's\n /// internal data structures. For example it is **not** safe\n /// to build a `Vec` from a pointer to a C `char` array and a `size_t`.\n ///\n /// The ownership of `ptr` is effectively transferred to the\n /// `Vec<'bump, T>` which may then deallocate, reallocate or change the\n /// contents of memory pointed to by the pointer at will. Ensure\n /// that nothing else uses the pointer after calling this\n /// function.\n ///\n /// [`String`]: ../string/struct.String.html\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// use std::ptr;\n /// use std::mem;\n ///\n /// let b = Bump::new();\n ///\n /// let mut v = bumpalo::vec![in &b; 1, 2, 3];\n ///\n /// // Pull out the various important pieces of information about `v`\n /// let p = v.as_mut_ptr();\n /// let len = v.len();\n /// let cap = v.capacity();\n ///\n /// unsafe {\n /// // Cast `v` into the void: no destructor run, so we are in\n /// // complete control of the allocation to which `p` points.\n /// mem::forget(v);\n ///\n /// // Overwrite memory with 4, 5, 6\n /// for i in 0..len as isize {\n /// ptr::write(p.offset(i), 4 + i);\n /// }\n ///\n /// // Put everything back together into a Vec\n /// let rebuilt = Vec::from_raw_parts_in(p, len, cap, &b);\n /// assert_eq!(rebuilt, [4, 5, 6]);\n /// }\n /// ```\n pub unsafe fn from_raw_parts_in(\n ptr: *mut T,\n length: usize,\n capacity: usize,\n bump: &'bump Bump,\n ) -> Vec<'bump, T> {\n Vec {\n buf: RawVec::from_raw_parts_in(ptr, capacity, bump),\n len: length,\n }\n }\n\n /// Returns a shared reference to the allocator backing this `Vec`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// // uses the same allocator as the provided `Vec`\n /// fn add_strings<'bump>(vec: &mut Vec<'bump, &'bump str>) {\n /// for string in [\"foo\", \"bar\", \"baz\"] {\n /// vec.push(vec.bump().alloc_str(string));\n /// }\n /// }\n /// ```\n #[inline]\n #[must_use]\n pub fn bump(&self) -> &'bump Bump {\n self.buf.bump()\n }\n\n /// Returns the number of elements the vector can hold without\n /// reallocating.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n /// let vec: Vec = Vec::with_capacity_in(10, &b);\n /// assert_eq!(vec.capacity(), 10);\n /// ```\n #[inline]\n pub fn capacity(&self) -> usize {\n self.buf.cap()\n }\n\n /// Reserves capacity for at least `additional` more elements to be inserted\n /// in the given `Vec<'bump, T>`. The collection may reserve more space to avoid\n /// frequent reallocations. After calling `reserve`, capacity will be\n /// greater than or equal to `self.len() + additional`. Does nothing if\n /// capacity is already sufficient.\n ///\n /// # Panics\n ///\n /// Panics if the new capacity overflows `usize`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n /// let mut vec = bumpalo::vec![in &b; 1];\n /// vec.reserve(10);\n /// assert!(vec.capacity() >= 11);\n /// ```\n pub fn reserve(&mut self, additional: usize) {\n self.buf.reserve(self.len, additional);\n }\n\n /// Reserves the minimum capacity for exactly `additional` more elements to\n /// be inserted in the given `Vec<'bump, T>`. After calling `reserve_exact`,\n /// capacity will be greater than or equal to `self.len() + additional`.\n /// Does nothing if the capacity is already sufficient.\n ///\n /// Note that the allocator may give the collection more space than it\n /// requests. Therefore capacity can not be relied upon to be precisely\n /// minimal. Prefer `reserve` if future insertions are expected.\n ///\n /// # Panics\n ///\n /// Panics if the new capacity overflows `usize`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n /// let mut vec = bumpalo::vec![in &b; 1];\n /// vec.reserve_exact(10);\n /// assert!(vec.capacity() >= 11);\n /// ```\n pub fn reserve_exact(&mut self, additional: usize) {\n self.buf.reserve_exact(self.len, additional);\n }\n\n /// Attempts to reserve capacity for at least `additional` more elements to be inserted\n /// in the given `Vec<'bump, T>`. The collection may reserve more space to avoid\n /// frequent reallocations. After calling `try_reserve`, capacity will be\n /// greater than or equal to `self.len() + additional`. Does nothing if\n /// capacity is already sufficient.\n ///\n /// # Panics\n ///\n /// Panics if the new capacity overflows `usize`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n /// let mut vec = bumpalo::vec![in &b; 1];\n /// vec.try_reserve(10).unwrap();\n /// assert!(vec.capacity() >= 11);\n /// ```\n pub fn try_reserve(&mut self, additional: usize) -> Result<(), CollectionAllocErr> {\n self.buf.try_reserve(self.len, additional)\n }\n\n /// Attempts to reserve the minimum capacity for exactly `additional` more elements to\n /// be inserted in the given `Vec<'bump, T>`. After calling `try_reserve_exact`,\n /// capacity will be greater than or equal to `self.len() + additional`.\n /// Does nothing if the capacity is already sufficient.\n ///\n /// Note that the allocator may give the collection more space than it\n /// requests. Therefore capacity can not be relied upon to be precisely\n /// minimal. Prefer `try_reserve` if future insertions are expected.\n ///\n /// # Panics\n ///\n /// Panics if the new capacity overflows `usize`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n /// let mut vec = bumpalo::vec![in &b; 1];\n /// vec.try_reserve_exact(10).unwrap();\n /// assert!(vec.capacity() >= 11);\n /// ```\n pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), CollectionAllocErr> {\n self.buf.try_reserve_exact(self.len, additional)\n }\n\n /// Shrinks the capacity of the vector as much as possible.\n ///\n /// It will drop down as close as possible to the length but the allocator\n /// may still inform the vector that there is space for a few more elements.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = Vec::with_capacity_in(10, &b);\n /// vec.extend([1, 2, 3].iter().cloned());\n /// assert_eq!(vec.capacity(), 10);\n /// vec.shrink_to_fit();\n /// assert!(vec.capacity() >= 3);\n /// ```\n pub fn shrink_to_fit(&mut self) {\n if self.capacity() != self.len {\n self.buf.shrink_to_fit(self.len);\n }\n }\n\n /// Converts the vector into `&'bump [T]`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n /// let v = bumpalo::vec![in &b; 1, 2, 3];\n ///\n /// let slice = v.into_bump_slice();\n /// assert_eq!(slice, [1, 2, 3]);\n /// ```\n pub fn into_bump_slice(self) -> &'bump [T] {\n unsafe {\n let ptr = self.as_ptr();\n let len = self.len();\n mem::forget(self);\n slice::from_raw_parts(ptr, len)\n }\n }\n\n /// Converts the vector into `&'bump mut [T]`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n /// let v = bumpalo::vec![in &b; 1, 2, 3];\n ///\n /// let mut slice = v.into_bump_slice_mut();\n ///\n /// slice[0] = 3;\n /// slice[2] = 1;\n ///\n /// assert_eq!(slice, [3, 2, 1]);\n /// ```\n pub fn into_bump_slice_mut(mut self) -> &'bump mut [T] {\n let ptr = self.as_mut_ptr();\n let len = self.len();\n mem::forget(self);\n\n unsafe { slice::from_raw_parts_mut(ptr, len) }\n }\n\n /// Shortens the vector, keeping the first `len` elements and dropping\n /// the rest.\n ///\n /// If `len` is greater than the vector's current length, this has no\n /// effect.\n ///\n /// The [`drain`] method can emulate `truncate`, but causes the excess\n /// elements to be returned instead of dropped.\n ///\n /// Note that this method has no effect on the allocated capacity\n /// of the vector.\n ///\n /// # Examples\n ///\n /// Truncating a five element vector to two elements:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 3, 4, 5];\n /// vec.truncate(2);\n /// assert_eq!(vec, [1, 2]);\n /// ```\n ///\n /// No truncation occurs when `len` is greater than the vector's current\n /// length:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 3];\n /// vec.truncate(8);\n /// assert_eq!(vec, [1, 2, 3]);\n /// ```\n ///\n /// Truncating when `len == 0` is equivalent to calling the [`clear`]\n /// method.\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 3];\n /// vec.truncate(0);\n /// assert_eq!(vec, []);\n /// ```\n ///\n /// [`clear`]: #method.clear\n /// [`drain`]: #method.drain\n pub fn truncate(&mut self, len: usize) {\n let current_len = self.len;\n unsafe {\n let mut ptr = self.as_mut_ptr().add(self.len);\n // Set the final length at the end, keeping in mind that\n // dropping an element might panic. Works around a missed\n // optimization, as seen in the following issue:\n // https://github.com/rust-lang/rust/issues/51802\n let mut local_len = SetLenOnDrop::new(&mut self.len);\n\n // drop any extra elements\n for _ in len..current_len {\n local_len.decrement_len(1);\n ptr = ptr.offset(-1);\n ptr::drop_in_place(ptr);\n }\n }\n }\n\n /// Extracts a slice containing the entire vector.\n ///\n /// Equivalent to `&s[..]`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n /// use std::io::{self, Write};\n ///\n /// let b = Bump::new();\n ///\n /// let buffer = bumpalo::vec![in &b; 1, 2, 3, 5, 8];\n /// io::sink().write(buffer.as_slice()).unwrap();\n /// ```\n #[inline]\n pub fn as_slice(&self) -> &[T] {\n self\n }\n\n /// Extracts a mutable slice of the entire vector.\n ///\n /// Equivalent to `&mut s[..]`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n /// use std::io::{self, Read};\n ///\n /// let b = Bump::new();\n /// let mut buffer = bumpalo::vec![in &b; 0; 3];\n /// io::repeat(0b101).read_exact(buffer.as_mut_slice()).unwrap();\n /// ```\n #[inline]\n pub fn as_mut_slice(&mut self) -> &mut [T] {\n self\n }\n\n /// Returns a raw pointer to the vector's buffer, or a dangling raw pointer\n /// valid for zero sized reads if the vector didn't allocate.\n ///\n /// The caller must ensure that the vector outlives the pointer this\n /// function returns, or else it will end up pointing to garbage.\n /// Modifying the vector may cause its buffer to be reallocated,\n /// which would also make any pointers to it invalid.\n ///\n /// The caller must also ensure that the memory the pointer (non-transitively) points to\n /// is never written to (except inside an `UnsafeCell`) using this pointer or any pointer\n /// derived from it. If you need to mutate the contents of the slice, use [`as_mut_ptr`].\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let bump = Bump::new();\n ///\n /// let x = bumpalo::vec![in ≎ 1, 2, 4];\n /// let x_ptr = x.as_ptr();\n ///\n /// unsafe {\n /// for i in 0..x.len() {\n /// assert_eq!(*x_ptr.add(i), 1 << i);\n /// }\n /// }\n /// ```\n ///\n /// [`as_mut_ptr`]: Vec::as_mut_ptr\n #[inline]\n pub fn as_ptr(&self) -> *const T {\n // We shadow the slice method of the same name to avoid going through\n // `deref`, which creates an intermediate reference.\n let ptr = self.buf.ptr();\n unsafe {\n if ptr.is_null() {\n core::hint::unreachable_unchecked();\n }\n }\n ptr\n }\n\n /// Returns an unsafe mutable pointer to the vector's buffer, or a dangling\n /// raw pointer valid for zero sized reads if the vector didn't allocate.\n ///\n /// The caller must ensure that the vector outlives the pointer this\n /// function returns, or else it will end up pointing to garbage.\n /// Modifying the vector may cause its buffer to be reallocated,\n /// which would also make any pointers to it invalid.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let bump = Bump::new();\n ///\n /// // Allocate vector big enough for 4 elements.\n /// let size = 4;\n /// let mut x: Vec = Vec::with_capacity_in(size, &bump);\n /// let x_ptr = x.as_mut_ptr();\n ///\n /// // Initialize elements via raw pointer writes, then set length.\n /// unsafe {\n /// for i in 0..size {\n /// x_ptr.add(i).write(i as i32);\n /// }\n /// x.set_len(size);\n /// }\n /// assert_eq!(&*x, &[0, 1, 2, 3]);\n /// ```\n #[inline]\n pub fn as_mut_ptr(&mut self) -> *mut T {\n // We shadow the slice method of the same name to avoid going through\n // `deref_mut`, which creates an intermediate reference.\n let ptr = self.buf.ptr();\n unsafe {\n if ptr.is_null() {\n core::hint::unreachable_unchecked();\n }\n }\n ptr\n }\n\n /// Sets the length of a vector.\n ///\n /// This will explicitly set the size of the vector, without actually\n /// modifying its buffers, so it is up to the caller to ensure that the\n /// vector is actually the specified size.\n ///\n /// # Safety\n ///\n /// - `new_len` must be less than or equal to [`capacity()`].\n /// - The elements at `old_len..new_len` must be initialized.\n ///\n /// [`capacity()`]: struct.Vec.html#method.capacity\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// use std::ptr;\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 'r', 'u', 's', 't'];\n ///\n /// unsafe {\n /// ptr::drop_in_place(&mut vec[3]);\n /// vec.set_len(3);\n /// }\n /// assert_eq!(vec, ['r', 'u', 's']);\n /// ```\n ///\n /// In this example, there is a memory leak since the memory locations\n /// owned by the inner vectors were not freed prior to the `set_len` call:\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b;\n /// bumpalo::vec![in &b; 1, 0, 0],\n /// bumpalo::vec![in &b; 0, 1, 0],\n /// bumpalo::vec![in &b; 0, 0, 1]];\n /// unsafe {\n /// vec.set_len(0);\n /// }\n /// ```\n ///\n /// In this example, the vector gets expanded from zero to four items\n /// but we directly initialize uninitialized memory:\n ///\n // TODO: rely upon `spare_capacity_mut`\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let len = 4;\n /// let b = Bump::new();\n ///\n /// let mut vec: Vec = Vec::with_capacity_in(len, &b);\n ///\n /// for i in 0..len {\n /// // SAFETY: we initialize memory via `pointer::write`\n /// unsafe { vec.as_mut_ptr().add(i).write(b'a') }\n /// }\n ///\n /// unsafe {\n /// vec.set_len(len);\n /// }\n ///\n /// assert_eq!(b\"aaaa\", &*vec);\n /// ```\n #[inline]\n pub unsafe fn set_len(&mut self, new_len: usize) {\n self.len = new_len;\n }\n\n /// Removes an element from the vector and returns it.\n ///\n /// The removed element is replaced by the last element of the vector.\n ///\n /// This does not preserve ordering, but is O(1).\n ///\n /// # Panics\n ///\n /// Panics if `index` is out of bounds.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut v = bumpalo::vec![in &b; \"foo\", \"bar\", \"baz\", \"qux\"];\n ///\n /// assert_eq!(v.swap_remove(1), \"bar\");\n /// assert_eq!(v, [\"foo\", \"qux\", \"baz\"]);\n ///\n /// assert_eq!(v.swap_remove(0), \"foo\");\n /// assert_eq!(v, [\"baz\", \"qux\"]);\n /// ```\n #[inline]\n pub fn swap_remove(&mut self, index: usize) -> T {\n unsafe {\n // We replace self[index] with the last element. Note that if the\n // bounds check on hole succeeds there must be a last element (which\n // can be self[index] itself).\n let hole: *mut T = &mut self[index];\n let last = ptr::read(self.get_unchecked(self.len - 1));\n self.len -= 1;\n ptr::replace(hole, last)\n }\n }\n\n /// Inserts an element at position `index` within the vector, shifting all\n /// elements after it to the right.\n ///\n /// # Panics\n ///\n /// Panics if `index > len`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 3];\n /// vec.insert(1, 4);\n /// assert_eq!(vec, [1, 4, 2, 3]);\n /// vec.insert(4, 5);\n /// assert_eq!(vec, [1, 4, 2, 3, 5]);\n /// ```\n pub fn insert(&mut self, index: usize, element: T) {\n let len = self.len();\n assert!(index <= len);\n\n // space for the new element\n if len == self.buf.cap() {\n self.reserve(1);\n }\n\n unsafe {\n // infallible\n // The spot to put the new value\n {\n let p = self.as_mut_ptr().add(index);\n // Shift everything over to make space. (Duplicating the\n // `index`th element into two consecutive places.)\n ptr::copy(p, p.offset(1), len - index);\n // Write it in, overwriting the first copy of the `index`th\n // element.\n ptr::write(p, element);\n }\n self.set_len(len + 1);\n }\n }\n\n /// Removes and returns the element at position `index` within the vector,\n /// shifting all elements after it to the left.\n ///\n /// # Panics\n ///\n /// Panics if `index` is out of bounds.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut v = bumpalo::vec![in &b; 1, 2, 3];\n /// assert_eq!(v.remove(1), 2);\n /// assert_eq!(v, [1, 3]);\n /// ```\n pub fn remove(&mut self, index: usize) -> T {\n let len = self.len();\n assert!(index < len);\n unsafe {\n // infallible\n let ret;\n {\n // the place we are taking from.\n let ptr = self.as_mut_ptr().add(index);\n // copy it out, unsafely having a copy of the value on\n // the stack and in the vector at the same time.\n ret = ptr::read(ptr);\n\n // Shift everything down to fill in that spot.\n ptr::copy(ptr.offset(1), ptr, len - index - 1);\n }\n self.set_len(len - 1);\n ret\n }\n }\n\n /// Retains only the elements specified by the predicate.\n ///\n /// In other words, remove all elements `e` such that `f(&e)` returns `false`.\n /// This method operates in place and preserves the order of the retained\n /// elements.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 3, 4];\n /// vec.retain(|x: &i32| *x % 2 == 0);\n /// assert_eq!(vec, [2, 4]);\n /// ```\n pub fn retain(&mut self, mut f: F)\n where\n F: FnMut(&T) -> bool,\n {\n self.drain_filter(|x| !f(x));\n }\n\n /// Retains only the elements specified by the predicate.\n ///\n /// In other words, remove all elements `e` such that `f(&mut e)` returns `false`.\n /// This method operates in place and preserves the order of the retained\n /// elements.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 3, 4];\n /// vec.retain_mut(|x: &mut i32| *x % 2 == 0);\n /// assert_eq!(vec, [2, 4]);\n /// ```\n pub fn retain_mut(&mut self, mut f: F)\n where\n F: FnMut(&mut T) -> bool,\n {\n self.drain_filter(|x| !f(x));\n }\n\n /// Creates an iterator that removes the elements in the vector\n /// for which the predicate returns `true` and yields the removed items.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::Bump;\n /// use bumpalo::collections::{CollectIn, Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut numbers = bumpalo::vec![in &b; 1, 2, 3, 4, 5];\n ///\n /// let evens: Vec<_> = numbers.drain_filter(|x| *x % 2 == 0).collect_in(&b);\n ///\n /// assert_eq!(numbers, &[1, 3, 5]);\n /// assert_eq!(evens, &[2, 4]);\n /// ```\n pub fn drain_filter<'a, F>(&'a mut self, filter: F) -> DrainFilter<'a, 'bump, T, F>\n where\n F: FnMut(&mut T) -> bool,\n {\n let old_len = self.len();\n\n // Guard against us getting leaked (leak amplification)\n unsafe {\n self.set_len(0);\n }\n\n DrainFilter {\n vec: self,\n idx: 0,\n write: 0,\n old_len,\n pred: filter,\n }\n }\n\n /// Removes all but the first of consecutive elements in the vector that resolve to the same\n /// key.\n ///\n /// If the vector is sorted, this removes all duplicates.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 10, 20, 21, 30, 20];\n ///\n /// vec.dedup_by_key(|i| *i / 10);\n ///\n /// assert_eq!(vec, [10, 20, 30, 20]);\n /// ```\n #[inline]\n pub fn dedup_by_key(&mut self, mut key: F)\n where\n F: FnMut(&mut T) -> K,\n K: PartialEq,\n {\n self.dedup_by(|a, b| key(a) == key(b))\n }\n\n /// Removes all but the first of consecutive elements in the vector satisfying a given equality\n /// relation.\n ///\n /// The `same_bucket` function is passed references to two elements from the vector and\n /// must determine if the elements compare equal. The elements are passed in opposite order\n /// from their order in the slice, so if `same_bucket(a, b)` returns `true`, `a` is removed.\n ///\n /// If the vector is sorted, this removes all duplicates.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; \"foo\", \"bar\", \"Bar\", \"baz\", \"bar\"];\n ///\n /// vec.dedup_by(|a, b| a.eq_ignore_ascii_case(b));\n ///\n /// assert_eq!(vec, [\"foo\", \"bar\", \"baz\", \"bar\"]);\n /// ```\n pub fn dedup_by(&mut self, same_bucket: F)\n where\n F: FnMut(&mut T, &mut T) -> bool,\n {\n let len = {\n let (dedup, _) = partition_dedup_by(self.as_mut_slice(), same_bucket);\n dedup.len()\n };\n self.truncate(len);\n }\n\n // Proven specification with verus, converted to comments.\n /// # Preconditions\n ///\n /// - old(self).len() < old(self).capacity(),\n ///\n /// # Postconditions\n ///\n /// - self.get_unchecked(old(self).len()) == value,\n /// - self.len() == old(self).len() + 1,\n /// - self.capacity() == old(self).capacity(),\n /// - forall|i: usize| implies(\n /// i < old(self).len(),\n /// self.get_unchecked(i) == old(self).get_unchecked(i)\n /// )\n #[allow(clippy::inline_always)]\n #[inline(always)]\n unsafe fn push_unchecked(&mut self, value: T) {\n debug_assert!(self.len() < self.capacity());\n\n // Divergence from verified impl:\n // Verified implementation has special handling for ZSTs\n // as ZSTs do not play nicely with separation logic.\n ptr::write(self.buf.ptr().add(self.len), value);\n\n self.len = self.len + 1;\n }\n\n /// Appends an element to the back of a vector.\n ///\n /// # Panics\n ///\n /// Panics if the number of elements in the vector overflows a `usize`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2];\n /// vec.push(3);\n /// assert_eq!(vec, [1, 2, 3]);\n /// ```\n #[inline]\n pub fn push(&mut self, value: T) {\n // This will panic or abort if we would allocate > isize::MAX bytes\n // or if the length increment would overflow for zero-sized types.\n if self.len == self.buf.cap() {\n self.reserve(1);\n }\n unsafe {\n self.push_unchecked(value);\n }\n }\n\n /// Removes the last element from a vector and returns it, or [`None`] if it\n /// is empty.\n ///\n /// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 3];\n /// assert_eq!(vec.pop(), Some(3));\n /// assert_eq!(vec, [1, 2]);\n /// ```\n #[inline]\n pub fn pop(&mut self) -> Option {\n if self.len == 0 {\n None\n } else {\n unsafe {\n self.len -= 1;\n Some(ptr::read(self.as_ptr().add(self.len())))\n }\n }\n }\n\n /// Removes and returns the last element from a vector if the predicate\n /// returns `true`, or [`None`] if the predicate returns false or the vector\n /// is empty (the predicate will not be called in that case).\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 3, 4];\n /// let pred = |x: &mut i32| *x % 2 == 0;\n ///\n /// assert_eq!(vec.pop_if(pred), Some(4));\n /// assert_eq!(vec, [1, 2, 3]);\n /// assert_eq!(vec.pop_if(pred), None);\n /// ```\n #[inline]\n pub fn pop_if(&mut self, predicate: impl FnOnce(&mut T) -> bool) -> Option {\n let last = self.last_mut()?;\n if predicate(last) {\n self.pop()\n } else {\n None\n }\n }\n\n /// Moves all the elements of `other` into `Self`, leaving `other` empty.\n ///\n /// # Panics\n ///\n /// Panics if the number of elements in the vector overflows a `usize`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 3];\n /// let mut vec2 = bumpalo::vec![in &b; 4, 5, 6];\n /// vec.append(&mut vec2);\n /// assert_eq!(vec, [1, 2, 3, 4, 5, 6]);\n /// assert_eq!(vec2, []);\n /// ```\n #[inline]\n pub fn append(&mut self, other: &mut Self) {\n unsafe {\n self.append_elements(other.as_slice() as _);\n other.set_len(0);\n }\n }\n\n /// Appends elements to `Self` from other buffer.\n #[inline]\n unsafe fn append_elements(&mut self, other: *const [T]) {\n let count = (&(*other)).len();\n self.reserve(count);\n let len = self.len();\n ptr::copy_nonoverlapping(other as *const T, self.as_mut_ptr().add(len), count);\n self.len += count;\n }\n\n /// Creates a draining iterator that removes the specified range in the vector\n /// and yields the removed items.\n ///\n /// Note 1: The element range is removed even if the iterator is only\n /// partially consumed or not consumed at all.\n ///\n /// Note 2: It is unspecified how many elements are removed from the vector\n /// if the `Drain` value is leaked.\n ///\n /// # Panics\n ///\n /// Panics if the starting point is greater than the end point or if\n /// the end point is greater than the length of the vector.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::Bump;\n /// use bumpalo::collections::{CollectIn, Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut v = bumpalo::vec![in &b; 1, 2, 3];\n ///\n /// let u: Vec<_> = v.drain(1..).collect_in(&b);\n ///\n /// assert_eq!(v, &[1]);\n /// assert_eq!(u, &[2, 3]);\n ///\n /// // A full range clears the vector\n /// v.drain(..);\n /// assert_eq!(v, &[]);\n /// ```\n pub fn drain<'a, R>(&'a mut self, range: R) -> Drain<'a, 'bump, T>\n where\n R: RangeBounds,\n {\n // Memory safety\n //\n // When the Drain is first created, it shortens the length of\n // the source vector to make sure no uninitialized or moved-from elements\n // are accessible at all if the Drain's destructor never gets to run.\n //\n // Drain will ptr::read out the values to remove.\n // When finished, remaining tail of the vec is copied back to cover\n // the hole, and the vector length is restored to the new length.\n //\n let len = self.len();\n let start = match range.start_bound() {\n Included(&n) => n,\n Excluded(&n) => n + 1,\n Unbounded => 0,\n };\n let end = match range.end_bound() {\n Included(&n) => n + 1,\n Excluded(&n) => n,\n Unbounded => len,\n };\n assert!(start <= end);\n assert!(end <= len);\n\n unsafe {\n // set self.vec length's to start, to be safe in case Drain is leaked\n self.set_len(start);\n // Use the borrow in the IterMut to indicate borrowing behavior of the\n // whole Drain iterator (like &mut T).\n let range_slice = slice::from_raw_parts_mut(self.as_mut_ptr().add(start), end - start);\n Drain {\n tail_start: end,\n tail_len: len - end,\n iter: range_slice.iter(),\n vec: NonNull::from(self),\n }\n }\n }\n\n /// Clears the vector, removing all values.\n ///\n /// Note that this method has no effect on the allocated capacity\n /// of the vector.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut v = bumpalo::vec![in &b; 1, 2, 3];\n ///\n /// v.clear();\n ///\n /// assert!(v.is_empty());\n /// ```\n #[inline]\n pub fn clear(&mut self) {\n self.truncate(0)\n }\n\n /// Returns the number of elements in the vector, also referred to\n /// as its 'length'.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let a = bumpalo::vec![in &b; 1, 2, 3];\n /// assert_eq!(a.len(), 3);\n /// ```\n #[inline]\n pub fn len(&self) -> usize {\n self.len\n }\n\n /// Returns `true` if the vector contains no elements.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut v = Vec::new_in(&b);\n /// assert!(v.is_empty());\n ///\n /// v.push(1);\n /// assert!(!v.is_empty());\n /// ```\n pub fn is_empty(&self) -> bool {\n self.len() == 0\n }\n\n /// Splits the collection into two at the given index.\n ///\n /// Returns a newly allocated vector. `self` contains elements `[0, at)`,\n /// and the returned vector contains elements `[at, len)`.\n ///\n /// Note that the capacity of `self` does not change.\n ///\n /// # Panics\n ///\n /// Panics if `at > len`.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 3];\n /// let vec2 = vec.split_off(1);\n /// assert_eq!(vec, [1]);\n /// assert_eq!(vec2, [2, 3]);\n /// ```\n #[inline]\n pub fn split_off(&mut self, at: usize) -> Self {\n assert!(at <= self.len(), \"`at` out of bounds\");\n\n let other_len = self.len - at;\n let mut other = Vec::with_capacity_in(other_len, self.buf.bump());\n\n // Unsafely `set_len` and copy items to `other`.\n unsafe {\n self.set_len(at);\n other.set_len(other_len);\n\n ptr::copy_nonoverlapping(self.as_ptr().add(at), other.as_mut_ptr(), other.len());\n }\n other\n }\n}\n\n#[cfg(feature = \"boxed\")]\nimpl<'bump, T> Vec<'bump, T> {\n /// Converts the vector into [`Box<[T]>`][owned slice].\n ///\n /// Note that this will drop any excess capacity.\n ///\n /// [owned slice]: ../../boxed/struct.Box.html\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec, vec};\n ///\n /// let b = Bump::new();\n ///\n /// let v = vec![in &b; 1, 2, 3];\n ///\n /// let slice = v.into_boxed_slice();\n /// ```\n pub fn into_boxed_slice(mut self) -> crate::boxed::Box<'bump, [T]> {\n use crate::boxed::Box;\n\n // Unlike `alloc::vec::Vec` shrinking here isn't necessary as `bumpalo::boxed::Box` doesn't own memory.\n unsafe {\n let slice = slice::from_raw_parts_mut(self.as_mut_ptr(), self.len);\n let output: Box<'bump, [T]> = Box::from_raw(slice);\n mem::forget(self);\n output\n }\n }\n}\n\nimpl<'bump, T: 'bump + Clone> Vec<'bump, T> {\n /// Resizes the `Vec` in-place so that `len` is equal to `new_len`.\n ///\n /// If `new_len` is greater than `len`, the `Vec` is extended by the\n /// difference, with each additional slot filled with `value`.\n /// If `new_len` is less than `len`, the `Vec` is simply truncated.\n ///\n /// This method requires [`Clone`] to be able clone the passed value. If\n /// you need more flexibility (or want to rely on [`Default`] instead of\n /// [`Clone`]), use [`resize_with`].\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; \"hello\"];\n /// vec.resize(3, \"world\");\n /// assert_eq!(vec, [\"hello\", \"world\", \"world\"]);\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 3, 4];\n /// vec.resize(2, 0);\n /// assert_eq!(vec, [1, 2]);\n /// ```\n ///\n /// [`Clone`]: https://doc.rust-lang.org/std/clone/trait.Clone.html\n /// [`Default`]: https://doc.rust-lang.org/std/default/trait.Default.html\n /// [`resize_with`]: #method.resize_with\n pub fn resize(&mut self, new_len: usize, value: T) {\n let len = self.len();\n\n if new_len > len {\n self.extend_with(new_len - len, ExtendElement(value))\n } else {\n self.truncate(new_len);\n }\n }\n\n // Proven specification with verus, converted to comments.\n /// # Preconditions\n ///\n /// - old(self).len() + slice.len() <= old(self).capacity(),\n ///\n /// # Postconditions\n ///\n /// - forall|i: usize| implies(\n /// i < old(self).len(),\n /// self.get_unchecked(i) == old(self).get_unchecked(i)\n /// ),\n /// - forall|i: usize| implies(\n /// i < slice.len(),\n /// self.get_unchecked((old(self).len() + i) as usize)\n /// == clone(slice.get_unchecked(i))\n /// ),\n /// - self.len() == old(self).len() + slice.len(),\n /// - self.capacity() == old(self).capacity(),\n #[inline]\n unsafe fn extend_from_slice_unchecked(&mut self, slice: &[T]) {\n // Guaranteed never to overflow for non ZSTs\n // size_of::() <= isize::MAX - (isize::MAX % align_of::()))\n // isize::MAX + isize::MAX < usize::MAX\n debug_assert!(\n core::mem::size_of::() == 0 || self.capacity() >= self.len() + slice.len()\n );\n debug_assert!(\n // is_zst::() ==> capacity >= slen + slice.len()\n core::mem::size_of::() != 0\n // Capacity is usize::MAX for ZSTs\n || self.len() <= usize::MAX - slice.len()\n );\n\n let mut pos = 0usize;\n\n loop\n /*\n invariants\n pos <= slice.len(),\n self.len() + (slice.len() - pos) <= old(self).capacity(),\n old(self).capacity() == self.capacity(),\n\n self.len() == old(self).len() + pos,\n\n forall|i: usize| implies(\n i < old(self).len(),\n self.get_unchecked(i) == old(self).get_unchecked(i)\n ),\n forall|i: usize| implies(\n i < pos,\n self.get_unchecked((old(self).len() + i) as usize)\n == clone(slice.get_unchecked(i))\n )\n */\n {\n if pos == slice.len() {\n /*\n pos = slice.len(),\n self.len() = old(self).len() + slice.len(),\n\n forall|i: usize| i < slice.len() implies {\n self.get_unchecked((old(self).len() + i) as usize)\n == clone(slice.get_unchecked(i))\n }\n by {\n i < pos\n }\n */\n return;\n }\n\n /*\n pos < slice.len(),\n self.len() < self.capacity()\n */\n\n let elem = slice.get_unchecked(pos);\n self.push_unchecked(elem.clone());\n\n /*\n ghost prev_pos = pos\n */\n\n pos = pos + 1;\n\n /*\n forall|i: usize| i < pos implies {\n self.get_unchecked((old(self).len() + i) as usize)\n == clone(slice.get_unchecked(i))\n }\n by {\n if i < pos - 1 {\n // By invariant\n }\n else {\n i == prev_pos\n }\n }\n */\n }\n }\n\n /// Clones and appends all elements in a slice to the `Vec`.\n ///\n /// Iterates over the slice `other`, clones each element, and then appends\n /// it to this `Vec`. The `other` vector is traversed in-order.\n ///\n /// Note that this function is same as [`extend`] except that it is\n /// specialized to work with slices instead. If and when Rust gets\n /// specialization this function will likely be deprecated (but still\n /// available).\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1];\n /// vec.extend_from_slice(&[2, 3, 4]);\n /// assert_eq!(vec, [1, 2, 3, 4]);\n /// ```\n ///\n /// [`extend`]: #method.extend\n pub fn extend_from_slice(&mut self, other: &[T]) {\n let capacity = self.capacity();\n\n /*\n Cannot underflow via invariant of the Vec (capacity >= length).\n\n This also holds for ZSTs, as capacity is usize::MAX\n */\n let remaining_cap = capacity - self.len;\n\n /*\n self.len() + other.len() <= self.capacity(),\n <==>\n other.len() <= self.capacity() - self.len()\n */\n\n if other.len() > remaining_cap {\n /*\n Divergence from verified impl:\n Verified implementation's reserve is not the same\n as bumpalo's. Verified implementation reserves with\n respect to capacity, not length. Thus this is equivalent\n to the verified implementation's:\n\n self.buf.reserve(other.len() - remaining_cap)\n\n */\n self.reserve(other.len());\n }\n\n /*\n self.capacity() >= self.len() + other.len()\n */\n\n unsafe {\n self.extend_from_slice_unchecked(other);\n }\n }\n}\n\nimpl<'bump, T: 'bump + Copy> Vec<'bump, T> {\n /// Helper method to copy all of the items in `other` and append them to the end of `self`.\n ///\n /// SAFETY:\n /// * The caller is responsible for:\n /// * calling [`reserve`](Self::reserve) beforehand to guarantee that there is enough\n /// capacity to store `other.len()` more items.\n /// * guaranteeing that `self` and `other` do not overlap.\n unsafe fn extend_from_slice_copy_unchecked(&mut self, other: &[T]) {\n let old_len = self.len();\n debug_assert!(old_len + other.len() <= self.capacity());\n\n // SAFETY:\n // * `src` is valid for reads of `other.len()` values by virtue of being a `&[T]`.\n // * `dst` is valid for writes of `other.len()` bytes because the caller of this\n // method is required to `reserve` capacity to store at least `other.len()` items\n // beforehand.\n // * Because `src` is a `&[T]` and dst is a `&[T]` within the `Vec`,\n // `copy_nonoverlapping`'s alignment requirements are met.\n // * Caller is required to guarantee that the source and destination ranges cannot overlap\n unsafe {\n let src = other.as_ptr();\n let dst = self.as_mut_ptr().add(old_len);\n ptr::copy_nonoverlapping(src, dst, other.len());\n self.set_len(old_len + other.len());\n }\n }\n\n /// Copies all elements in the slice `other` and appends them to the `Vec`.\n ///\n /// Note that this function is same as [`extend_from_slice`] except that it is optimized for\n /// slices of types that implement the `Copy` trait. If and when Rust gets specialization\n /// this function will likely be deprecated (but still available).\n ///\n /// To copy and append the data from multiple source slices at once, see\n /// [`extend_from_slices_copy`].\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1];\n /// vec.extend_from_slice_copy(&[2, 3, 4]);\n /// assert_eq!(vec, [1, 2, 3, 4]);\n /// ```\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 'H' as u8];\n /// vec.extend_from_slice_copy(\"ello, world!\".as_bytes());\n /// assert_eq!(vec, \"Hello, world!\".as_bytes());\n /// ```\n ///\n /// [`extend_from_slice`]: #method.extend_from_slice\n /// [`extend_from_slices`]: #method.extend_from_slices\n pub fn extend_from_slice_copy(&mut self, other: &[T]) {\n // Reserve space in the Vec for the values to be added\n self.reserve(other.len());\n\n // Copy values into the space that was just reserved\n // SAFETY:\n // * `self` has enough capacity to store `other.len()` more items as `self.reserve(other.len())`\n // above guarantees that.\n // * Source and destination data ranges cannot overlap as we just reserved the destination\n // range from the bump.\n unsafe {\n self.extend_from_slice_copy_unchecked(other);\n }\n }\n\n /// For each slice in `slices`, copies all elements in the slice and appends them to the `Vec`.\n ///\n /// This method is equivalent to calling [`extend_from_slice_copy`] in a loop, but is able\n /// to precompute the total amount of space to reserve in advance. This reduces the potential\n /// maximum number of reallocations needed from one-per-slice to just one.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1];\n /// vec.extend_from_slices_copy(&[&[2, 3], &[], &[4]]);\n /// assert_eq!(vec, [1, 2, 3, 4]);\n /// ```\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 'H' as u8];\n /// vec.extend_from_slices_copy(&[\"ello,\".as_bytes(), &[], \" world!\".as_bytes()]);\n /// assert_eq!(vec, \"Hello, world!\".as_bytes());\n /// ```\n ///\n /// [`extend_from_slice_copy`]: #method.extend_from_slice_copy\n pub fn extend_from_slices_copy(&mut self, slices: &[&[T]]) {\n // Reserve the total amount of capacity we'll need to safely append the aggregated contents\n // of each slice in `slices`.\n let capacity_to_reserve: usize = slices.iter().map(|slice| slice.len()).sum();\n self.reserve(capacity_to_reserve);\n\n // SAFETY:\n // * `dst` is valid for writes of `capacity_to_reserve` items as\n // `self.reserve(capacity_to_reserve)` above guarantees that.\n // * Source and destination ranges cannot overlap as we just reserved the destination\n // range from the bump.\n unsafe {\n // Copy the contents of each slice onto the end of `self`\n slices.iter().for_each(|slice| {\n self.extend_from_slice_copy_unchecked(slice);\n });\n }\n }\n}\n\n// This code generalises `extend_with_{element,default}`.\ntrait ExtendWith {\n fn next(&mut self) -> T;\n fn last(self) -> T;\n}\n\nstruct ExtendElement(T);\nimpl ExtendWith for ExtendElement {\n fn next(&mut self) -> T {\n self.0.clone()\n }\n fn last(self) -> T {\n self.0\n }\n}\n\nimpl<'bump, T: 'bump> Vec<'bump, T> {\n /// Extend the vector by `n` values, using the given generator.\n fn extend_with>(&mut self, n: usize, mut value: E) {\n self.reserve(n);\n\n unsafe {\n let mut ptr = self.as_mut_ptr().add(self.len());\n // Use SetLenOnDrop to work around bug where compiler\n // may not realize the store through `ptr` through self.set_len()\n // don't alias.\n let mut local_len = SetLenOnDrop::new(&mut self.len);\n\n // Write all elements except the last one\n for _ in 1..n {\n ptr::write(ptr, value.next());\n ptr = ptr.offset(1);\n // Increment the length in every step in case next() panics\n local_len.increment_len(1);\n }\n\n if n > 0 {\n // We can write the last element directly without cloning needlessly\n ptr::write(ptr, value.last());\n local_len.increment_len(1);\n }\n\n // len set by scope guard\n }\n }\n}\n\n// Set the length of the vec when the `SetLenOnDrop` value goes out of scope.\n//\n// The idea is: The length field in SetLenOnDrop is a local variable\n// that the optimizer will see does not alias with any stores through the Vec's data\n// pointer. This is a workaround for alias analysis issue #32155\nstruct SetLenOnDrop<'a> {\n len: &'a mut usize,\n local_len: usize,\n}\n\nimpl<'a> SetLenOnDrop<'a> {\n #[inline]\n fn new(len: &'a mut usize) -> Self {\n SetLenOnDrop {\n local_len: *len,\n len,\n }\n }\n\n #[inline]\n fn increment_len(&mut self, increment: usize) {\n self.local_len += increment;\n }\n\n #[inline]\n fn decrement_len(&mut self, decrement: usize) {\n self.local_len -= decrement;\n }\n}\n\nimpl<'a> Drop for SetLenOnDrop<'a> {\n #[inline]\n fn drop(&mut self) {\n *self.len = self.local_len;\n }\n}\n\nimpl<'bump, T: 'bump + PartialEq> Vec<'bump, T> {\n /// Removes consecutive repeated elements in the vector according to the\n /// [`PartialEq`] trait implementation.\n ///\n /// If the vector is sorted, this removes all duplicates.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut vec = bumpalo::vec![in &b; 1, 2, 2, 3, 2];\n ///\n /// vec.dedup();\n ///\n /// assert_eq!(vec, [1, 2, 3, 2]);\n /// ```\n #[inline]\n pub fn dedup(&mut self) {\n self.dedup_by(|a, b| a == b)\n }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// Common trait implementations for Vec\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<'bump, T: 'bump + Clone> Clone for Vec<'bump, T> {\n #[cfg(not(test))]\n fn clone(&self) -> Vec<'bump, T> {\n let mut v = Vec::with_capacity_in(self.len(), self.buf.bump());\n v.extend(self.iter().cloned());\n v\n }\n\n // HACK(japaric): with cfg(test) the inherent `[T]::to_vec` method, which is\n // required for this method definition, is not available. Instead use the\n // `slice::to_vec` function which is only available with cfg(test)\n // NB see the slice::hack module in slice.rs for more information\n #[cfg(test)]\n fn clone(&self) -> Vec<'bump, T> {\n let mut v = Vec::new_in(self.buf.bump());\n v.extend(self.iter().cloned());\n v\n }\n}\n\nimpl<'bump, T: 'bump + Hash> Hash for Vec<'bump, T> {\n #[inline]\n fn hash(&self, state: &mut H) {\n Hash::hash(&**self, state)\n }\n}\n\nimpl<'bump, T, I> Index for Vec<'bump, T>\nwhere\n I: ::core::slice::SliceIndex<[T]>,\n{\n type Output = I::Output;\n\n #[inline]\n fn index(&self, index: I) -> &Self::Output {\n Index::index(&**self, index)\n }\n}\n\nimpl<'bump, T, I> IndexMut for Vec<'bump, T>\nwhere\n I: ::core::slice::SliceIndex<[T]>,\n{\n #[inline]\n fn index_mut(&mut self, index: I) -> &mut Self::Output {\n IndexMut::index_mut(&mut **self, index)\n }\n}\n\nimpl<'bump, T: 'bump> ops::Deref for Vec<'bump, T> {\n type Target = [T];\n\n fn deref(&self) -> &[T] {\n unsafe {\n let p = self.buf.ptr();\n // assume(!p.is_null());\n slice::from_raw_parts(p, self.len)\n }\n }\n}\n\nimpl<'bump, T: 'bump> ops::DerefMut for Vec<'bump, T> {\n fn deref_mut(&mut self) -> &mut [T] {\n unsafe {\n let ptr = self.buf.ptr();\n // assume(!ptr.is_null());\n slice::from_raw_parts_mut(ptr, self.len)\n }\n }\n}\n\nimpl<'bump, T: 'bump> IntoIterator for Vec<'bump, T> {\n type Item = T;\n type IntoIter = IntoIter<'bump, T>;\n\n /// Creates a consuming iterator, that is, one that moves each value out of\n /// the vector (from start to end). The vector cannot be used after calling\n /// this.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let v = bumpalo::vec![in &b; \"a\".to_string(), \"b\".to_string()];\n /// for s in v.into_iter() {\n /// // s has type String, not &String\n /// println!(\"{}\", s);\n /// }\n /// ```\n #[inline]\n fn into_iter(mut self) -> IntoIter<'bump, T> {\n unsafe {\n let begin = self.as_mut_ptr();\n // assume(!begin.is_null());\n let end = if mem::size_of::() == 0 {\n arith_offset(begin as *const i8, self.len() as isize) as *const T\n } else {\n begin.add(self.len()) as *const T\n };\n mem::forget(self);\n IntoIter {\n phantom: PhantomData,\n ptr: begin,\n end,\n }\n }\n }\n}\n\nimpl<'a, 'bump, T> IntoIterator for &'a Vec<'bump, T> {\n type Item = &'a T;\n type IntoIter = slice::Iter<'a, T>;\n\n fn into_iter(self) -> slice::Iter<'a, T> {\n self.iter()\n }\n}\n\nimpl<'a, 'bump, T> IntoIterator for &'a mut Vec<'bump, T> {\n type Item = &'a mut T;\n type IntoIter = slice::IterMut<'a, T>;\n\n fn into_iter(self) -> slice::IterMut<'a, T> {\n self.iter_mut()\n }\n}\n\nimpl<'bump, T: 'bump> Extend for Vec<'bump, T> {\n #[inline]\n fn extend>(&mut self, iter: I) {\n let iter = iter.into_iter();\n self.reserve(iter.size_hint().0);\n\n for t in iter {\n self.push(t);\n }\n }\n}\n\nimpl<'bump, T: 'bump> Vec<'bump, T> {\n /// Creates a splicing iterator that replaces the specified range in the vector\n /// with the given `replace_with` iterator and yields the removed items.\n /// `replace_with` does not need to be the same length as `range`.\n ///\n /// Note 1: The element range is removed even if the iterator is not\n /// consumed until the end.\n ///\n /// Note 2: It is unspecified how many elements are removed from the vector,\n /// if the `Splice` value is leaked.\n ///\n /// Note 3: The input iterator `replace_with` is only consumed\n /// when the `Splice` value is dropped.\n ///\n /// Note 4: This is optimal if:\n ///\n /// * The tail (elements in the vector after `range`) is empty,\n /// * or `replace_with` yields fewer elements than `range`’s length\n /// * or the lower bound of its `size_hint()` is exact.\n ///\n /// Otherwise, a temporary vector is allocated and the tail is moved twice.\n ///\n /// # Panics\n ///\n /// Panics if the starting point is greater than the end point or if\n /// the end point is greater than the length of the vector.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let mut v = bumpalo::vec![in &b; 1, 2, 3];\n /// let new = [7, 8];\n /// let u: Vec<_> = Vec::from_iter_in(v.splice(..2, new.iter().cloned()), &b);\n /// assert_eq!(v, &[7, 8, 3]);\n /// assert_eq!(u, &[1, 2]);\n /// ```\n #[inline]\n pub fn splice<'a, R, I>(\n &'a mut self,\n range: R,\n replace_with: I,\n ) -> Splice<'a, 'bump, I::IntoIter>\n where\n R: RangeBounds,\n I: IntoIterator,\n {\n Splice {\n drain: self.drain(range),\n replace_with: replace_with.into_iter(),\n }\n }\n}\n\n/// Extend implementation that copies elements out of references before pushing them onto the Vec.\n///\n/// This implementation is specialized for slice iterators, where it uses [`copy_from_slice`] to\n/// append the entire slice at once.\n///\n/// [`copy_from_slice`]: https://doc.rust-lang.org/std/primitive.slice.html#method.copy_from_slice\nimpl<'a, 'bump, T: 'a + Copy> Extend<&'a T> for Vec<'bump, T> {\n fn extend>(&mut self, iter: I) {\n self.extend(iter.into_iter().cloned())\n }\n}\n\nmacro_rules! __impl_slice_eq1 {\n ($Lhs: ty, $Rhs: ty) => {\n __impl_slice_eq1! { $Lhs, $Rhs, Sized }\n };\n ($Lhs: ty, $Rhs: ty, $Bound: ident) => {\n impl<'a, 'b, A: $Bound, B> PartialEq<$Rhs> for $Lhs\n where\n A: PartialEq,\n {\n #[inline]\n fn eq(&self, other: &$Rhs) -> bool {\n self[..] == other[..]\n }\n }\n };\n}\n\n__impl_slice_eq1! { Vec<'a, A>, Vec<'b, B> }\n__impl_slice_eq1! { Vec<'a, A>, &'b [B] }\n__impl_slice_eq1! { Vec<'a, A>, &'b mut [B] }\n// __impl_slice_eq1! { Cow<'a, [A]>, Vec<'b, B>, Clone }\n\nmacro_rules! __impl_slice_eq1_array {\n ($Lhs: ty, $Rhs: ty) => {\n impl<'a, 'b, A, B, const N: usize> PartialEq<$Rhs> for $Lhs\n where\n A: PartialEq,\n {\n #[inline]\n fn eq(&self, other: &$Rhs) -> bool {\n self[..] == other[..]\n }\n }\n };\n}\n\n__impl_slice_eq1_array! { Vec<'a, A>, [B; N] }\n__impl_slice_eq1_array! { Vec<'a, A>, &'b [B; N] }\n__impl_slice_eq1_array! { Vec<'a, A>, &'b mut [B; N] }\n\n/// Implements comparison of vectors, lexicographically.\nimpl<'bump, T: 'bump + PartialOrd> PartialOrd for Vec<'bump, T> {\n #[inline]\n fn partial_cmp(&self, other: &Vec<'bump, T>) -> Option {\n PartialOrd::partial_cmp(&**self, &**other)\n }\n}\n\nimpl<'bump, T: 'bump + Eq> Eq for Vec<'bump, T> {}\n\n/// Implements ordering of vectors, lexicographically.\nimpl<'bump, T: 'bump + Ord> Ord for Vec<'bump, T> {\n #[inline]\n fn cmp(&self, other: &Vec<'bump, T>) -> Ordering {\n Ord::cmp(&**self, &**other)\n }\n}\n\nimpl<'bump, T: 'bump + fmt::Debug> fmt::Debug for Vec<'bump, T> {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n fmt::Debug::fmt(&**self, f)\n }\n}\n\nimpl<'bump, T: 'bump> AsRef> for Vec<'bump, T> {\n fn as_ref(&self) -> &Vec<'bump, T> {\n self\n }\n}\n\nimpl<'bump, T: 'bump> AsMut> for Vec<'bump, T> {\n fn as_mut(&mut self) -> &mut Vec<'bump, T> {\n self\n }\n}\n\nimpl<'bump, T: 'bump> AsRef<[T]> for Vec<'bump, T> {\n fn as_ref(&self) -> &[T] {\n self\n }\n}\n\nimpl<'bump, T: 'bump> AsMut<[T]> for Vec<'bump, T> {\n fn as_mut(&mut self) -> &mut [T] {\n self\n }\n}\n\n#[cfg(feature = \"boxed\")]\nimpl<'bump, T: 'bump> From> for crate::boxed::Box<'bump, [T]> {\n fn from(v: Vec<'bump, T>) -> crate::boxed::Box<'bump, [T]> {\n v.into_boxed_slice()\n }\n}\n\nimpl<'bump, T: 'bump> Borrow<[T]> for Vec<'bump, T> {\n #[inline]\n fn borrow(&self) -> &[T] {\n &self[..]\n }\n}\n\nimpl<'bump, T: 'bump> BorrowMut<[T]> for Vec<'bump, T> {\n #[inline]\n fn borrow_mut(&mut self) -> &mut [T] {\n &mut self[..]\n }\n}\n\nimpl<'bump, T> Drop for Vec<'bump, T> {\n fn drop(&mut self) {\n unsafe {\n // use drop for [T]\n // use a raw slice to refer to the elements of the vector as weakest necessary type;\n // could avoid questions of validity in certain cases\n ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.as_mut_ptr(), self.len))\n }\n // RawVec handles deallocation\n }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n// Clone-on-write\n////////////////////////////////////////////////////////////////////////////////\n\n// impl<'a, 'bump, T: Clone> From> for Cow<'a, [T]> {\n// fn from(v: Vec<'bump, T>) -> Cow<'a, [T]> {\n// Cow::Owned(v)\n// }\n// }\n\n// impl<'a, 'bump, T: Clone> From<&'a Vec<'bump, T>> for Cow<'a, [T]> {\n// fn from(v: &'a Vec<'bump, T>) -> Cow<'a, [T]> {\n// Cow::Borrowed(v.as_slice())\n// }\n// }\n\n////////////////////////////////////////////////////////////////////////////////\n// Iterators\n////////////////////////////////////////////////////////////////////////////////\n\n/// An iterator that moves out of a vector.\n///\n/// This `struct` is created by the [`Vec::into_iter`] method\n/// (provided by the [`IntoIterator`] trait).\n///\n/// [`IntoIterator`]: https://doc.rust-lang.org/std/iter/trait.IntoIterator.html\npub struct IntoIter<'bump, T> {\n phantom: PhantomData<&'bump [T]>,\n ptr: *const T,\n end: *const T,\n}\n\nimpl<'bump, T: fmt::Debug> fmt::Debug for IntoIter<'bump, T> {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n f.debug_tuple(\"IntoIter\").field(&self.as_slice()).finish()\n }\n}\n\nimpl<'bump, T: 'bump> IntoIter<'bump, T> {\n /// Returns the remaining items of this iterator as a slice.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let vec = bumpalo::vec![in &b; 'a', 'b', 'c'];\n /// let mut into_iter = vec.into_iter();\n /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']);\n /// let _ = into_iter.next().unwrap();\n /// assert_eq!(into_iter.as_slice(), &['b', 'c']);\n /// ```\n pub fn as_slice(&self) -> &[T] {\n unsafe { slice::from_raw_parts(self.ptr, self.len()) }\n }\n\n /// Returns the remaining items of this iterator as a mutable slice.\n ///\n /// # Examples\n ///\n /// ```\n /// use bumpalo::{Bump, collections::Vec};\n ///\n /// let b = Bump::new();\n ///\n /// let vec = bumpalo::vec![in &b; 'a', 'b', 'c'];\n /// let mut into_iter = vec.into_iter();\n /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']);\n /// into_iter.as_mut_slice()[2] = 'z';\n /// assert_eq!(into_iter.next().unwrap(), 'a');\n /// assert_eq!(into_iter.next().unwrap(), 'b');\n /// assert_eq!(into_iter.next().unwrap(), 'z');\n /// ```\n pub fn as_mut_slice(&mut self) -> &mut [T] {\n unsafe { slice::from_raw_parts_mut(self.ptr as *mut T, self.len()) }\n }\n}\n\nunsafe impl<'bump, T: Send> Send for IntoIter<'bump, T> {}\nunsafe impl<'bump, T: Sync> Sync for IntoIter<'bump, T> {}\n\nimpl<'bump, T: 'bump> Iterator for IntoIter<'bump, T> {\n type Item = T;\n\n #[inline]\n fn next(&mut self) -> Option {\n unsafe {\n if self.ptr as *const _ == self.end {\n None\n } else if mem::size_of::() == 0 {\n // purposefully don't use 'ptr.offset' because for\n // vectors with 0-size elements this would return the\n // same pointer.\n self.ptr = arith_offset(self.ptr as *const i8, 1) as *mut T;\n\n // Make up a value of this ZST.\n Some(mem::zeroed())\n } else {\n let old = self.ptr;\n self.ptr = self.ptr.offset(1);\n\n Some(ptr::read(old))\n }\n }\n }\n\n #[inline]\n fn size_hint(&self) -> (usize, Option) {\n let exact = if mem::size_of::() == 0 {\n (self.end as usize).wrapping_sub(self.ptr as usize)\n } else {\n unsafe { offset_from(self.end, self.ptr) as usize }\n };\n (exact, Some(exact))\n }\n\n #[inline]\n fn count(self) -> usize {\n self.len()\n }\n}\n\nimpl<'bump, T: 'bump> DoubleEndedIterator for IntoIter<'bump, T> {\n #[inline]\n fn next_back(&mut self) -> Option {\n unsafe {\n if self.end == self.ptr {\n None\n } else if mem::size_of::() == 0 {\n // See above for why 'ptr.offset' isn't used\n self.end = arith_offset(self.end as *const i8, -1) as *mut T;\n\n // Make up a value of this ZST.\n Some(mem::zeroed())\n } else {\n self.end = self.end.offset(-1);\n\n Some(ptr::read(self.end))\n }\n }\n }\n}\n\nimpl<'bump, T: 'bump> ExactSizeIterator for IntoIter<'bump, T> {}\n\nimpl<'bump, T: 'bump> FusedIterator for IntoIter<'bump, T> {}\n\nimpl<'bump, T> Drop for IntoIter<'bump, T> {\n fn drop(&mut self) {\n // drop all remaining elements\n self.for_each(drop);\n }\n}\n\n/// A draining iterator for `Vec<'bump, T>`.\n///\n/// This `struct` is created by the [`Vec::drain`] method.\npub struct Drain<'a, 'bump, T: 'a + 'bump> {\n /// Index of tail to preserve\n tail_start: usize,\n /// Length of tail\n tail_len: usize,\n /// Current remaining range to remove\n iter: slice::Iter<'a, T>,\n vec: NonNull>,\n}\n\nimpl<'a, 'bump, T: 'a + 'bump + fmt::Debug> fmt::Debug for Drain<'a, 'bump, T> {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n f.debug_tuple(\"Drain\").field(&self.iter.as_slice()).finish()\n }\n}\n\nunsafe impl<'a, 'bump, T: Sync> Sync for Drain<'a, 'bump, T> {}\nunsafe impl<'a, 'bump, T: Send> Send for Drain<'a, 'bump, T> {}\n\nimpl<'a, 'bump, T> Iterator for Drain<'a, 'bump, T> {\n type Item = T;\n\n #[inline]\n fn next(&mut self) -> Option {\n self.iter\n .next()\n .map(|elt| unsafe { ptr::read(elt as *const _) })\n }\n\n fn size_hint(&self) -> (usize, Option) {\n self.iter.size_hint()\n }\n}\n\nimpl<'a, 'bump, T> DoubleEndedIterator for Drain<'a, 'bump, T> {\n #[inline]\n fn next_back(&mut self) -> Option {\n self.iter\n .next_back()\n .map(|elt| unsafe { ptr::read(elt as *const _) })\n }\n}\n\nimpl<'a, 'bump, T> Drop for Drain<'a, 'bump, T> {\n fn drop(&mut self) {\n // exhaust self first\n self.for_each(drop);\n\n if self.tail_len > 0 {\n unsafe {\n let source_vec = self.vec.as_mut();\n // memmove back untouched tail, update to new length\n let start = source_vec.len();\n let tail = self.tail_start;\n if tail != start {\n let src = source_vec.as_ptr().add(tail);\n let dst = source_vec.as_mut_ptr().add(start);\n ptr::copy(src, dst, self.tail_len);\n }\n source_vec.set_len(start + self.tail_len);\n }\n }\n }\n}\n\nimpl<'a, 'bump, T> ExactSizeIterator for Drain<'a, 'bump, T> {}\n\nimpl<'a, 'bump, T> FusedIterator for Drain<'a, 'bump, T> {}\n\n/// A splicing iterator for `Vec`.\n///\n/// This struct is created by the [`Vec::splice`] method. See its\n/// documentation for more information.\n#[derive(Debug)]\npub struct Splice<'a, 'bump, I>\nwhere\n I: Iterator,\n I::Item: 'a + 'bump,\n{\n drain: Drain<'a, 'bump, I::Item>,\n replace_with: I,\n}\n\nimpl<'a, 'bump, I: Iterator> Iterator for Splice<'a, 'bump, I> {\n type Item = I::Item;\n\n fn next(&mut self) -> Option {\n self.drain.next()\n }\n\n fn size_hint(&self) -> (usize, Option) {\n self.drain.size_hint()\n }\n}\n\nimpl<'a, 'bump, I: Iterator> DoubleEndedIterator for Splice<'a, 'bump, I> {\n fn next_back(&mut self) -> Option {\n self.drain.next_back()\n }\n}\n\nimpl<'a, 'bump, I: Iterator> ExactSizeIterator for Splice<'a, 'bump, I> {}\n\nimpl<'a, 'bump, I: Iterator> Drop for Splice<'a, 'bump, I> {\n fn drop(&mut self) {\n self.drain.by_ref().for_each(drop);\n\n unsafe {\n if self.drain.tail_len == 0 {\n self.drain.vec.as_mut().extend(self.replace_with.by_ref());\n return;\n }\n\n // First fill the range left by drain().\n if !self.drain.fill(&mut self.replace_with) {\n return;\n }\n\n // There may be more elements. Use the lower bound as an estimate.\n // FIXME: Is the upper bound a better guess? Or something else?\n let (lower_bound, _upper_bound) = self.replace_with.size_hint();\n if lower_bound > 0 {\n self.drain.move_tail(lower_bound);\n if !self.drain.fill(&mut self.replace_with) {\n return;\n }\n }\n\n // Collect any remaining elements.\n // This is a zero-length vector which does not allocate if `lower_bound` was exact.\n let mut collected = Vec::new_in(self.drain.vec.as_ref().buf.bump());\n collected.extend(self.replace_with.by_ref());\n let mut collected = collected.into_iter();\n // Now we have an exact count.\n if collected.len() > 0 {\n self.drain.move_tail(collected.len());\n let filled = self.drain.fill(&mut collected);\n debug_assert!(filled);\n debug_assert_eq!(collected.len(), 0);\n }\n }\n // Let `Drain::drop` move the tail back if necessary and restore `vec.len`.\n }\n}\n\n/// Private helper methods for `Splice::drop`\nimpl<'a, 'bump, T> Drain<'a, 'bump, T> {\n /// The range from `self.vec.len` to `self.tail_start` contains elements\n /// that have been moved out.\n /// Fill that range as much as possible with new elements from the `replace_with` iterator.\n /// Return whether we filled the entire range. (`replace_with.next()` didn’t return `None`.)\n unsafe fn fill>(&mut self, replace_with: &mut I) -> bool {\n let vec = self.vec.as_mut();\n let range_start = vec.len;\n let range_end = self.tail_start;\n let range_slice =\n slice::from_raw_parts_mut(vec.as_mut_ptr().add(range_start), range_end - range_start);\n\n for place in range_slice {\n if let Some(new_item) = replace_with.next() {\n ptr::write(place, new_item);\n vec.len += 1;\n } else {\n return false;\n }\n }\n true\n }\n\n /// Make room for inserting more elements before the tail.\n unsafe fn move_tail(&mut self, extra_capacity: usize) {\n let vec = self.vec.as_mut();\n let used_capacity = self.tail_start + self.tail_len;\n vec.buf.reserve(used_capacity, extra_capacity);\n\n let new_tail_start = self.tail_start + extra_capacity;\n let src = vec.as_ptr().add(self.tail_start);\n let dst = vec.as_mut_ptr().add(new_tail_start);\n ptr::copy(src, dst, self.tail_len);\n self.tail_start = new_tail_start;\n }\n}\n\n/// An iterator produced by calling [`Vec::drain_filter`].\n#[derive(Debug)]\npub struct DrainFilter<'a, 'bump: 'a, T: 'a + 'bump, F>\nwhere\n F: FnMut(&mut T) -> bool,\n{\n vec: &'a mut Vec<'bump, T>,\n pred: F,\n\n // Iteration index.\n idx: usize,\n // Index to write kept items into.\n write: usize,\n // The original length.\n old_len: usize,\n}\n\nimpl<'a, 'bump, T, F> Iterator for DrainFilter<'a, 'bump, T, F>\nwhere\n F: FnMut(&mut T) -> bool,\n{\n type Item = T;\n\n fn next(&mut self) -> Option {\n debug_assert_eq!(self.vec.len(), 0);\n debug_assert!(self.write <= self.idx);\n unsafe {\n while self.idx != self.old_len {\n let i = self.idx;\n self.idx += 1;\n let v = slice::from_raw_parts_mut(self.vec.as_mut_ptr(), self.old_len);\n if (self.pred)(&mut v[i]) {\n return Some(ptr::read(&v[i]));\n } else {\n if i != self.write {\n let src: *const T = &v[i];\n let dst: *mut T = &mut v[self.write];\n ptr::copy_nonoverlapping(src, dst, 1);\n }\n self.write += 1;\n }\n }\n None\n }\n }\n\n fn size_hint(&self) -> (usize, Option) {\n (0, Some(self.old_len - self.idx))\n }\n}\n\nimpl<'a, 'bump, T, F> Drop for DrainFilter<'a, 'bump, T, F>\nwhere\n F: FnMut(&mut T) -> bool,\n{\n fn drop(&mut self) {\n /// Guard to shift the trailing items into place if the `DrainFilter`'s\n /// predicate panics, leaving it in a valid state.\n struct BackshiftOnDrop<'a, 'bump, T> {\n vec: &'a mut Vec<'bump, T>,\n idx: usize,\n write: usize,\n old_len: usize,\n }\n\n impl Drop for BackshiftOnDrop<'_, '_, T> {\n fn drop(&mut self) {\n unsafe {\n let tail_len = self.old_len - self.idx;\n if tail_len > 0 && self.write < self.idx {\n let ptr = self.vec.as_mut_ptr();\n ptr::copy(ptr.add(self.idx), ptr.add(self.write), tail_len);\n }\n self.vec.set_len(self.write + tail_len);\n }\n }\n }\n\n let mut guard = BackshiftOnDrop {\n vec: self.vec,\n idx: self.idx,\n write: self.write,\n old_len: self.old_len,\n };\n\n unsafe {\n while guard.idx < guard.old_len {\n let i = guard.idx;\n let v = slice::from_raw_parts_mut(guard.vec.as_mut_ptr(), guard.old_len);\n let drained = (self.pred)(&mut v[i]);\n guard.idx += 1;\n if drained {\n ptr::drop_in_place(&mut v[i]);\n } else {\n if i != guard.write {\n let src: *const T = &v[i];\n let dst: *mut T = &mut v[guard.write];\n ptr::copy_nonoverlapping(src, dst, 1);\n }\n guard.write += 1;\n }\n }\n }\n }\n}\n\n#[cfg(feature = \"std\")]\nimpl<'bump> io::Write for Vec<'bump, u8> {\n #[inline]\n fn write(&mut self, buf: &[u8]) -> io::Result {\n self.extend_from_slice_copy(buf);\n Ok(buf.len())\n }\n\n #[inline]\n fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {\n self.extend_from_slice_copy(buf);\n Ok(())\n }\n\n #[inline]\n fn flush(&mut self) -> io::Result<()> {\n Ok(())\n }\n}\n\n#[cfg(feature = \"serde\")]\nmod serialize {\n use super::*;\n\n use serde::{ser::SerializeSeq, Serialize, Serializer};\n\n impl<'a, T> Serialize for Vec<'a, T>\n where\n T: Serialize,\n {\n fn serialize(&self, serializer: S) -> Result\n where\n S: Serializer,\n {\n let mut seq = serializer.serialize_seq(Some(self.len))?;\n for e in self.iter() {\n seq.serialize_element(e)?;\n }\n seq.end()\n }\n }\n}\n" }, { "package": "bytes@1.5.0", "sourcePath": "README.md", "sha256": "c1b2b54999d4829f9f64fb41cbdf05a72d565be0dd078a8633d34631147498a1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# Bytes\n\nA utility library for working with bytes.\n\n[![Crates.io][crates-badge]][crates-url]\n[![Build Status][ci-badge]][ci-url]\n\n[crates-badge]: https://img.shields.io/crates/v/bytes.svg\n[crates-url]: https://crates.io/crates/bytes\n[ci-badge]: https://github.com/tokio-rs/bytes/workflows/CI/badge.svg\n[ci-url]: https://github.com/tokio-rs/bytes/actions\n\n[Documentation](https://docs.rs/bytes)\n\n## Usage\n\nTo use `bytes`, first add this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nbytes = \"1\"\n```\n\nNext, add this to your crate:\n\n```rust\nuse bytes::{Bytes, BytesMut, Buf, BufMut};\n```\n\n## Serde support\n\nSerde support is optional and disabled by default. To enable use the feature `serde`.\n\n```toml\n[dependencies]\nbytes = { version = \"1\", features = [\"serde\"] }\n```\n\n## Building documentation\n\nWhen building the `bytes` documentation the `docsrs` option should be used, otherwise\nfeature gates will not be shown. This requires a nightly toolchain:\n\n```\nRUSTDOCFLAGS=\"--cfg docsrs\" cargo +nightly doc\n```\n\n## License\n\nThis project is licensed under the [MIT license](LICENSE).\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in `bytes` by you, shall be licensed as MIT, without any additional\nterms or conditions.\n" }, { "package": "cc@1.4.7", "sourcePath": "README.md", "sha256": "cc952a506dc726eed3e128d47d62defbac4470bbb0e465f6a043ae8bf0c96d19", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# cc-rs\n\nA library for\n[Cargo build scripts](https://doc.rust-lang.org/cargo/reference/build-scripts.html)\nto compile a set of C/C++/assembly/CUDA files into a static archive for Cargo to\nlink into the crate being built. This crate does not compile code itself; it\ncalls out to the default compiler for the platform. This crate will\nautomatically detect situations such as cross compilation and various\nenvironment variables and will build code appropriately.\n\nRefer to the [documentation](https://docs.rs/cc) for detailed usage\ninstructions.\n\n## License\n\nThis project is licensed under either of\n\n * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or\n https://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or\n https://opensource.org/license/mit)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in cc-rs by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "package": "cfg-if@1.0.5", "sourcePath": "README.md", "sha256": "d16553c2dc0028d196c2ceb82e1685932061daacd3e0240339d9e393695c1e71", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# cfg-if\n\n> [!WARNING]\n> This crate has been \"replaced\" by the [`cfg_select!`] macro, which is stable since Rust 1.95.0 with a slightly different syntax. Barring breakages and security fixes, this crate will no longer be updated.\n\n[`cfg_select!`]: https://doc.rust-lang.org/std/macro.cfg_select.html\n\n[Documentation](https://docs.rs/cfg-if)\n\nA macro to ergonomically define an item depending on a large number of #[cfg]\nparameters. Structured like an if-else chain, the first matching branch is the\nitem that gets emitted.\n\n```toml\n[dependencies]\ncfg-if = \"1.0\"\n```\n\n## Example\n\n```rust\ncfg_if::cfg_if! {\n if #[cfg(unix)] {\n fn foo() { /* unix specific functionality */ }\n } else if #[cfg(target_pointer_width = \"32\")] {\n fn foo() { /* non-unix, 32-bit functionality */ }\n } else {\n fn foo() { /* fallback implementation */ }\n }\n}\n\nfn main() {\n foo();\n}\n```\nThe `cfg_if!` block above is expanded to:\n```rust\n#[cfg(unix)]\nfn foo() { /* unix specific functionality */ }\n#[cfg(all(target_pointer_width = \"32\", not(unix)))]\nfn foo() { /* non-unix, 32-bit functionality */ }\n#[cfg(not(any(unix, target_pointer_width = \"32\")))]\nfn foo() { /* fallback implementation */ }\n```\n\n# License\n\nThis project is licensed under either of\n\n * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or\n https://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or\n https://opensource.org/licenses/MIT)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in `cfg-if` by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "package": "color_quant@1.1.0", "sourcePath": "src/lib.rs", "sha256": "17a7ed7a6c994b475976558f3492c8890d089c1ee19f4ea3cd246c28145c895a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/*\nNeuQuant Neural-Net Quantization Algorithm by Anthony Dekker, 1994.\nSee \"Kohonen neural networks for optimal colour quantization\"\nin \"Network: Computation in Neural Systems\" Vol. 5 (1994) pp 351-367.\nfor a discussion of the algorithm.\nSee also http://members.ozemail.com.au/~dekker/NEUQUANT.HTML\n\nIncorporated bugfixes and alpha channel handling from pngnq\nhttp://pngnq.sourceforge.net\n\nCopyright (c) 2014 The Piston Developers\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n\nNeuQuant Neural-Net Quantization Algorithm\n------------------------------------------\n\nCopyright (c) 1994 Anthony Dekker\n\nNEUQUANT Neural-Net quantization algorithm by Anthony Dekker, 1994.\nSee \"Kohonen neural networks for optimal colour quantization\"\nin \"Network: Computation in Neural Systems\" Vol. 5 (1994) pp 351-367.\nfor a discussion of the algorithm.\nSee also http://members.ozemail.com.au/~dekker/NEUQUANT.HTML\n\nAny party obtaining a copy of these files from the author, directly or\nindirectly, is granted, free of charge, a full and unrestricted irrevocable,\nworld-wide, paid up, royalty-free, nonexclusive right and license to deal\nin this software and documentation files (the \"Software\"), including without\nlimitation the rights to use, copy, modify, merge, publish, distribute, sublicense,\nand/or sell copies of the Software, and to permit persons who receive\ncopies from any such party to do so, with the only requirement being\nthat this copyright notice remain intact.\n\n*/\n\n//! # Color quantization library\n//!\n//! This library provides a color quantizer based on the [NEUQUANT](http://members.ozemail.com.au/~dekker/NEUQUANT.HTML)\n//!\n//! Original literature: Dekker, A. H. (1994). Kohonen neural networks for\n//! optimal colour quantization. *Network: Computation in Neural Systems*, 5(3), 351-367.\n//! [doi: 10.1088/0954-898X_5_3_003](https://doi.org/10.1088/0954-898X_5_3_003)\n//!\n//! See also \n//!\n//! ## Usage\n//!\n//! ```\n//! let data = vec![0; 40];\n//! let nq = color_quant::NeuQuant::new(10, 256, &data);\n//! let indixes: Vec = data.chunks(4).map(|pix| nq.index_of(pix) as u8).collect();\n//! let color_map = nq.color_map_rgba();\n//! ```\n\nmod math;\nuse crate::math::clamp;\n\nuse std::cmp::{max, min};\n\nconst CHANNELS: usize = 4;\n\nconst RADIUS_DEC: i32 = 30; // factor of 1/30 each cycle\n\nconst ALPHA_BIASSHIFT: i32 = 10; // alpha starts at 1\nconst INIT_ALPHA: i32 = 1 << ALPHA_BIASSHIFT; // biased by 10 bits\n\nconst GAMMA: f64 = 1024.0;\nconst BETA: f64 = 1.0 / GAMMA;\nconst BETAGAMMA: f64 = BETA * GAMMA;\n\n// four primes near 500 - assume no image has a length so large\n// that it is divisible by all four primes\nconst PRIMES: [usize; 4] = [499, 491, 478, 503];\n\n#[derive(Clone, Copy)]\nstruct Quad {\n r: T,\n g: T,\n b: T,\n a: T,\n}\n\ntype Neuron = Quad;\ntype Color = Quad;\n\npub struct NeuQuant {\n network: Vec,\n colormap: Vec,\n netindex: Vec,\n bias: Vec, // bias and freq arrays for learning\n freq: Vec,\n samplefac: i32,\n netsize: usize,\n}\n\nimpl NeuQuant {\n /// Creates a new neuronal network and trains it with the supplied data.\n ///\n /// Pixels are assumed to be in RGBA format.\n /// `colors` should be $>=64$. `samplefac` determines the faction of\n /// the sample that will be used to train the network. Its value must be in the\n /// range $[1, 30]$. A value of $1$ thus produces the best result but is also\n /// slowest. $10$ is a good compromise between speed and quality.\n pub fn new(samplefac: i32, colors: usize, pixels: &[u8]) -> Self {\n let netsize = colors;\n let mut this = NeuQuant {\n network: Vec::with_capacity(netsize),\n colormap: Vec::with_capacity(netsize),\n netindex: vec![0; 256],\n bias: Vec::with_capacity(netsize),\n freq: Vec::with_capacity(netsize),\n samplefac: samplefac,\n netsize: colors,\n };\n this.init(pixels);\n this\n }\n\n /// Initializes the neuronal network and trains it with the supplied data.\n ///\n /// This method gets called by `Self::new`.\n pub fn init(&mut self, pixels: &[u8]) {\n self.network.clear();\n self.colormap.clear();\n self.bias.clear();\n self.freq.clear();\n let freq = (self.netsize as f64).recip();\n for i in 0..self.netsize {\n let tmp = (i as f64) * 256.0 / (self.netsize as f64);\n // Sets alpha values at 0 for dark pixels.\n let a = if i < 16 { i as f64 * 16.0 } else { 255.0 };\n self.network.push(Neuron {\n r: tmp,\n g: tmp,\n b: tmp,\n a: a,\n });\n self.colormap.push(Color {\n r: 0,\n g: 0,\n b: 0,\n a: 255,\n });\n self.freq.push(freq);\n self.bias.push(0.0);\n }\n self.learn(pixels);\n self.build_colormap();\n self.build_netindex();\n }\n\n /// Maps the rgba-pixel in-place to the best-matching color in the color map.\n #[inline(always)]\n pub fn map_pixel(&self, pixel: &mut [u8]) {\n assert!(pixel.len() == 4);\n let (r, g, b, a) = (pixel[0], pixel[1], pixel[2], pixel[3]);\n let i = self.search_netindex(b, g, r, a);\n pixel[0] = self.colormap[i].r as u8;\n pixel[1] = self.colormap[i].g as u8;\n pixel[2] = self.colormap[i].b as u8;\n pixel[3] = self.colormap[i].a as u8;\n }\n\n /// Finds the best-matching index in the color map.\n ///\n /// `pixel` is assumed to be in RGBA format.\n #[inline(always)]\n pub fn index_of(&self, pixel: &[u8]) -> usize {\n assert!(pixel.len() == 4);\n let (r, g, b, a) = (pixel[0], pixel[1], pixel[2], pixel[3]);\n self.search_netindex(b, g, r, a)\n }\n\n /// Lookup pixel values for color at `idx` in the colormap.\n pub fn lookup(&self, idx: usize) -> Option<[u8; 4]> {\n self.colormap\n .get(idx)\n .map(|p| [p.r as u8, p.g as u8, p.b as u8, p.a as u8])\n }\n\n /// Returns the RGBA color map calculated from the sample.\n pub fn color_map_rgba(&self) -> Vec {\n let mut map = Vec::with_capacity(self.netsize * 4);\n for entry in &self.colormap {\n map.push(entry.r as u8);\n map.push(entry.g as u8);\n map.push(entry.b as u8);\n map.push(entry.a as u8);\n }\n map\n }\n\n /// Returns the RGBA color map calculated from the sample.\n pub fn color_map_rgb(&self) -> Vec {\n let mut map = Vec::with_capacity(self.netsize * 3);\n for entry in &self.colormap {\n map.push(entry.r as u8);\n map.push(entry.g as u8);\n map.push(entry.b as u8);\n }\n map\n }\n\n /// Move neuron i towards biased (a,b,g,r) by factor alpha\n fn salter_single(&mut self, alpha: f64, i: i32, quad: Quad) {\n let n = &mut self.network[i as usize];\n n.b -= alpha * (n.b - quad.b);\n n.g -= alpha * (n.g - quad.g);\n n.r -= alpha * (n.r - quad.r);\n n.a -= alpha * (n.a - quad.a);\n }\n\n /// Move neuron adjacent neurons towards biased (a,b,g,r) by factor alpha\n fn alter_neighbour(&mut self, alpha: f64, rad: i32, i: i32, quad: Quad) {\n let lo = max(i - rad, 0);\n let hi = min(i + rad, self.netsize as i32);\n let mut j = i + 1;\n let mut k = i - 1;\n let mut q = 0;\n\n while (j < hi) || (k > lo) {\n let rad_sq = rad as f64 * rad as f64;\n let alpha = (alpha * (rad_sq - q as f64 * q as f64)) / rad_sq;\n q += 1;\n if j < hi {\n let p = &mut self.network[j as usize];\n p.b -= alpha * (p.b - quad.b);\n p.g -= alpha * (p.g - quad.g);\n p.r -= alpha * (p.r - quad.r);\n p.a -= alpha * (p.a - quad.a);\n j += 1;\n }\n if k > lo {\n let p = &mut self.network[k as usize];\n p.b -= alpha * (p.b - quad.b);\n p.g -= alpha * (p.g - quad.g);\n p.r -= alpha * (p.r - quad.r);\n p.a -= alpha * (p.a - quad.a);\n k -= 1;\n }\n }\n }\n\n /// Search for biased BGR values\n /// finds closest neuron (min dist) and updates freq\n /// finds best neuron (min dist-bias) and returns position\n /// for frequently chosen neurons, freq[i] is high and bias[i] is negative\n /// bias[i] = gamma*((1/self.netsize)-freq[i])\n fn contest(&mut self, b: f64, g: f64, r: f64, a: f64) -> i32 {\n use std::f64;\n\n let mut bestd = f64::MAX;\n let mut bestbiasd: f64 = bestd;\n let mut bestpos = -1;\n let mut bestbiaspos: i32 = bestpos;\n\n for i in 0..self.netsize {\n let bestbiasd_biased = bestbiasd + self.bias[i];\n let mut dist;\n let n = &self.network[i];\n dist = (n.b - b).abs();\n dist += (n.r - r).abs();\n if dist < bestd || dist < bestbiasd_biased {\n dist += (n.g - g).abs();\n dist += (n.a - a).abs();\n if dist < bestd {\n bestd = dist;\n bestpos = i as i32;\n }\n let biasdist = dist - self.bias[i];\n if biasdist < bestbiasd {\n bestbiasd = biasdist;\n bestbiaspos = i as i32;\n }\n }\n self.freq[i] -= BETA * self.freq[i];\n self.bias[i] += BETAGAMMA * self.freq[i];\n }\n self.freq[bestpos as usize] += BETA;\n self.bias[bestpos as usize] -= BETAGAMMA;\n return bestbiaspos;\n }\n\n /// Main learning loop\n /// Note: the number of learning cycles is crucial and the parameters are not\n /// optimized for net sizes < 26 or > 256. 1064 colors seems to work fine\n fn learn(&mut self, pixels: &[u8]) {\n let initrad: i32 = self.netsize as i32 / 8; // for 256 cols, radius starts at 32\n let radiusbiasshift: i32 = 6;\n let radiusbias: i32 = 1 << radiusbiasshift;\n let init_bias_radius: i32 = initrad * radiusbias;\n let mut bias_radius = init_bias_radius;\n let alphadec = 30 + ((self.samplefac - 1) / 3);\n let lengthcount = pixels.len() / CHANNELS;\n let samplepixels = lengthcount / self.samplefac as usize;\n // learning cycles\n let n_cycles = match self.netsize >> 1 {\n n if n <= 100 => 100,\n n => n,\n };\n let delta = match samplepixels / n_cycles {\n 0 => 1,\n n => n,\n };\n let mut alpha = INIT_ALPHA;\n\n let mut rad = bias_radius >> radiusbiasshift;\n if rad <= 1 {\n rad = 0\n };\n\n let mut pos = 0;\n let step = *PRIMES\n .iter()\n .find(|&&prime| lengthcount % prime != 0)\n .unwrap_or(&PRIMES[3]);\n\n let mut i = 0;\n while i < samplepixels {\n let (r, g, b, a) = {\n let p = &pixels[CHANNELS * pos..][..CHANNELS];\n (p[0] as f64, p[1] as f64, p[2] as f64, p[3] as f64)\n };\n\n let j = self.contest(b, g, r, a);\n\n let alpha_ = (1.0 * alpha as f64) / INIT_ALPHA as f64;\n self.salter_single(alpha_, j, Quad { b, g, r, a });\n if rad > 0 {\n self.alter_neighbour(alpha_, rad, j, Quad { b, g, r, a })\n };\n\n pos += step;\n while pos >= lengthcount {\n pos -= lengthcount\n }\n\n i += 1;\n if i % delta == 0 {\n alpha -= alpha / alphadec;\n bias_radius -= bias_radius / RADIUS_DEC;\n rad = bias_radius >> radiusbiasshift;\n if rad <= 1 {\n rad = 0\n };\n }\n }\n }\n\n /// initializes the color map\n fn build_colormap(&mut self) {\n for i in 0usize..self.netsize {\n self.colormap[i].b = clamp(self.network[i].b.round() as i32);\n self.colormap[i].g = clamp(self.network[i].g.round() as i32);\n self.colormap[i].r = clamp(self.network[i].r.round() as i32);\n self.colormap[i].a = clamp(self.network[i].a.round() as i32);\n }\n }\n\n /// Insertion sort of network and building of netindex[0..255]\n fn build_netindex(&mut self) {\n let mut previouscol = 0;\n let mut startpos = 0;\n\n for i in 0..self.netsize {\n let mut p = self.colormap[i];\n let mut q;\n let mut smallpos = i;\n let mut smallval = p.g as usize; // index on g\n // find smallest in i..netsize-1\n for j in (i + 1)..self.netsize {\n q = self.colormap[j];\n if (q.g as usize) < smallval {\n // index on g\n smallpos = j;\n smallval = q.g as usize; // index on g\n }\n }\n q = self.colormap[smallpos];\n // swap p (i) and q (smallpos) entries\n if i != smallpos {\n ::std::mem::swap(&mut p, &mut q);\n self.colormap[i] = p;\n self.colormap[smallpos] = q;\n }\n // smallval entry is now in position i\n if smallval != previouscol {\n self.netindex[previouscol] = (startpos + i) >> 1;\n for j in (previouscol + 1)..smallval {\n self.netindex[j] = i\n }\n previouscol = smallval;\n startpos = i;\n }\n }\n let max_netpos = self.netsize - 1;\n self.netindex[previouscol] = (startpos + max_netpos) >> 1;\n for j in (previouscol + 1)..256 {\n self.netindex[j] = max_netpos\n } // really 256\n }\n\n /// Search for best matching color\n fn search_netindex(&self, b: u8, g: u8, r: u8, a: u8) -> usize {\n let mut bestd = 1 << 30; // ~ 1_000_000\n let mut best = 0;\n // start at netindex[g] and work outwards\n let mut i = self.netindex[g as usize];\n let mut j = if i > 0 { i - 1 } else { 0 };\n\n while (i < self.netsize) || (j > 0) {\n if i < self.netsize {\n let p = self.colormap[i];\n let mut e = p.g - g as i32;\n let mut dist = e * e; // inx key\n if dist >= bestd {\n break;\n } else {\n e = p.b - b as i32;\n dist += e * e;\n if dist < bestd {\n e = p.r - r as i32;\n dist += e * e;\n if dist < bestd {\n e = p.a - a as i32;\n dist += e * e;\n if dist < bestd {\n bestd = dist;\n best = i;\n }\n }\n }\n i += 1;\n }\n }\n if j > 0 {\n let p = self.colormap[j];\n let mut e = p.g - g as i32;\n let mut dist = e * e; // inx key\n if dist >= bestd {\n break;\n } else {\n e = p.b - b as i32;\n dist += e * e;\n if dist < bestd {\n e = p.r - r as i32;\n dist += e * e;\n if dist < bestd {\n e = p.a - a as i32;\n dist += e * e;\n if dist < bestd {\n bestd = dist;\n best = j;\n }\n }\n }\n j -= 1;\n }\n }\n }\n best\n }\n}\n" }, { "package": "colorchoice@1.0.5", "sourcePath": "README.md", "sha256": "fc9b47302f5a3244b4e447e5bf6f86a35c64a16bd1abef8c27f31c4b5abac7a2", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# colorchoice\n\n> Global override of color control\n\n[![Documentation](https://img.shields.io/badge/docs-master-blue.svg)][Documentation]\n![License](https://img.shields.io/crates/l/colorchoice.svg)\n[![Crates Status](https://img.shields.io/crates/v/colorchoice.svg)](https://crates.io/crates/colorchoice)\n\n## License\n\nLicensed under either of\n\n* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or )\n* MIT license ([LICENSE-MIT](LICENSE-MIT) or )\n\nat your option.\n\n## [Contribute](../../CONTRIBUTING.md)\n\nSpecial note: to be successful, this crate **cannot** break compatibility or\nelse different crates in the hierarchy will be reading different globals.\nWhile end users can work around this, it isn't ideal. If we need a new API, we can make\nthe old API an adapter to the new logic.\n\nSimilarly, we should strive to reduce **risk** of breaking compatibility by\nexposing as little as possible. Anything more should be broken out into a\nseparate crate that this crate can call into.\n\nUnless you explicitly state otherwise, any contribution intentionally\nsubmitted for inclusion in the work by you, as defined in the Apache-2.0\nlicense, shall be dual-licensed as above, without any additional terms or\nconditions.\n\n[Documentation]: https://docs.rs/colorchoice\n" }, { "package": "convert_case@0.6.0", "sourcePath": "README.md", "sha256": "6bb2369385222ffa8bb09c5e70e8e58fda1b3d46ee165749f15bcffc3da415f4", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# Convert Case\n\nConverts to and from various cases.\n\n## Rust Library `convert_case`\n\nConvert case was written in Rust and is ready to be used inline with your rust code as a library.\n```{rust}\nuse convert_case::{Case, Casing};\n\nassert_eq!(\"ronnieJamesDio\", \"Ronnie_James_dio\".to_case(Case::Camel));\nassert_eq!(\"io_stream\", \"IOStream\".to_case(Case::Snake));\nassert_eq!(\n \"2020-04-16 My Cat Cali\",\n \"2020-04-16_my_cat_cali\".from_case(Case::Snake).to_case(Case::Title)\n);\n```\nYou can read the API documentation on [docs.rs](https://docs.rs/convert_case/) for a list of all features and read lots of examples.\n\n## Command Line Utility `ccase`\n\nThe command line utility `ccase` was made to leverage the tools in the `convert_case` library.\n```\n$ ccase -t title super_mario_64\nSuper Mario 64\n\n$ ccase -f snake -t title 2020-04-15_my_cat_cali\n2020-04-16 My Cat Cali\n\n$ ccase -t camel \"convert to camel\"\nconvertToCamel\n```\n\nYou can read more about the `ccase` executable in the [`ccase` directory](https://github.com/rutrum/convert-case/tree/master/ccase) within this repository.\n\n## Links\n\n| | `convert_case` | `ccase` |\n| --- | --- | --- |\n| Repository | [github](https://github.com/rutrum/convert-case) | [github](https://github.com/rutrum/convert-case/tree/master/ccase) |\n| Crate | [crates.io](https://crates.io/crates/convert_case) | [crates.io](https://crates.io/crates/ccase) |\n| Documentation | [docs.rs](https://docs.rs/convert_case) | |\n\n## Cases\n\nThis is list of cases that convert\\_case supports. Some cases are simply aliases of others. The \"Random\" and \"PseudoRandom\" cases are provided in the `convert_case` library with the \"random\" feature, and are automatically provided in the `ccase` binary.\n\n| Case | Example |\n| ---- | ------- |\n| Upper | MY VARIABLE NAME |\n| Lower | my variable name |\n| Title | My Variable Name |\n| Toggle | mY vARIABLE nAME |\n| Alternating | mY vArIaBlE nAmE |\n| Camel | myVariableName |\n| Pascal | MyVariableName |\n| UpperCamel | MyVariableName |\n| Snake | my\\_variable\\_name |\n| UpperSnake | MY\\_VARIABLE\\_NAME |\n| ScreamingSnake | MY\\_VARIABLE\\_NAME |\n| Kebab | my-variable-name |\n| Cobol | MY-VARIABLE-NAME |\n| Train | My-Variable-Name |\n| Flat | myvariablename |\n| UpperFlat | MYVARIABLENAME |\n| Random | MY vaRiabLe nAME |\n| PseudoRandom | mY VaRiAblE nAMe |\n\n## License\n\nLicensed under [MIT License](./LICENSE).\n" }, { "package": "crc32fast@1.5.2", "sourcePath": "README.md", "sha256": "cc3944235f56e3fa42d98197f4a461ebeb0cd46bcb5a4d0f1506079c93d43c34", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# crc32fast [![Build Status][build-img]][build] [![Crates.io][crates-img]][crates] [![Documentation][docs-img]][docs]\n\n[build-img]: https://github.com/srijs/rust-crc32fast/actions/workflows/ci.yml/badge.svg\n[build]: https://github.com/srijs/rust-crc32fast/actions/workflows/ci.yml\n[crates-img]: https://img.shields.io/crates/v/crc32fast.svg\n[crates]: https://crates.io/crates/crc32fast\n[docs-img]: https://docs.rs/crc32fast/badge.svg\n[docs]: https://docs.rs/crc32fast\n\n_Fast, SIMD-accelerated CRC32 (IEEE) checksum computation_\n\n## Usage\n\n### Simple usage\n\nFor simple use-cases, you can call the `hash` convenience function to\ndirectly compute the CRC32 checksum for a given byte slice:\n\n```rust\nlet checksum = crc32fast::hash(b\"foo bar baz\");\n```\n\n### Advanced usage\n\nFor use-cases that require more flexibility or performance, for example when\nprocessing large amounts of data, you can create and manipulate a `Hasher`:\n\n```rust\nuse crc32fast::Hasher;\n\nlet mut hasher = Hasher::new();\nhasher.update(b\"foo bar baz\");\nlet checksum = hasher.finalize();\n```\n\n## Performance\n\nThis crate contains multiple CRC32 implementations:\n\n- A fast baseline implementation which processes up to 16 bytes per iteration\n- An optimized implementation for modern `x86` using `sse` and `pclmulqdq` instructions\n- Wider `x86` implementations using `vpclmulqdq` on 256-bit (`avx2`) and 512-bit (`avx512f`)\n registers, folding many independent streams per instruction (compiled with Rust 1.89+; the\n crate's minimum supported Rust version is otherwise unchanged)\n- An optimized implementation for `aarch64` using `crc32` instructions, with a 3-way interleaved\n variant that hides instruction latency on larger inputs\n\nCalling the `Hasher::new` constructor at runtime will perform a feature detection to select the most\noptimal implementation for the current CPU feature set.\n\n| crate | version | variant | ns/iter | MB/s |\n| ----------------------------------- | ------- | --------- | ------- | ---- |\n| [crc](https://crates.io/crates/crc) | 1.8.1 | n/a | 4,926 | 207 |\n| crc32fast (this crate) | 1.0.0 | baseline | 683 | 1499 |\n| crc32fast (this crate) | 1.0.0 | pclmulqdq | 140 | 7314 |\n\n## Memory Safety\n\nDue to the use of SIMD intrinsics for the optimized implementations, this crate contains some amount of `unsafe` code.\n\nIn order to ensure memory safety, the relevant code has been fuzz tested using `libFuzzer` (via [cargo-fuzz](https://github.com/rust-fuzz/cargo-fuzz)) with millions of iterations in both `debug` and `release` build settings. You can inspect the test setup in the `fuzz` sub-directory, which also has instructions on how to run the tests yourself.\n\nOn top of that, every commit is tested using an address sanitizer in CI to catch any out of bounds memory accesses.\n\nEven though neither fuzzing nor sanitization has revealed any safety bugs yet, please don't hesitate to file an issue if you run into any crashes or other unexpected behaviour.\n\n## Available feature flags\n\n### `std` (default: enabled)\n\nThis library supports being built without the Rust `std` library, which is useful for low-level use-cases such as embedded where no operating system is available. To build the crate in a `no_std` context, disable the default `std` feature.\n\nNote: Because runtime CPU feature detection requires OS support, the specialized SIMD implementations will be unavailable when the `std` feature is disabled.\n\n### `nightly` (default: disabled)\n\nThis feature flag enables unstable features that are only available on the `nightly` channel. Keep in mind that when enabling this feature flag, you\nmight experience breaking changes when updating compiler versions.\n\n## License\n\nThis project is licensed under either of\n\n- Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or\n https://www.apache.org/licenses/LICENSE-2.0)\n- MIT license ([LICENSE-MIT](LICENSE-MIT) or\n https://opensource.org/license/mit)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this project by you, as defined in the Apache-2.0 license,\nshall be dual licensed as above, without any additional terms or conditions.\n" }, { "package": "deflate@1.0.0", "sourcePath": "src/bitstream.rs", "sha256": "5bdde1cdbc1d29ee2bbfb3040c0f5fcc06599c9ff464cdb6b360cdfa9fed0b9b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// This was originally based on code from: https://github.com/nwin/lzw\n// Copyright (c) 2015 nwin\n// which is under both Apache 2.0 and MIT\n\n//! This module provides a bit writer\nuse std::io::{self, Write};\n\n#[cfg(target_pointer_width = \"64\")]\n#[macro_use]\nmod arch_dep {\n /// The data type of the accumulator.\n /// a 64-bit value allows us to store more before\n /// each push to the vector, but is sub-optimal\n /// on 32-bit platforms.\n pub type AccType = u64;\n pub const FLUSH_AT: u8 = 48;\n /// Push pending bits to vector.\n /// Using a macro here since an inline function.\n /// didn't optimise properly.\n /// TODO June 2019: See if it's still needed.\n macro_rules! push {\n ($s:ident) => {\n $s.w.extend_from_slice(\n &[\n $s.acc as u8,\n ($s.acc >> 8) as u8,\n ($s.acc >> 16) as u8,\n ($s.acc >> 24) as u8,\n ($s.acc >> 32) as u8,\n ($s.acc >> 40) as u8,\n ][..],\n )\n };\n }\n}\n#[cfg(not(target_pointer_width = \"64\"))]\n#[macro_use]\nmod arch_dep {\n pub type AccType = u32;\n pub const FLUSH_AT: u8 = 16;\n macro_rules! push {\n ($s:ident) => {\n // Unlike the 64-bit case, using copy_from_slice seemed to worsen performance here.\n // TODO: Needs benching on a 32-bit system to see what works best.\n $s.w.push($s.acc as u8);\n $s.w.push(($s.acc >> 8) as u8);\n };\n }\n}\n\nuse self::arch_dep::*;\n\n/// Writes bits to a byte stream, LSB first.\npub struct LsbWriter {\n // Public for now so it can be replaced after initialization.\n pub w: Vec,\n bits: u8,\n acc: AccType,\n}\n\nimpl LsbWriter {\n /// Creates a new bit reader\n pub const fn new(writer: Vec) -> LsbWriter {\n LsbWriter {\n w: writer,\n bits: 0,\n acc: 0,\n }\n }\n\n pub const fn pending_bits(&self) -> u8 {\n self.bits\n }\n\n /// Buffer n number of bits, and write them to the vec if there are enough pending bits.\n pub fn write_bits(&mut self, v: u16, n: u8) {\n // NOTE: This outputs garbage data if n is 0, but v is not 0\n self.acc |= (AccType::from(v)) << self.bits;\n self.bits += n;\n // Waiting until we have FLUSH_AT bits and pushing them all in one batch.\n while self.bits >= FLUSH_AT {\n push!(self);\n self.acc >>= FLUSH_AT;\n self.bits -= FLUSH_AT;\n }\n }\n\n fn write_bits_finish(&mut self, v: u16, n: u8) {\n // NOTE: This outputs garbage data if n is 0, but v is not 0\n self.acc |= (AccType::from(v)) << self.bits;\n self.bits += n % 8;\n while self.bits >= 8 {\n self.w.push(self.acc as u8);\n self.acc >>= 8;\n self.bits -= 8;\n }\n }\n\n pub fn flush_raw(&mut self) {\n let missing = FLUSH_AT - self.bits;\n // Have to test for self.bits > 0 here,\n // otherwise flush would output an extra byte when flush was called at a byte boundary\n if missing > 0 && self.bits > 0 {\n self.write_bits_finish(0, missing);\n }\n }\n}\n\nimpl Write for LsbWriter {\n fn write(&mut self, buf: &[u8]) -> io::Result {\n if self.acc == 0 {\n self.w.extend_from_slice(buf)\n } else {\n for &byte in buf.iter() {\n self.write_bits(u16::from(byte), 8)\n }\n }\n Ok(buf.len())\n }\n\n fn flush(&mut self) -> io::Result<()> {\n self.flush_raw();\n Ok(())\n }\n}\n\n#[cfg(test)]\nmod test {\n use super::LsbWriter;\n\n #[test]\n fn write_bits() {\n let input = [\n (3, 3),\n (10, 8),\n (88, 7),\n (0, 2),\n (0, 5),\n (0, 0),\n (238, 8),\n (126, 8),\n (161, 8),\n (10, 8),\n (238, 8),\n (174, 8),\n (126, 8),\n (174, 8),\n (65, 8),\n (142, 8),\n (62, 8),\n (10, 8),\n (1, 8),\n (161, 8),\n (78, 8),\n (62, 8),\n (158, 8),\n (206, 8),\n (10, 8),\n (64, 7),\n (0, 0),\n (24, 5),\n (0, 0),\n (174, 8),\n (126, 8),\n (193, 8),\n (174, 8),\n ];\n let expected = [\n 83, 192, 2, 220, 253, 66, 21, 220, 93, 253, 92, 131, 28, 125, 20, 2, 66, 157, 124, 60,\n 157, 21, 128, 216, 213, 47, 216, 21,\n ];\n let mut writer = LsbWriter::new(Vec::new());\n for v in input.iter() {\n writer.write_bits(v.0, v.1);\n }\n writer.flush_raw();\n assert_eq!(writer.w, expected);\n }\n}\n\n#[cfg(all(test, feature = \"benchmarks\"))]\nmod bench {\n use super::LsbWriter;\n use test_std::Bencher;\n #[bench]\n fn bit_writer(b: &mut Bencher) {\n let input = [\n (3, 3),\n (10, 8),\n (88, 7),\n (0, 2),\n (0, 5),\n (0, 0),\n (238, 8),\n (126, 8),\n (161, 8),\n (10, 8),\n (238, 8),\n (174, 8),\n (126, 8),\n (174, 8),\n (65, 8),\n (142, 8),\n (62, 8),\n (10, 8),\n (1, 8),\n (161, 8),\n (78, 8),\n (62, 8),\n (158, 8),\n (206, 8),\n (10, 8),\n (64, 7),\n (0, 0),\n (24, 5),\n (0, 0),\n (174, 8),\n (126, 8),\n (193, 8),\n (174, 8),\n ];\n let mut writer = LsbWriter::new(Vec::with_capacity(100));\n b.iter(|| {\n for v in input.iter() {\n let _ = writer.write_bits(v.0, v.1);\n }\n });\n }\n}\n" }, { "package": "defmt@1.1.1", "sourcePath": "README.md", "sha256": "0869dedb2a47f8ea123e19ad3b68fb56d1be253016687f03d74b5c693416aae4", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# `defmt`\n\n`defmt` (\"de format\", short for \"deferred formatting\") is a highly efficient logging framework that targets resource-constrained devices, like microcontrollers.\n\nFor more details about the framework check the book at .\n\n## Setup\n\n### New project\n\nThe fastest way to get started with `defmt` is to use our [app-template] to set up a new Cortex-M embedded project.\n\n[app-template]: https://github.com/knurling-rs/app-template\n\n### Existing project\n\nTo include `defmt` in your existing project, follow our [Application Setup guide].\n\n[Application Setup guide]: https://defmt.ferrous-systems.com/setup.html\n\n## MSRV\n\nThe minimum supported Rust version is 1.76 (or Ferrocene 24.05). This crate is tested against the latest stable Rust version and the MSRV.\n\n## Support\n\n`defmt` is part of the [Knurling] project, [Ferrous Systems]' effort at\nimproving tooling used to develop for embedded systems.\n\nIf you think that our work is useful, consider sponsoring it via [GitHub\nSponsors].\n\n## License\n\nLicensed under either of\n\n- Apache License, Version 2.0 ([LICENSE-APACHE](../LICENSE-APACHE) or\n )\n\n- MIT license ([LICENSE-MIT](../LICENSE-MIT) or )\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\nlicensed as above, without any additional terms or conditions.\n\n[Knurling]: https://knurling.ferrous-systems.com/\n[Ferrous Systems]: https://ferrous-systems.com/\n[GitHub Sponsors]: https://github.com/sponsors/knurling-rs\n" }, { "package": "defmt-macros@1.1.1", "sourcePath": "README.md", "sha256": "54bca1c967197f21a24c9a6989a67574780aad189149aa21aa363058b5351af5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# `defmt-macros`\n\n`defmt` (\"de format\", short for \"deferred formatting\") is a highly efficient logging framework that targets resource-constrained devices, like microcontrollers.\n\nFor more details about the framework check the book at .\n\nThis library contains the proc macros used by\n[`defmt`](https://crates.io/crates/defmt).\n\n## MSRV\n\nThe minimum supported Rust version is 1.76 (or Ferrocene 24.05). This crate is tested against the latest stable Rust version and the MSRV.\n\n## Support\n\n`defmt` is part of the [Knurling] project, [Ferrous Systems]' effort at\nimproving tooling used to develop for embedded systems.\n\nIf you think that our work is useful, consider sponsoring it via [GitHub\nSponsors].\n\n## License\n\nLicensed under either of\n\n- Apache License, Version 2.0 ([LICENSE-APACHE](../LICENSE-APACHE) or\n )\n\n- MIT license ([LICENSE-MIT](../LICENSE-MIT) or )\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\nlicensed as above, without any additional terms or conditions.\n\n[Knurling]: https://knurling.ferrous-systems.com/\n[Ferrous Systems]: https://ferrous-systems.com/\n[GitHub Sponsors]: https://github.com/sponsors/knurling-rs\n" }, { "package": "defmt-parser@1.0.0", "sourcePath": "README.md", "sha256": "a40aae66ea32ccd2b2c30ea976d4ac02fab1025a8dece8bfae6aa83bdcfd9c0e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# `defmt-parser`\n\n`defmt` (\"de format\", short for \"deferred formatting\") is a highly efficient logging framework that targets resource-constrained devices, like microcontrollers.\n\nFor more details about the framework check the book at .\n\nThis library decodes `defmt` frames into Rust structures. It is mainly used by\n[`defmt-decoder`](https://crates.io/crates/defmt-decoder), and you should prefer\nusing that crate to this one.\n\n## MSRV\n\nThe minimum supported Rust version is 1.76 (or Ferrocene 24.05). `defmt` is tested against the latest stable Rust version and the MSRV.\n\n## Support\n\n`defmt` is part of the [Knurling] project, [Ferrous Systems]' effort at\nimproving tooling used to develop for embedded systems.\n\nIf you think that our work is useful, consider sponsoring it via [GitHub\nSponsors].\n\n## License\n\nLicensed under either of\n\n- Apache License, Version 2.0 ([LICENSE-APACHE](../LICENSE-APACHE) or\n )\n\n- MIT license ([LICENSE-MIT](../LICENSE-MIT) or )\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\nlicensed as above, without any additional terms or conditions.\n\n[Knurling]: https://knurling.ferrous-systems.com/\n[Ferrous Systems]: https://ferrous-systems.com/\n[GitHub Sponsors]: https://github.com/sponsors/knurling-rs\n" }, { "package": "env_logger@0.11.11", "sourcePath": "src/lib.rs", "sha256": "ad6c81c8ff53ef7a07a3ff4dda4cf16a55ad58cd4275df96bbab3aaf4990500c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! A simple logger that can be configured via environment variables, for use\n//! with the logging facade exposed by the [`log` crate][log-crate-url].\n//!\n//! Despite having \"env\" in its name, **`env_logger`** can also be configured by\n//! other means besides environment variables. See [the examples][gh-repo-examples]\n//! in the source repository for more approaches.\n//!\n//! By default, `env_logger` writes logs to `stderr`, but can be configured to\n//! instead write them to `stdout`.\n//!\n//! ## Example\n//!\n//! ```\n//! use log::{debug, error, log_enabled, info, Level};\n//!\n//! env_logger::init();\n//!\n//! debug!(\"this is a debug {}\", \"message\");\n//! error!(\"this is printed by default\");\n//!\n//! if log_enabled!(Level::Info) {\n//! let x = 3 * 4; // expensive computation\n//! info!(\"the answer was: {}\", x);\n//! }\n//! ```\n//!\n//! Assumes the binary is `main`:\n//!\n//! ```console\n//! $ RUST_LOG=error ./main\n//! [2017-11-09T02:12:24Z ERROR main] this is printed by default\n//! ```\n//!\n//! ```console\n//! $ RUST_LOG=info ./main\n//! [2017-11-09T02:12:24Z ERROR main] this is printed by default\n//! [2017-11-09T02:12:24Z INFO main] the answer was: 12\n//! ```\n//!\n//! ```console\n//! $ RUST_LOG=debug ./main\n//! [2017-11-09T02:12:24Z DEBUG main] this is a debug message\n//! [2017-11-09T02:12:24Z ERROR main] this is printed by default\n//! [2017-11-09T02:12:24Z INFO main] the answer was: 12\n//! ```\n//!\n//! You can also set the log level on a per module basis:\n//!\n//! ```console\n//! $ RUST_LOG=main=info ./main\n//! [2017-11-09T02:12:24Z ERROR main] this is printed by default\n//! [2017-11-09T02:12:24Z INFO main] the answer was: 12\n//! ```\n//!\n//! And enable all logging:\n//!\n//! ```console\n//! $ RUST_LOG=main ./main\n//! [2017-11-09T02:12:24Z DEBUG main] this is a debug message\n//! [2017-11-09T02:12:24Z ERROR main] this is printed by default\n//! [2017-11-09T02:12:24Z INFO main] the answer was: 12\n//! ```\n//!\n//! If the binary name contains hyphens, you will need to replace\n//! them with underscores:\n//!\n//! ```console\n//! $ RUST_LOG=my_app ./my-app\n//! [2017-11-09T02:12:24Z DEBUG my_app] this is a debug message\n//! [2017-11-09T02:12:24Z ERROR my_app] this is printed by default\n//! [2017-11-09T02:12:24Z INFO my_app] the answer was: 12\n//! ```\n//!\n//! This is because Rust modules and crates cannot contain hyphens\n//! in their name, although `cargo` continues to accept them.\n//!\n//! See the documentation for the [`log` crate][log-crate-url] for more\n//! information about its API.\n//!\n//! ## Enabling logging\n//!\n//! **By default all logging is disabled except for the `error` level**\n//!\n//! The **`RUST_LOG`** environment variable controls logging with the syntax:\n//! ```console\n//! RUST_LOG=[target][=][level][,...]\n//! ```\n//! Or in other words, its a comma-separated list of directives.\n//! Directives can filter by **target**, by **level**, or both (using `=`).\n//!\n//! For example,\n//! ```console\n//! RUST_LOG=data=debug,hardware=debug\n//! ```\n//!\n//! **target** is typically the path of the module the message\n//! in question originated from, though it can be overridden.\n//! The path is rooted in the name of the crate it was compiled for, so if\n//! your program is in a file called, for example, `hello.rs`, the path would\n//! simply be `hello`.\n//!\n//! Furthermore, the log can be filtered using prefix-search based on the\n//! specified log target.\n//!\n//! For example, `RUST_LOG=example` would match the following targets:\n//! - `example`\n//! - `example::test`\n//! - `example::test::module::submodule`\n//! - `examples::and_more_examples`\n//!\n//! When providing the crate name or a module path, explicitly specifying the\n//! log level is optional. If omitted, all logging for the item will be\n//! enabled.\n//!\n//! **level** is the maximum [`log::Level`][level-enum] to be shown and includes:\n//! - `error`\n//! - `warn`\n//! - `info`\n//! - `debug`\n//! - `trace`\n//! - `off` (pseudo level to disable all logging for the target)\n//!\n//! Logging level names are case-insensitive; e.g.,\n//! `debug`, `DEBUG`, and `dEbuG` all represent the same logging level. For\n//! consistency, our convention is to use the lower case names. Where our docs\n//! do use other forms, they do so in the context of specific examples, so you\n//! won't be surprised if you see similar usage in the wild.\n//!\n//! Some examples of valid values of `RUST_LOG` are:\n//!\n//! - `RUST_LOG=hello` turns on all logging for the `hello` module\n//! - `RUST_LOG=trace` turns on all logging for the application, regardless of its name\n//! - `RUST_LOG=TRACE` turns on all logging for the application, regardless of its name (same as previous)\n//! - `RUST_LOG=info` turns on all info logging\n//! - `RUST_LOG=INFO` turns on all info logging (same as previous)\n//! - `RUST_LOG=hello=debug` turns on debug logging for `hello`\n//! - `RUST_LOG=hello=DEBUG` turns on debug logging for `hello` (same as previous)\n//! - `RUST_LOG=hello,std::option` turns on `hello`, and std's option logging\n//! - `RUST_LOG=error,hello=warn` turn on global error logging and also warn for `hello`\n//! - `RUST_LOG=error,hello=off` turn on global error logging, but turn off logging for `hello`\n//! - `RUST_LOG=off` turns off all logging for the application\n//! - `RUST_LOG=OFF` turns off all logging for the application (same as previous)\n//!\n//! ## Filtering results\n//!\n//! A `RUST_LOG` directive may include a regex filter. The syntax is to append `/`\n//! followed by a regex. Each message is checked against the regex, and is only\n//! logged if it matches. Note that the matching is done after formatting the\n//! log string but before adding any logging meta-data. There is a single filter\n//! for all modules.\n//!\n//! Some examples:\n//!\n//! * `hello/foo` turns on all logging for the 'hello' module where the log\n//! message includes 'foo'.\n//! * `info/f.o` turns on all info logging where the log message includes 'foo',\n//! 'f1o', 'fao', etc.\n//! * `hello=debug/foo*foo` turns on debug logging for 'hello' where the log\n//! message includes 'foofoo' or 'fofoo' or 'fooooooofoo', etc.\n//! * `error,hello=warn/[0-9]scopes` turn on global error logging and also\n//! warn for hello. In both cases the log message must include a single digit\n//! number followed by 'scopes'.\n//!\n//! ## Capturing logs in tests\n//!\n//! Records logged during `cargo test` will not be captured by the test harness by default.\n//! The [`Builder::is_test`] method can be used in unit tests to ensure logs will be captured:\n//!\n//! ```test_harness\n//! #[cfg(test)]\n//! mod tests {\n//! use log::info;\n//!\n//! fn init() {\n//! let _ = env_logger::builder().is_test(true).try_init();\n//! }\n//!\n//! #[test]\n//! fn it_works() {\n//! init();\n//!\n//! info!(\"This record will be captured by `cargo test`\");\n//!\n//! assert_eq!(2, 1 + 1);\n//! }\n//! }\n//! ```\n//!\n//! Enabling test capturing comes at the expense of color and other style support\n//! and may have performance implications.\n//!\n//! ## Colors\n//!\n//! Outputting of colors and other styles can be controlled by the `RUST_LOG_STYLE`\n//! environment variable. It accepts the following [values][fmt::WriteStyle]:\n//!\n//! * `auto` (default) will attempt to print style characters, but don't force the issue.\n//! If the console isn't available on Windows, or if TERM=dumb, for example, then don't print colors.\n//! * `always` will always print style characters even if they aren't supported by the terminal.\n//! This includes emitting ANSI colors on Windows if the console API is unavailable.\n//! * `never` will never print style characters.\n//!\n//! Color may be applied in the logged message or a [custom formatter][fmt].\n//!\n//!
\n//!\n//! Logging of untrusted inputs can cause unexpected behavior as they may include ANSI escape codes which\n//! will be forwarded to the users terminal as part of \"Weaponizing ANSI Escape Sequences\".\n//!\n//! Mitigations include:\n//! - Setting `RUST_LOG_STYLE=never` to have all ANSI escape codes stripped\n//! - In the application, calling [`Builder::write_style(Never)`][Builder::write_style] to have all ANSI escape codes stripped\n//! - In the application, [stripping ANSI escape codes](https://docs.rs/anstream/latest/anstream/adapter/fn.strip_str.html)\n//! from user inputs\n//!\n//! Note: deactivating the build-time feature `color` is not a mitigation as that removes all ANSI escape code\n//! stripping from `env_logger`.\n//!\n//!
\n//!\n//! ## Tweaking the default format\n//!\n//! Parts of the default format can be excluded from the log output using the [`Builder`].\n//! The following example excludes the timestamp from the log output:\n//!\n//! ```\n//! env_logger::builder()\n//! .format_timestamp(None)\n//! .init();\n//! ```\n//!\n//! ### Stability of the default format\n//!\n//! The default format won't optimise for long-term stability, and explicitly makes no\n//! guarantees about the stability of its output across major, minor or patch version\n//! bumps during `0.x`.\n//!\n//! If you want to capture or interpret the output of `env_logger` programmatically\n//! then you should use a custom format.\n//!\n//! ### Using a custom format\n//!\n//! Custom formats can be provided as closures to the [`Builder`].\n//! These closures take a [`Formatter`][crate::fmt::Formatter] and `log::Record` as arguments:\n//!\n//! ```\n//! use std::io::Write;\n//!\n//! env_logger::builder()\n//! .format(|buf, record| {\n//! writeln!(buf, \"{}: {}\", record.level(), record.args())\n//! })\n//! .init();\n//! ```\n//!\n//! See the [`fmt`] module for more details about custom formats.\n//!\n//! ## Specifying defaults for environment variables\n//!\n//! `env_logger` can read configuration from environment variables.\n//! If these variables aren't present, the default value to use can be tweaked with the [`Env`] type.\n//! The following example defaults to log `warn` and above if the `RUST_LOG` environment variable\n//! isn't set:\n//!\n//! ```\n//! use env_logger::Env;\n//!\n//! env_logger::Builder::from_env(Env::default().default_filter_or(\"warn\")).init();\n//! ```\n//!\n//! [gh-repo-examples]: https://github.com/rust-cli/env_logger/tree/main/examples\n//! [level-enum]: https://docs.rs/log/latest/log/enum.Level.html\n//! [log-crate-url]: https://docs.rs/log\n\n#![cfg_attr(docsrs, feature(doc_cfg))]\n#![warn(clippy::print_stderr)]\n#![warn(clippy::print_stdout)]\n#![allow(clippy::test_attr_in_doctest)]\n\nmod logger;\nmod writer;\n\npub mod fmt;\n\npub use self::fmt::{Target, TimestampPrecision, WriteStyle};\npub use self::logger::*;\n\n#[doc = include_str!(\"../README.md\")]\n#[cfg(doctest)]\npub struct ReadmeDoctests;\n" }, { "package": "find-msvc-tools@0.1.13", "sourcePath": "README.md", "sha256": "5e7b2b1f5ffc1da7695d3e151a2d4d0b8f681e65a24ddf85a353d4bcfa81967a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# find-msvc-tools \n\n> This crate is maintained by the library team, primarily for use by the `cc`\n> crate and not intended for external use (except as a transitive dependency).\n> This crate may make major changes to its APIs or be deprecated without\n> warning.\n\nAn internal use library for finding windows-specific tools, reading MSVC\nversions from the registry and from COM interfaces.\n\nRefer to the [documentation](https://docs.rs/find-msvc-tools) for detailed usage\ninstructions.\n\n## License\n\nThis project is licensed under either of\n\n * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or\n https://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or\n https://opensource.org/license/mit)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution transitively\nintentionally submitted for inclusion in find-msvc-tools by you, as defined in\nthe Apache-2.0 license, shall be dual licensed as above, without any additional\nterms or conditions." }, { "package": "find-msvc-tools@0.1.13", "sourcePath": "src/com.rs", "sha256": "6706eb2a4d8d1920b66759776a78ab054b493a81dca99a90f96f9e53c0cd1852", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright © 2017 winapi-rs developers\n// Licensed under the Apache License, Version 2.0\n// or the MIT license\n// , at your option.\n// All files in the project carrying such notice may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::{\n winapi::{IUnknown, Interface},\n windows_sys::{\n CoInitializeEx, SysFreeString, SysStringLen, BSTR, COINIT_MULTITHREADED, HRESULT, S_FALSE,\n S_OK,\n },\n};\nuse std::{\n convert::TryInto,\n ffi::OsString,\n ops::Deref,\n os::windows::ffi::OsStringExt,\n ptr::{null, null_mut},\n slice::from_raw_parts,\n};\n\npub fn initialize() -> Result<(), HRESULT> {\n let err = unsafe { CoInitializeEx(null(), COINIT_MULTITHREADED.try_into().unwrap()) };\n if err != S_OK && err != S_FALSE {\n // S_FALSE just means COM is already initialized\n Err(err)\n } else {\n Ok(())\n }\n}\n\npub struct ComPtr(*mut T)\nwhere\n T: Interface;\nimpl ComPtr\nwhere\n T: Interface,\n{\n /// Creates a `ComPtr` to wrap a raw pointer.\n /// It takes ownership over the pointer which means it does __not__ call `AddRef`.\n /// `T` __must__ be a COM interface that inherits from `IUnknown`.\n pub unsafe fn from_raw(ptr: *mut T) -> ComPtr {\n assert!(!ptr.is_null());\n ComPtr(ptr)\n }\n /// For internal use only.\n fn as_unknown(&self) -> &IUnknown {\n unsafe { &*self.0.cast() }\n }\n /// Performs `QueryInterface` fun.\n pub fn cast(&self) -> Result, i32>\n where\n U: Interface,\n {\n let mut obj = null_mut();\n let err = unsafe { self.as_unknown().QueryInterface(&U::uuidof(), &mut obj) };\n if err < 0 {\n return Err(err);\n }\n Ok(unsafe { ComPtr::from_raw(obj.cast()) })\n }\n}\nimpl Deref for ComPtr\nwhere\n T: Interface,\n{\n type Target = T;\n fn deref(&self) -> &T {\n unsafe { &*self.0 }\n }\n}\nimpl Clone for ComPtr\nwhere\n T: Interface,\n{\n fn clone(&self) -> Self {\n unsafe {\n self.as_unknown().AddRef();\n ComPtr::from_raw(self.0)\n }\n }\n}\nimpl Drop for ComPtr\nwhere\n T: Interface,\n{\n fn drop(&mut self) {\n unsafe {\n self.as_unknown().Release();\n }\n }\n}\npub struct BStr(BSTR);\nimpl BStr {\n pub unsafe fn from_raw(s: BSTR) -> BStr {\n BStr(s)\n }\n pub fn to_osstring(&self) -> OsString {\n let len = unsafe { SysStringLen(self.0) };\n let slice = unsafe { from_raw_parts(self.0, len as usize) };\n OsStringExt::from_wide(slice)\n }\n}\nimpl Drop for BStr {\n fn drop(&mut self) {\n unsafe { SysFreeString(self.0) };\n }\n}\n" }, { "package": "find-msvc-tools@0.1.13", "sourcePath": "src/find_tools.rs", "sha256": "62862c642802e52167a1d916538cfd165e088bda9bce851522da4405af3b2d4b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! An internal use crate to looking for windows-specific tools:\n//! 1. On Windows host, probe the Windows Registry if needed;\n//! 2. On non-Windows host, check specified environment variables.\n\n#![allow(missing_docs)]\n#![allow(clippy::upper_case_acronyms)]\n\nuse std::{\n env,\n ffi::{OsStr, OsString},\n ops::Deref,\n path::{Path, PathBuf},\n process::Command,\n sync::Arc,\n};\n\nuse crate::Tool;\n\n/// The target provided by the user.\n#[derive(Copy, Clone, PartialEq, Eq)]\nenum TargetArch {\n X86,\n X64,\n Arm,\n Arm64,\n Arm64ec,\n}\nimpl TargetArch {\n /// Parse the `TargetArch` from a str. Returns `None` if the arch is unrecognized.\n fn new(arch: &str) -> Option {\n // NOTE: Keep up to date with docs in [`find`].\n match arch {\n \"x64\" | \"x86_64\" => Some(Self::X64),\n \"arm64\" | \"aarch64\" => Some(Self::Arm64),\n \"arm64ec\" => Some(Self::Arm64ec),\n \"x86\" | \"i686\" | \"i586\" => Some(Self::X86),\n \"arm\" | \"thumbv7a\" => Some(Self::Arm),\n _ => None,\n }\n }\n\n #[cfg(windows)]\n /// Gets the Visual Studio name for the architecture.\n fn as_vs_arch(&self) -> &'static str {\n match self {\n Self::X64 => \"x64\",\n Self::Arm64 | Self::Arm64ec => \"arm64\",\n Self::X86 => \"x86\",\n Self::Arm => \"arm\",\n }\n }\n}\n\n#[derive(Debug, Clone)]\n#[non_exhaustive]\npub enum Env {\n Owned(OsString),\n Arced(Arc),\n}\n\nimpl AsRef for Env {\n fn as_ref(&self) -> &OsStr {\n self.deref()\n }\n}\n\nimpl Deref for Env {\n type Target = OsStr;\n\n fn deref(&self) -> &Self::Target {\n match self {\n Env::Owned(os_str) => os_str,\n Env::Arced(os_str) => os_str,\n }\n }\n}\n\nimpl From for PathBuf {\n fn from(env: Env) -> Self {\n match env {\n Env::Owned(os_str) => PathBuf::from(os_str),\n Env::Arced(os_str) => PathBuf::from(os_str.deref()),\n }\n }\n}\n\npub trait EnvGetter {\n fn get_env(&self, name: &'static str) -> Option;\n}\n\nstruct StdEnvGetter;\n\nimpl EnvGetter for StdEnvGetter {\n #[allow(clippy::disallowed_methods)]\n fn get_env(&self, name: &'static str) -> Option {\n env::var_os(name).map(Env::Owned)\n }\n}\n\n/// Attempts to find a tool within an MSVC installation using the Windows\n/// registry as a point to search from.\n///\n/// The `arch_or_target` argument is the architecture or the Rust target name\n/// that the tool should work for (e.g. compile or link for). The supported\n/// architecture names are:\n/// - `\"x64\"` or `\"x86_64\"`\n/// - `\"arm64\"` or `\"aarch64\"`\n/// - `\"arm64ec\"`\n/// - `\"x86\"`, `\"i586\"` or `\"i686\"`\n/// - `\"arm\"` or `\"thumbv7a\"`\n///\n/// The `tool` argument is the tool to find. Supported tools include:\n/// - MSVC tools: `cl.exe`, `link.exe`, `lib.exe`, etc.\n/// - `MSBuild`: `msbuild.exe`\n/// - Visual Studio IDE: `devenv.exe`\n/// - Clang/LLVM tools: `clang.exe`, `clang++.exe`, `clang-*.exe`, `llvm-*.exe`, `lld.exe`, etc.\n///\n/// This function will return `None` if the tool could not be found, or it will\n/// return `Some(cmd)` which represents a command that's ready to execute the\n/// tool with the appropriate environment variables set.\n///\n/// To find MSVC tools, this function will first attempt to detect if we are\n/// running in the context of a developer command prompt, and then use the tools\n/// as found in the current `PATH`. If that fails, it will attempt to locate\n/// the newest MSVC toolset in the newest installed version of Visual Studio.\n/// To limit the search to a specific version of the MSVC toolset, set the\n/// `VCToolsVersion` environment variable to the desired version (e.g. \"14.44.35207\").\n///\n/// Note that this function always returns `None` for non-MSVC targets (if a\n/// full target name was specified).\npub fn find(arch_or_target: &str, tool: &str) -> Option {\n find_tool(arch_or_target, tool).map(|c| c.to_command())\n}\n\n/// Similar to the `find` function above, this function will attempt the same\n/// operation (finding a MSVC tool in a local install) but instead returns a\n/// `Tool` which may be introspected.\npub fn find_tool(arch_or_target: &str, tool: &str) -> Option {\n let full_arch = if let Some((full_arch, rest)) = arch_or_target.split_once(\"-\") {\n // The logic is all tailored for MSVC, if the target is not that then\n // bail out early.\n if !rest.contains(\"msvc\") {\n return None;\n }\n full_arch\n } else {\n arch_or_target\n };\n find_tool_with_env(full_arch, tool, &StdEnvGetter)\n}\n\npub fn find_tool_with_env(full_arch: &str, tool: &str, env_getter: &dyn EnvGetter) -> Option {\n // We only need the arch.\n let target = TargetArch::new(full_arch)?;\n\n // Looks like msbuild isn't located in the same location as other tools like\n // cl.exe and lib.exe.\n if tool.contains(\"msbuild\") {\n return impl_::find_msbuild(target, env_getter);\n }\n\n // Looks like devenv isn't located in the same location as other tools like\n // cl.exe and lib.exe.\n if tool.contains(\"devenv\") {\n return impl_::find_devenv(target, env_getter);\n }\n\n // Clang/LLVM isn't located in the same location as other tools like\n // cl.exe and lib.exe.\n if [\"clang\", \"lldb\", \"llvm\", \"ld\", \"lld\"]\n .iter()\n .any(|&t| tool.contains(t))\n {\n return impl_::find_llvm_tool(tool, target, env_getter);\n }\n\n // Ok, if we're here, now comes the fun part of the probing. Default shells\n // or shells like MSYS aren't really configured to execute `cl.exe` and the\n // various compiler tools shipped as part of Visual Studio. Here we try to\n // first find the relevant tool, then we also have to be sure to fill in\n // environment variables like `LIB`, `INCLUDE`, and `PATH` to ensure that\n // the tool is actually usable.\n\n impl_::find_msvc_environment(tool, target, env_getter)\n .or_else(|| impl_::find_msvc_15plus(tool, target, env_getter))\n .or_else(|| impl_::find_msvc_14(tool, target, env_getter))\n}\n\n/// An SDK found by [`find_windows_sdk`]\n#[derive(Default)]\npub struct Sdk {\n libs: Vec,\n path: Vec,\n include: Vec,\n version: String,\n}\n\nimpl Sdk {\n /// Directories containing static libraries (e.g. `.lib` files).\n pub fn libs(&self) -> impl Iterator {\n self.libs.iter().map(PathBuf::as_path)\n }\n\n /// Directories containing binaries (`.exe` and `.dll` files).\n pub fn path(&self) -> impl Iterator {\n self.path.iter().map(PathBuf::as_path)\n }\n\n /// Directories containing source headers (e.g. `.h` files)\n pub fn include(&self) -> impl Iterator {\n self.include.iter().map(PathBuf::as_path)\n }\n\n /// The Windows SDK version that was found.\n pub fn sdk_version(&self) -> &str {\n &self.version\n }\n}\n\n/// Find the directories of the latest installed Windows SDK.\n///\n/// It's not necessary to call this if using [`find`] or [`find_tool`] as they will populate the relvant\n/// environment variables with the SDK directories.\npub fn find_windows_sdk(full_arch: &str) -> Option {\n let target = TargetArch::new(full_arch)?;\n impl_::get_sdks(target, &StdEnvGetter)\n}\n\n/// A version of Visual Studio\n#[derive(Debug, PartialEq, Eq, Copy, Clone)]\n#[non_exhaustive]\npub enum VsVers {\n /// Visual Studio 12 (2013)\n #[deprecated(\n note = \"Visual Studio 12 is no longer supported. cc will never return this value.\"\n )]\n Vs12,\n /// Visual Studio 14 (2015)\n Vs14,\n /// Visual Studio 15 (2017)\n Vs15,\n /// Visual Studio 16 (2019)\n Vs16,\n /// Visual Studio 17 (2022)\n Vs17,\n /// Visual Studio 18 (2026)\n Vs18,\n}\n\n/// Find the most recent installed version of Visual Studio\n///\n/// This is used by the cmake crate to figure out the correct\n/// generator.\n#[allow(clippy::disallowed_methods)]\npub fn find_vs_version() -> Result {\n fn has_msbuild_version(version: &str) -> bool {\n impl_::has_msbuild_version(version, &StdEnvGetter)\n }\n\n match std::env::var(\"VisualStudioVersion\") {\n Ok(version) => match &version[..] {\n \"18.0\" => Ok(VsVers::Vs18),\n \"17.0\" => Ok(VsVers::Vs17),\n \"16.0\" => Ok(VsVers::Vs16),\n \"15.0\" => Ok(VsVers::Vs15),\n \"14.0\" => Ok(VsVers::Vs14),\n vers => Err(format!(\n \"\\n\\n\\\n unsupported or unknown VisualStudio version: {vers}\\n\\\n if another version is installed consider running \\\n the appropriate vcvars script before building this \\\n crate\\n\\\n \"\n )),\n },\n _ => {\n // Check for the presence of a specific registry key\n // that indicates visual studio is installed.\n if has_msbuild_version(\"18.0\") {\n Ok(VsVers::Vs18)\n } else if has_msbuild_version(\"17.0\") {\n Ok(VsVers::Vs17)\n } else if has_msbuild_version(\"16.0\") {\n Ok(VsVers::Vs16)\n } else if has_msbuild_version(\"15.0\") {\n Ok(VsVers::Vs15)\n } else if has_msbuild_version(\"14.0\") {\n Ok(VsVers::Vs14)\n } else {\n Err(\"\\n\\n\\\n couldn't determine visual studio generator\\n\\\n if VisualStudio is installed, however, consider \\\n running the appropriate vcvars script before building \\\n this crate\\n\\\n \"\n .to_string())\n }\n }\n }\n}\n\n/// To find the Universal CRT we look in a specific registry key for where\n/// all the Universal CRTs are located and then sort them asciibetically to\n/// find the newest version. While this sort of sorting isn't ideal, it is\n/// what vcvars does so that's good enough for us.\n///\n/// Returns a pair of (root, version) for the ucrt dir if found\npub fn get_ucrt_dir() -> Option<(PathBuf, String)> {\n impl_::get_ucrt_dir()\n}\n\n/// Windows Implementation.\n#[cfg(windows)]\nmod impl_ {\n use crate::com;\n use crate::registry::{RegistryKey, LOCAL_MACHINE};\n use crate::setup_config::SetupConfiguration;\n use crate::vs_instances::{VsInstances, VswhereInstance};\n use crate::windows_sys::{\n GetMachineTypeAttributes, GetProcAddress, LoadLibraryA, UserEnabled, HMODULE,\n IMAGE_FILE_MACHINE_AMD64, MACHINE_ATTRIBUTES, S_OK,\n };\n use std::convert::TryFrom;\n use std::env;\n use std::ffi::OsString;\n use std::fs::File;\n use std::io::Read;\n use std::iter;\n use std::mem;\n use std::path::{Path, PathBuf};\n use std::process::Command;\n use std::str::FromStr;\n use std::sync::atomic::{AtomicBool, Ordering};\n use std::sync::Once;\n\n use super::{EnvGetter, Sdk, TargetArch};\n use crate::Tool;\n\n struct MsvcTool {\n tool: PathBuf,\n libs: Vec,\n path: Vec,\n include: Vec,\n }\n\n struct LibraryHandle(HMODULE);\n\n impl LibraryHandle {\n fn new(name: &[u8]) -> Option {\n let handle = unsafe { LoadLibraryA(name.as_ptr().cast()) };\n (!handle.is_null()).then_some(Self(handle))\n }\n\n /// Get a function pointer to a function in the library.\n /// # SAFETY\n ///\n /// The caller must ensure that the function signature matches the actual function.\n /// The easiest way to do this is to add an entry to `windows_sys_no_link.list` and use the\n /// generated function for `func_signature`.\n ///\n /// The function returned cannot be used after the handle is dropped.\n unsafe fn get_proc_address(&self, name: &[u8]) -> Option {\n let symbol = GetProcAddress(self.0, name.as_ptr().cast());\n symbol.map(|symbol| mem::transmute_copy(&symbol))\n }\n }\n\n type GetMachineTypeAttributesFuncType =\n unsafe extern \"system\" fn(u16, *mut MACHINE_ATTRIBUTES) -> i32;\n const _: () = {\n // Ensure that our hand-written signature matches the actual function signature.\n // We can't use `GetMachineTypeAttributes` outside of a const scope otherwise we'll end up statically linking to\n // it, which will fail to load on older versions of Windows.\n let _: GetMachineTypeAttributesFuncType = GetMachineTypeAttributes;\n };\n\n fn is_amd64_emulation_supported_inner() -> Option {\n // GetMachineTypeAttributes is only available on Win11 22000+, so dynamically load it.\n let kernel32 = LibraryHandle::new(b\"kernel32.dll\\0\")?;\n // SAFETY: GetMachineTypeAttributesFuncType is checked to match the real function signature.\n let get_machine_type_attributes = unsafe {\n kernel32\n .get_proc_address::(b\"GetMachineTypeAttributes\\0\")\n }?;\n let mut attributes = Default::default();\n if unsafe {\n get_machine_type_attributes(\n IMAGE_FILE_MACHINE_AMD64.try_into().unwrap(),\n &mut attributes,\n )\n } == S_OK\n {\n Some((attributes & UserEnabled) != 0)\n } else {\n Some(false)\n }\n }\n\n fn is_amd64_emulation_supported() -> bool {\n // TODO: Replace with a OnceLock once MSRV is 1.70.\n static LOAD_VALUE: Once = Once::new();\n static IS_SUPPORTED: AtomicBool = AtomicBool::new(false);\n\n // Using Relaxed ordering since the Once is providing synchronization.\n LOAD_VALUE.call_once(|| {\n IS_SUPPORTED.store(\n is_amd64_emulation_supported_inner().unwrap_or(false),\n Ordering::Relaxed,\n );\n });\n IS_SUPPORTED.load(Ordering::Relaxed)\n }\n\n impl MsvcTool {\n fn new(tool: PathBuf) -> MsvcTool {\n MsvcTool {\n tool,\n libs: Vec::new(),\n path: Vec::new(),\n include: Vec::new(),\n }\n }\n\n fn add_sdk(&mut self, sdk_info: Sdk) {\n self.libs.extend(sdk_info.libs);\n self.path.extend(sdk_info.path);\n self.include.extend(sdk_info.include);\n }\n\n fn into_tool(self, env_getter: &dyn EnvGetter) -> Tool {\n let MsvcTool {\n tool,\n libs,\n path,\n include,\n } = self;\n let mut tool = Tool {\n tool,\n is_clang_cl: false,\n env: Vec::new(),\n };\n add_env(&mut tool, \"LIB\", libs, env_getter);\n add_env(&mut tool, \"PATH\", path, env_getter);\n add_env(&mut tool, \"INCLUDE\", include, env_getter);\n tool\n }\n }\n\n impl Sdk {\n fn find_tool(&self, tool: &str) -> Option {\n self.path.iter().map(|p| p.join(tool)).find_map(|mut p| {\n (p.exists() || (p.set_extension(env::consts::EXE_SUFFIX) && p.exists()))\n .then_some(p)\n })\n }\n }\n\n /// Checks to see if the target's arch matches the VS environment. Returns `None` if the\n /// environment is unknown.\n fn is_vscmd_target(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n is_vscmd_target_env(target, env_getter).or_else(|| is_vscmd_target_cl(target, env_getter))\n }\n\n /// Checks to see if the `VSCMD_ARG_TGT_ARCH` environment variable matches the\n /// given target's arch. Returns `None` if the variable does not exist.\n fn is_vscmd_target_env(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n let vscmd_arch = env_getter.get_env(\"VSCMD_ARG_TGT_ARCH\")?;\n Some(target.as_vs_arch() == vscmd_arch.as_ref())\n }\n\n /// Checks if the cl.exe target matches the given target's arch. Returns `None` if anything\n /// fails.\n fn is_vscmd_target_cl(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n let cmd_target = vscmd_target_cl(env_getter)?;\n Some(target.as_vs_arch() == cmd_target)\n }\n\n /// Detect the target architecture of `cl.exe` in the current path, and return `None` if this\n /// fails for any reason.\n fn vscmd_target_cl(env_getter: &dyn EnvGetter) -> Option<&'static str> {\n let cl_exe = env_getter.get_env(\"PATH\").and_then(|path| {\n env::split_paths(&path)\n .map(|p| p.join(\"cl.exe\"))\n .find(|p| p.exists())\n })?;\n let mut cl = Command::new(cl_exe);\n cl.stderr(std::process::Stdio::piped())\n .stdout(std::process::Stdio::null());\n\n let out = cl.output().ok()?;\n let cl_arch = out\n .stderr\n .split(|&b| b == b'\\n' || b == b'\\r')\n .next()?\n .rsplit(|&b| b == b' ')\n .next()?;\n\n match cl_arch {\n b\"x64\" => Some(\"x64\"),\n b\"x86\" => Some(\"x86\"),\n b\"ARM64\" => Some(\"arm64\"),\n b\"ARM\" => Some(\"arm\"),\n _ => None,\n }\n }\n\n /// Attempt to find the tool using environment variables set by vcvars.\n pub(super) fn find_msvc_environment(\n tool: &str,\n target: TargetArch,\n env_getter: &dyn EnvGetter,\n ) -> Option {\n // Early return if the environment isn't one that is known to have compiler toolsets in PATH\n // `VCINSTALLDIR` is set from vcvarsall.bat (developer command prompt)\n // `VSTEL_MSBuildProjectFullPath` is set by msbuild when invoking custom build steps\n // NOTE: `VisualStudioDir` used to be used but this isn't set when invoking msbuild from the commandline\n if env_getter.get_env(\"VCINSTALLDIR\").is_none()\n && env_getter.get_env(\"VSTEL_MSBuildProjectFullPath\").is_none()\n {\n return None;\n }\n\n // If the vscmd target differs from the requested target then\n // attempt to get the tool using the VS install directory.\n if is_vscmd_target(target, env_getter) == Some(false) {\n // We will only get here with versions 15+.\n let vs_install_dir: PathBuf = env_getter.get_env(\"VSINSTALLDIR\")?.into();\n tool_from_vs15plus_instance(tool, target, &vs_install_dir, env_getter)\n } else {\n // Fallback to simply using the current environment.\n env_getter\n .get_env(\"PATH\")\n .and_then(|path| {\n env::split_paths(&path)\n .map(|p| p.join(tool))\n .find(|p| p.exists())\n })\n .map(|path| Tool {\n tool: path,\n is_clang_cl: false,\n env: Vec::new(),\n })\n }\n }\n\n fn find_msbuild_vs18(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n find_tool_in_vs16plus_path(r\"MSBuild\\Current\\Bin\\MSBuild.exe\", target, \"18\", env_getter)\n }\n\n fn find_msbuild_vs17(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n find_tool_in_vs16plus_path(r\"MSBuild\\Current\\Bin\\MSBuild.exe\", target, \"17\", env_getter)\n }\n\n #[allow(bare_trait_objects)]\n fn vs16plus_instances(\n target: TargetArch,\n version: &'static str,\n env_getter: &dyn EnvGetter,\n ) -> Box> {\n Box::new(\n vs15plus_instances(target, env_getter)\n .into_iter()\n .flatten()\n .filter_map(move |instance| {\n instance\n .installation_name()\n .filter(|name| {\n [\"VisualStudio\", \"VisualStudioPreview\"]\n .into_iter()\n .any(|kind| name.starts_with(&format!(\"{kind}/{version}.\")))\n })\n .and_then(|_| instance.installation_path())\n }),\n )\n }\n\n fn find_tool_in_vs16plus_path(\n tool: &str,\n target: TargetArch,\n version: &'static str,\n env_getter: &dyn EnvGetter,\n ) -> Option {\n vs16plus_instances(target, version, env_getter)\n .filter_map(|path| {\n let path = path.join(tool);\n if !path.is_file() {\n return None;\n }\n let mut tool = Tool {\n tool: path,\n is_clang_cl: false,\n env: Vec::new(),\n };\n if target == TargetArch::X64 {\n tool.env.push((\"Platform\".into(), \"X64\".into()));\n }\n if matches!(target, TargetArch::Arm64 | TargetArch::Arm64ec) {\n tool.env.push((\"Platform\".into(), \"ARM64\".into()));\n }\n Some(tool)\n })\n .next()\n }\n\n fn find_msbuild_vs16(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n find_tool_in_vs16plus_path(r\"MSBuild\\Current\\Bin\\MSBuild.exe\", target, \"16\", env_getter)\n }\n\n pub(super) fn find_llvm_tool(\n tool: &str,\n target: TargetArch,\n env_getter: &dyn EnvGetter,\n ) -> Option {\n find_llvm_tool_vs17plus(tool, target, env_getter, \"18\")\n .or_else(|| find_llvm_tool_vs17plus(tool, target, env_getter, \"17\"))\n }\n\n fn find_llvm_tool_vs17plus(\n tool: &str,\n target: TargetArch,\n env_getter: &dyn EnvGetter,\n version: &'static str,\n ) -> Option {\n vs16plus_instances(target, version, env_getter)\n .filter_map(|mut base_path| {\n base_path.push(r\"VC\\Tools\\LLVM\");\n let host_folder = match host_arch() {\n // The default LLVM bin folder is x86, and there's separate subfolders\n // for the x64 and ARM64 host tools.\n X86 => \"\",\n X86_64 => \"x64\",\n AARCH64 => \"ARM64\",\n _ => return None,\n };\n if !host_folder.is_empty() {\n // E.g. C:\\...\\VC\\Tools\\LLVM\\x64\n base_path.push(host_folder);\n }\n // E.g. C:\\...\\VC\\Tools\\LLVM\\x64\\bin\\clang.exe\n base_path.push(\"bin\");\n base_path.push(tool);\n let is_clang_cl = tool.contains(\"clang-cl\");\n base_path.is_file().then(|| Tool {\n tool: base_path,\n is_clang_cl,\n env: Vec::new(),\n })\n })\n .next()\n }\n\n // In MSVC 15 (2017) MS once again changed the scheme for locating\n // the tooling. Now we must go through some COM interfaces, which\n // is super fun for Rust.\n //\n // Note that much of this logic can be found [online] wrt paths, COM, etc.\n //\n // [online]: https://blogs.msdn.microsoft.com/vcblog/2017/03/06/finding-the-visual-c-compiler-tools-in-visual-studio-2017/\n //\n // Returns MSVC 15+ instances (15, 16 right now), the order should be consider undefined.\n //\n // However, on ARM64 this method doesn't work because VS Installer fails to register COM component on ARM64.\n // Hence, as the last resort we try to use vswhere.exe to list available instances.\n fn vs15plus_instances(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n vs15plus_instances_using_com()\n .or_else(|| vs15plus_instances_using_vswhere(target, env_getter))\n }\n\n fn vs15plus_instances_using_com() -> Option {\n com::initialize().ok()?;\n\n let config = SetupConfiguration::new().ok()?;\n let enum_setup_instances = config.enum_all_instances().ok()?;\n\n Some(VsInstances::ComBased(enum_setup_instances))\n }\n\n fn vs15plus_instances_using_vswhere(\n target: TargetArch,\n env_getter: &dyn EnvGetter,\n ) -> Option {\n let program_files_path = env_getter\n .get_env(\"ProgramFiles(x86)\")\n .or_else(|| env_getter.get_env(\"ProgramFiles\"))?;\n\n let program_files_path = Path::new(program_files_path.as_ref());\n\n let vswhere_path =\n program_files_path.join(r\"Microsoft Visual Studio\\Installer\\vswhere.exe\");\n\n if !vswhere_path.exists() {\n return None;\n }\n\n let tools_arch = match target {\n TargetArch::X86 | TargetArch::X64 => Some(\"x86.x64\"),\n TargetArch::Arm => Some(\"ARM\"),\n TargetArch::Arm64 | TargetArch::Arm64ec => Some(\"ARM64\"),\n };\n\n let vswhere_output = Command::new(vswhere_path)\n .args([\n \"-latest\",\n \"-products\",\n \"*\",\n \"-requires\",\n &format!(\"Microsoft.VisualStudio.Component.VC.Tools.{}\", tools_arch?),\n \"-format\",\n \"text\",\n \"-nologo\",\n ])\n .stderr(std::process::Stdio::inherit())\n .output()\n .ok()?;\n\n let vs_instances =\n VsInstances::VswhereBased(VswhereInstance::try_from(&vswhere_output.stdout).ok()?);\n\n Some(vs_instances)\n }\n\n // Inspired from official microsoft/vswhere ParseVersionString\n // i.e. at most four u16 numbers separated by '.'\n fn parse_version(version: &str) -> Option<[u16; 4]> {\n let mut iter = version.split('.').map(u16::from_str).fuse();\n let mut get_next_number = move || match iter.next() {\n Some(Ok(version_part)) => Some(version_part),\n Some(Err(_)) => None,\n None => Some(0),\n };\n Some([\n get_next_number()?,\n get_next_number()?,\n get_next_number()?,\n get_next_number()?,\n ])\n }\n\n pub(super) fn find_msvc_15plus(\n tool: &str,\n target: TargetArch,\n env_getter: &dyn EnvGetter,\n ) -> Option {\n let iter = vs15plus_instances(target, env_getter)?;\n iter.into_iter()\n .filter_map(|instance| {\n let version = parse_version(&instance.installation_version()?)?;\n let instance_path = instance.installation_path()?;\n let tool = tool_from_vs15plus_instance(tool, target, &instance_path, env_getter)?;\n Some((version, tool))\n })\n .max_by_key(|(version, _)| *version)\n .map(|(_version, tool)| tool)\n }\n\n // While the paths to Visual Studio 2017's devenv and MSBuild could\n // potentially be retrieved from the registry, finding them via\n // SetupConfiguration has shown to be [more reliable], and is preferred\n // according to Microsoft. To help head off potential regressions though,\n // we keep the registry method as a fallback option.\n //\n // [more reliable]: https://github.com/rust-lang/cc-rs/pull/331\n fn find_tool_in_vs15_path(\n tool: &str,\n target: TargetArch,\n env_getter: &dyn EnvGetter,\n ) -> Option {\n let mut path = match vs15plus_instances(target, env_getter) {\n Some(instances) => instances\n .into_iter()\n .filter_map(|instance| instance.installation_path())\n .map(|path| path.join(tool))\n .find(|path| path.is_file()),\n None => None,\n };\n\n if path.is_none() {\n let key = r\"SOFTWARE\\WOW6432Node\\Microsoft\\VisualStudio\\SxS\\VS7\";\n path = LOCAL_MACHINE\n .open(key.as_ref())\n .ok()\n .and_then(|key| key.query_str(\"15.0\").ok())\n .map(|path| PathBuf::from(path).join(tool))\n .filter(|path| path.is_file());\n }\n\n path.map(|path| {\n let mut tool = Tool {\n tool: path,\n is_clang_cl: false,\n env: Vec::new(),\n };\n if target == TargetArch::X64 {\n tool.env.push((\"Platform\".into(), \"X64\".into()));\n } else if matches!(target, TargetArch::Arm64 | TargetArch::Arm64ec) {\n tool.env.push((\"Platform\".into(), \"ARM64\".into()));\n }\n tool\n })\n }\n\n fn tool_from_vs15plus_instance(\n tool: &str,\n target: TargetArch,\n instance_path: &Path,\n env_getter: &dyn EnvGetter,\n ) -> Option {\n let (root_path, bin_path, host_dylib_path, lib_path, alt_lib_path, include_path) =\n vs15plus_vc_paths(target, instance_path, env_getter)?;\n let sdk_info = get_sdks(target, env_getter)?;\n let mut tool_path = bin_path.join(tool);\n if !tool_path.exists() {\n tool_path = sdk_info.find_tool(tool)?;\n };\n\n let mut tool = MsvcTool::new(tool_path);\n tool.path.push(bin_path.clone());\n tool.path.push(host_dylib_path);\n if let Some(alt_lib_path) = alt_lib_path {\n tool.libs.push(alt_lib_path);\n }\n tool.libs.push(lib_path);\n tool.include.push(include_path);\n\n if let Some((atl_lib_path, atl_include_path)) = atl_paths(target, &root_path) {\n tool.libs.push(atl_lib_path);\n tool.include.push(atl_include_path);\n }\n\n tool.add_sdk(sdk_info);\n\n Some(tool.into_tool(env_getter))\n }\n\n fn vs15plus_vc_paths(\n target_arch: TargetArch,\n instance_path: &Path,\n env_getter: &dyn EnvGetter,\n ) -> Option<(PathBuf, PathBuf, PathBuf, PathBuf, Option, PathBuf)> {\n let version = vs15plus_vc_read_version(instance_path, env_getter)?;\n\n let hosts = match host_arch() {\n X86 => &[\"X86\"],\n X86_64 => &[\"X64\"],\n // Starting with VS 17.4, there is a natively hosted compiler on ARM64:\n // https://devblogs.microsoft.com/visualstudio/arm64-visual-studio-is-officially-here/\n // On older versions of VS, we use x64 if running under emulation is supported,\n // otherwise use x86.\n AARCH64 => {\n if is_amd64_emulation_supported() {\n &[\"ARM64\", \"X64\", \"X86\"][..]\n } else {\n &[\"ARM64\", \"X86\"]\n }\n }\n _ => return None,\n };\n let target_dir = target_arch.as_vs_arch();\n // The directory layout here is MSVC/bin/Host$host/$target/\n let path = instance_path.join(r\"VC\\Tools\\MSVC\").join(version);\n // We use the first available host architecture that can build for the target\n let (host_path, host) = hosts.iter().find_map(|&x| {\n let candidate = path.join(\"bin\").join(format!(\"Host{}\", x));\n candidate\n .join(target_dir)\n .exists()\n .then_some((candidate, x))\n })?;\n // This is the path to the toolchain for a particular target, running\n // on a given host\n let bin_path = host_path.join(target_dir);\n // But! we also need PATH to contain the target directory for the host\n // architecture, because it contains dlls like mspdb140.dll compiled for\n // the host architecture.\n let host_dylib_path = host_path.join(host.to_lowercase());\n let lib_fragment = if use_spectre_mitigated_libs(env_getter) {\n r\"lib\\spectre\"\n } else {\n \"lib\"\n };\n let lib_path = path.join(lib_fragment).join(target_dir);\n let alt_lib_path =\n (target_arch == TargetArch::Arm64ec).then(|| path.join(lib_fragment).join(\"arm64ec\"));\n let include_path = path.join(\"include\");\n Some((\n path,\n bin_path,\n host_dylib_path,\n lib_path,\n alt_lib_path,\n include_path,\n ))\n }\n\n fn vs15plus_vc_read_version(dir: &Path, env_getter: &dyn EnvGetter) -> Option {\n if let Some(version) = env_getter.get_env(\"VCToolsVersion\") {\n // Restrict the search to a specific msvc version; if it doesn't exist then\n // our caller will fail to find the tool for this instance and move on.\n return version.to_str().map(ToString::to_string);\n }\n\n // Try to open the default version file.\n let mut version_path: PathBuf =\n dir.join(r\"VC\\Auxiliary\\Build\\Microsoft.VCToolsVersion.default.txt\");\n let mut version_file = if let Ok(f) = File::open(&version_path) {\n f\n } else {\n // If the default doesn't exist, search for other version files.\n // These are in the form Microsoft.VCToolsVersion.v143.default.txt\n // where `143` is any three decimal digit version number.\n // This sorts versions by lexical order and selects the highest version.\n let mut version_file = String::new();\n version_path.pop();\n for file in version_path.read_dir().ok()? {\n let name = file.ok()?.file_name();\n let name = name.to_str()?;\n if name.starts_with(\"Microsoft.VCToolsVersion.v\")\n && name.ends_with(\".default.txt\")\n && name > &version_file\n {\n version_file.replace_range(.., name);\n }\n }\n if version_file.is_empty() {\n // If all else fails, manually search for bin directories.\n let tools_dir: PathBuf = dir.join(r\"VC\\Tools\\MSVC\");\n return tools_dir\n .read_dir()\n .ok()?\n .filter_map(|file| {\n let file = file.ok()?;\n let name = file.file_name().into_string().ok()?;\n\n file.path().join(\"bin\").exists().then(|| {\n let version = parse_version(&name);\n (name, version)\n })\n })\n .max_by_key(|(_, version)| *version)\n .map(|(name, _)| name);\n }\n version_path.push(version_file);\n File::open(version_path).ok()?\n };\n\n // Get the version string from the file we found.\n let mut version = String::new();\n version_file.read_to_string(&mut version).ok()?;\n version.truncate(version.trim_end().len());\n Some(version)\n }\n\n fn use_spectre_mitigated_libs(env_getter: &dyn EnvGetter) -> bool {\n env_getter\n .get_env(\"VSCMD_ARG_VCVARS_SPECTRE\")\n .map(|env| env.as_ref() == \"spectre\")\n .unwrap_or_default()\n }\n\n fn atl_paths(target: TargetArch, path: &Path) -> Option<(PathBuf, PathBuf)> {\n let atl_path = path.join(\"atlmfc\");\n atl_path.exists().then(|| {\n let sub = target.as_vs_arch();\n (atl_path.join(\"lib\").join(sub), atl_path.join(\"include\"))\n })\n }\n\n // For MSVC 14 we need to find the Universal CRT as well as either\n // the Windows 10 SDK or Windows 8.1 SDK.\n pub(super) fn find_msvc_14(\n tool: &str,\n target: TargetArch,\n env_getter: &dyn EnvGetter,\n ) -> Option {\n if env_getter.get_env(\"VCToolsVersion\").is_some() {\n // VCToolsVersion is not set/supported for MSVC 14\n return None;\n }\n\n let vcdir = get_vc_dir(\"14.0\")?;\n let sdk_info = get_sdks(target, env_getter)?;\n let mut tool = get_tool(tool, &vcdir, target, &sdk_info)?;\n tool.add_sdk(sdk_info);\n Some(tool.into_tool(env_getter))\n }\n\n pub(super) fn get_sdks(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n let sub = target.as_vs_arch();\n let (ucrt, ucrt_version) = get_ucrt_dir()?;\n\n let host = match host_arch() {\n X86 => \"x86\",\n X86_64 => \"x64\",\n AARCH64 => \"arm64\",\n _ => return None,\n };\n\n let mut info = Sdk::default();\n\n info.path\n .push(ucrt.join(\"bin\").join(&ucrt_version).join(host));\n\n let ucrt_include = ucrt.join(\"include\").join(&ucrt_version);\n info.include.push(ucrt_include.join(\"ucrt\"));\n\n let ucrt_lib = ucrt.join(\"lib\").join(&ucrt_version);\n info.libs.push(ucrt_lib.join(\"ucrt\").join(sub));\n\n if let Some((sdk, version)) = get_sdk10_dir(env_getter) {\n info.path.push(sdk.join(\"bin\").join(host));\n let sdk_lib = sdk.join(\"lib\").join(&version);\n info.libs.push(sdk_lib.join(\"um\").join(sub));\n let sdk_include = sdk.join(\"include\").join(&version);\n info.include.push(sdk_include.join(\"um\"));\n info.include.push(sdk_include.join(\"cppwinrt\"));\n info.include.push(sdk_include.join(\"winrt\"));\n info.include.push(sdk_include.join(\"shared\"));\n info.version = version;\n } else if let Some(sdk) = get_sdk81_dir() {\n info.path.push(sdk.join(\"bin\").join(host));\n let sdk_lib = sdk.join(\"lib\").join(\"winv6.3\");\n info.libs.push(sdk_lib.join(\"um\").join(sub));\n let sdk_include = sdk.join(\"include\");\n info.include.push(sdk_include.join(\"um\"));\n info.include.push(sdk_include.join(\"winrt\"));\n info.include.push(sdk_include.join(\"shared\"));\n info.version = \"8.1\".into();\n }\n\n Some(info)\n }\n\n fn add_env(\n tool: &mut Tool,\n env: &'static str,\n paths: Vec,\n env_getter: &dyn EnvGetter,\n ) {\n let prev = env_getter.get_env(env);\n let prev = prev.as_ref().map(AsRef::as_ref).unwrap_or_default();\n let prev = env::split_paths(&prev);\n let new = paths.into_iter().chain(prev);\n tool.env\n .push((env.to_string().into(), env::join_paths(new).unwrap()));\n }\n\n // Given a possible MSVC installation directory, we look for the linker and\n // then add the MSVC library path.\n fn get_tool(tool: &str, path: &Path, target: TargetArch, sdk_info: &Sdk) -> Option {\n bin_subdir(target)\n .into_iter()\n .map(|(sub, host)| {\n (\n path.join(\"bin\").join(sub).join(tool),\n Some(path.join(\"bin\").join(host)),\n )\n })\n .filter(|(path, _)| path.is_file())\n .chain(iter::once_with(|| Some((sdk_info.find_tool(tool)?, None))).flatten())\n .map(|(tool_path, host)| {\n let mut tool = MsvcTool::new(tool_path);\n tool.path.extend(host);\n let sub = vc_lib_subdir(target);\n tool.libs.push(path.join(\"lib\").join(sub));\n tool.include.push(path.join(\"include\"));\n let atlmfc_path = path.join(\"atlmfc\");\n if atlmfc_path.exists() {\n tool.libs.push(atlmfc_path.join(\"lib\").join(sub));\n tool.include.push(atlmfc_path.join(\"include\"));\n }\n tool\n })\n .next()\n }\n\n // To find MSVC we look in a specific registry key for the version we are\n // trying to find.\n fn get_vc_dir(ver: &str) -> Option {\n let key = r\"SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VC7\";\n let key = LOCAL_MACHINE.open(key.as_ref()).ok()?;\n let path = key.query_str(ver).ok()?;\n Some(path.into())\n }\n\n // To find the Universal CRT we look in a specific registry key for where\n // all the Universal CRTs are located and then sort them asciibetically to\n // find the newest version. While this sort of sorting isn't ideal, it is\n // what vcvars does so that's good enough for us.\n //\n // Returns a pair of (root, version) for the ucrt dir if found\n pub(super) fn get_ucrt_dir() -> Option<(PathBuf, String)> {\n let key = r\"SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots\";\n let key = LOCAL_MACHINE.open(key.as_ref()).ok()?;\n let root = key.query_str(\"KitsRoot10\").ok()?;\n let readdir = Path::new(&root).join(\"lib\").read_dir().ok()?;\n let max_libdir = readdir\n .filter_map(|dir| dir.ok())\n .map(|dir| dir.path())\n .filter(|dir| {\n dir.components()\n .next_back()\n .and_then(|c| c.as_os_str().to_str())\n .map(|c| c.starts_with(\"10.\") && dir.join(\"ucrt\").is_dir())\n .unwrap_or(false)\n })\n .max()?;\n let version = max_libdir.components().next_back().unwrap();\n let version = version.as_os_str().to_str().unwrap().to_string();\n Some((root.into(), version))\n }\n\n // Vcvars finds the correct version of the Windows 10 SDK by looking\n // for the include `um\\Windows.h` because sometimes a given version will\n // only have UCRT bits without the rest of the SDK. Since we only care about\n // libraries and not includes, we instead look for `um\\x64\\kernel32.lib`.\n // Since the 32-bit and 64-bit libraries are always installed together we\n // only need to bother checking x64, making this code a tiny bit simpler.\n // Like we do for the Universal CRT, we sort the possibilities\n // asciibetically to find the newest one as that is what vcvars does.\n // Before doing that, we check the \"WindowsSdkDir\" and \"WindowsSDKVersion\"\n // environment variables set by vcvars to use the environment sdk version\n // if one is already configured.\n fn get_sdk10_dir(env_getter: &dyn EnvGetter) -> Option<(PathBuf, String)> {\n if let (Some(root), Some(version)) = (\n env_getter.get_env(\"WindowsSdkDir\"),\n env_getter\n .get_env(\"WindowsSDKVersion\")\n .as_ref()\n .and_then(|version| version.as_ref().to_str()),\n ) {\n return Some((\n PathBuf::from(root),\n version.trim_end_matches('\\\\').to_string(),\n ));\n }\n\n let key = r\"SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v10.0\";\n let key = LOCAL_MACHINE.open(key.as_ref()).ok()?;\n let root = key.query_str(\"InstallationFolder\").ok()?;\n let readdir = Path::new(&root).join(\"lib\").read_dir().ok()?;\n let mut dirs = readdir\n .filter_map(|dir| dir.ok())\n .map(|dir| dir.path())\n .collect::>();\n dirs.sort();\n let dir = dirs\n .into_iter()\n .rev()\n .find(|dir| dir.join(\"um\").join(\"x64\").join(\"kernel32.lib\").is_file())?;\n let version = dir.components().next_back().unwrap();\n let version = version.as_os_str().to_str().unwrap().to_string();\n Some((root.into(), version))\n }\n\n // Interestingly there are several subdirectories, `win7` `win8` and\n // `winv6.3`. Vcvars seems to only care about `winv6.3` though, so the same\n // applies to us. Note that if we were targeting kernel mode drivers\n // instead of user mode applications, we would care.\n fn get_sdk81_dir() -> Option {\n let key = r\"SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\v8.1\";\n let key = LOCAL_MACHINE.open(key.as_ref()).ok()?;\n let root = key.query_str(\"InstallationFolder\").ok()?;\n Some(root.into())\n }\n\n const PROCESSOR_ARCHITECTURE_INTEL: u16 = 0;\n const PROCESSOR_ARCHITECTURE_AMD64: u16 = 9;\n const PROCESSOR_ARCHITECTURE_ARM64: u16 = 12;\n const X86: u16 = PROCESSOR_ARCHITECTURE_INTEL;\n const X86_64: u16 = PROCESSOR_ARCHITECTURE_AMD64;\n const AARCH64: u16 = PROCESSOR_ARCHITECTURE_ARM64;\n\n // When choosing the tool to use, we have to choose the one which matches\n // the target architecture. Otherwise we end up in situations where someone\n // on 32-bit Windows is trying to cross compile to 64-bit and it tries to\n // invoke the native 64-bit compiler which won't work.\n //\n // For the return value of this function, the first member of the tuple is\n // the folder of the tool we will be invoking, while the second member is\n // the folder of the host toolchain for that tool which is essential when\n // using a cross linker. We return a Vec since on x64 there are often two\n // linkers that can target the architecture we desire. The 64-bit host\n // linker is preferred, and hence first, due to 64-bit allowing it more\n // address space to work with and potentially being faster.\n fn bin_subdir(target: TargetArch) -> Vec<(&'static str, &'static str)> {\n match (target, host_arch()) {\n (TargetArch::X86, X86) => vec![(\"\", \"\")],\n (TargetArch::X86, X86_64) => vec![(\"amd64_x86\", \"amd64\"), (\"\", \"\")],\n (TargetArch::X64, X86) => vec![(\"x86_amd64\", \"\")],\n (TargetArch::X64, X86_64) => vec![(\"amd64\", \"amd64\"), (\"x86_amd64\", \"\")],\n (TargetArch::Arm, X86) => vec![(\"x86_arm\", \"\")],\n (TargetArch::Arm, X86_64) => vec![(\"amd64_arm\", \"amd64\"), (\"x86_arm\", \"\")],\n _ => vec![],\n }\n }\n\n // MSVC's x86 libraries are not in a subfolder\n fn vc_lib_subdir(target: TargetArch) -> &'static str {\n match target {\n TargetArch::X86 => \"\",\n TargetArch::X64 => \"amd64\",\n TargetArch::Arm => \"arm\",\n TargetArch::Arm64 | TargetArch::Arm64ec => \"arm64\",\n }\n }\n\n #[allow(bad_style)]\n fn host_arch() -> u16 {\n type DWORD = u32;\n type WORD = u16;\n type LPVOID = *mut u8;\n type DWORD_PTR = usize;\n\n #[repr(C)]\n struct SYSTEM_INFO {\n wProcessorArchitecture: WORD,\n _wReserved: WORD,\n _dwPageSize: DWORD,\n _lpMinimumApplicationAddress: LPVOID,\n _lpMaximumApplicationAddress: LPVOID,\n _dwActiveProcessorMask: DWORD_PTR,\n _dwNumberOfProcessors: DWORD,\n _dwProcessorType: DWORD,\n _dwAllocationGranularity: DWORD,\n _wProcessorLevel: WORD,\n _wProcessorRevision: WORD,\n }\n\n extern \"system\" {\n fn GetNativeSystemInfo(lpSystemInfo: *mut SYSTEM_INFO);\n }\n\n unsafe {\n let mut info = mem::zeroed();\n GetNativeSystemInfo(&mut info);\n info.wProcessorArchitecture\n }\n }\n\n // Given a registry key, look at all the sub keys and find the one which has\n // the maximal numeric value.\n //\n // Returns the name of the maximal key as well as the opened maximal key.\n fn max_version(key: &RegistryKey) -> Option<(OsString, RegistryKey)> {\n let mut max_vers = 0;\n let mut max_key = None;\n for subkey in key.iter().filter_map(|k| k.ok()) {\n let val = subkey\n .to_str()\n .and_then(|s| s.trim_start_matches('v').replace('.', \"\").parse().ok());\n let Some(val) = val else { continue };\n if val > max_vers {\n if let Ok(k) = key.open(&subkey) {\n max_vers = val;\n max_key = Some((subkey, k));\n }\n }\n }\n max_key\n }\n\n #[inline(always)]\n pub(super) fn has_msbuild_version(version: &str, env_getter: &dyn EnvGetter) -> bool {\n match version {\n \"18.0\" => {\n find_msbuild_vs18(TargetArch::X64, env_getter).is_some()\n || find_msbuild_vs18(TargetArch::X86, env_getter).is_some()\n || find_msbuild_vs18(TargetArch::Arm64, env_getter).is_some()\n }\n \"17.0\" => {\n find_msbuild_vs17(TargetArch::X64, env_getter).is_some()\n || find_msbuild_vs17(TargetArch::X86, env_getter).is_some()\n || find_msbuild_vs17(TargetArch::Arm64, env_getter).is_some()\n }\n \"16.0\" => {\n find_msbuild_vs16(TargetArch::X64, env_getter).is_some()\n || find_msbuild_vs16(TargetArch::X86, env_getter).is_some()\n || find_msbuild_vs16(TargetArch::Arm64, env_getter).is_some()\n }\n \"15.0\" => {\n find_msbuild_vs15(TargetArch::X64, env_getter).is_some()\n || find_msbuild_vs15(TargetArch::X86, env_getter).is_some()\n || find_msbuild_vs15(TargetArch::Arm64, env_getter).is_some()\n }\n \"14.0\" => LOCAL_MACHINE\n .open(&OsString::from(format!(\n \"SOFTWARE\\\\Microsoft\\\\MSBuild\\\\ToolsVersions\\\\{}\",\n version\n )))\n .is_ok(),\n _ => false,\n }\n }\n\n pub(super) fn find_devenv(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n find_devenv_vs15(target, env_getter)\n }\n\n fn find_devenv_vs15(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n find_tool_in_vs15_path(r\"Common7\\IDE\\devenv.exe\", target, env_getter)\n }\n\n // see http://stackoverflow.com/questions/328017/path-to-msbuild\n pub(super) fn find_msbuild(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n // VS 15 (2017) changed how to locate msbuild\n if let Some(r) = find_msbuild_vs18(target, env_getter) {\n Some(r)\n } else if let Some(r) = find_msbuild_vs17(target, env_getter) {\n Some(r)\n } else if let Some(r) = find_msbuild_vs16(target, env_getter) {\n Some(r)\n } else if let Some(r) = find_msbuild_vs15(target, env_getter) {\n Some(r)\n } else {\n find_old_msbuild(target)\n }\n }\n\n fn find_msbuild_vs15(target: TargetArch, env_getter: &dyn EnvGetter) -> Option {\n find_tool_in_vs15_path(r\"MSBuild\\15.0\\Bin\\MSBuild.exe\", target, env_getter)\n }\n\n fn find_old_msbuild(target: TargetArch) -> Option {\n let key = r\"SOFTWARE\\Microsoft\\MSBuild\\ToolsVersions\";\n LOCAL_MACHINE\n .open(key.as_ref())\n .ok()\n .and_then(|key| {\n max_version(&key).and_then(|(_vers, key)| key.query_str(\"MSBuildToolsPath\").ok())\n })\n .map(|path| {\n let mut path = PathBuf::from(path);\n path.push(\"MSBuild.exe\");\n let mut tool = Tool {\n tool: path,\n is_clang_cl: false,\n env: Vec::new(),\n };\n if target == TargetArch::X64 {\n tool.env.push((\"Platform\".into(), \"X64\".into()));\n }\n tool\n })\n }\n\n #[cfg(test)]\n mod tests {\n use super::*;\n use std::path::Path;\n // Import the find function from the module level\n use crate::find_tools::find;\n\n fn host_arch_to_string(host_arch_value: u16) -> &'static str {\n match host_arch_value {\n X86 => \"x86\",\n X86_64 => \"x64\",\n AARCH64 => \"arm64\",\n _ => panic!(\"Unsupported host architecture: {}\", host_arch_value),\n }\n }\n\n #[test]\n fn test_find_cl_exe() {\n // Test that we can find cl.exe for common target architectures\n // and validate the correct host-target combination paths\n // This should pass on Windows CI with Visual Studio installed\n\n let target_architectures = [\"x64\", \"x86\", \"arm64\"];\n let mut found_any = false;\n\n // Determine the host architecture\n let host_arch_value = host_arch();\n let host_name = host_arch_to_string(host_arch_value);\n\n for &target_arch in &target_architectures {\n if let Some(cmd) = find(target_arch, \"cl.exe\") {\n // Verify the command looks valid\n assert!(\n !cmd.get_program().is_empty(),\n \"cl.exe program path should not be empty\"\n );\n assert!(\n Path::new(cmd.get_program()).exists(),\n \"cl.exe should exist at: {:?}\",\n cmd.get_program()\n );\n\n // Verify the path contains the correct host-target combination\n // Use case-insensitive comparison since VS IDE uses \"Hostx64\" while Build Tools use \"HostX64\"\n let path_str = cmd.get_program().to_string_lossy();\n let path_str_lower = path_str.to_lowercase();\n let expected_host_target_path =\n format!(\"\\\\bin\\\\host{host_name}\\\\{target_arch}\");\n let expected_host_target_path_unix =\n expected_host_target_path.replace(\"\\\\\", \"/\");\n\n assert!(\n path_str_lower.contains(&expected_host_target_path) || path_str_lower.contains(&expected_host_target_path_unix),\n \"cl.exe path should contain host-target combination (case-insensitive) '{}' for {} host targeting {}, but found: {}\",\n expected_host_target_path,\n host_name,\n target_arch,\n path_str\n );\n\n found_any = true;\n }\n }\n\n assert!(found_any, \"Expected to find cl.exe for at least one target architecture (x64, x86, or arm64) on Windows CI with Visual Studio installed\");\n }\n\n #[test]\n #[cfg(not(disable_clang_cl_tests))]\n fn test_find_llvm_tools() {\n // Import StdEnvGetter from the parent module\n use crate::find_tools::StdEnvGetter;\n\n // Test the actual find_llvm_tool function with various LLVM tools\n // This test assumes CI environment has Visual Studio + Clang installed\n // We test against x64 target since clang can cross-compile to any target\n let target_arch = TargetArch::new(\"x64\").expect(\"Should support x64 architecture\");\n let llvm_tools = [\"clang.exe\", \"clang++.exe\", \"lld.exe\", \"llvm-ar.exe\"];\n\n // Determine expected host-specific path based on host architecture\n let host_arch_value = host_arch();\n let expected_host_path = match host_arch_value {\n X86 => \"LLVM\\\\bin\", // x86 host\n X86_64 => \"LLVM\\\\x64\\\\bin\", // x64 host\n AARCH64 => \"LLVM\\\\ARM64\\\\bin\", // arm64 host\n _ => panic!(\"Unsupported host architecture: {}\", host_arch_value),\n };\n\n let host_name = host_arch_to_string(host_arch_value);\n\n let mut found_tools_count = 0;\n\n for &tool in &llvm_tools {\n // Test finding LLVM tools using the standard environment getter\n let env_getter = StdEnvGetter;\n let result = find_llvm_tool(tool, target_arch, &env_getter);\n\n if let Some(found_tool) = result {\n found_tools_count += 1;\n\n // Verify the found tool has a valid, non-empty path\n assert!(\n !found_tool.path().as_os_str().is_empty(),\n \"Found LLVM tool '{}' should have a non-empty path\",\n tool\n );\n\n // Verify the tool path actually exists on filesystem\n assert!(\n found_tool.path().exists(),\n \"LLVM tool '{}' path should exist: {:?}\",\n tool,\n found_tool.path()\n );\n\n // Verify the tool path contains the expected tool name\n let path_str = found_tool.path().to_string_lossy();\n assert!(\n path_str.contains(tool.trim_end_matches(\".exe\")),\n \"Tool path '{}' should contain tool name '{}'\",\n path_str,\n tool\n );\n\n // Verify it's in the correct host-specific VS LLVM directory\n assert!(\n path_str.contains(expected_host_path) || path_str.contains(&expected_host_path.replace(\"\\\\\", \"/\")),\n \"LLVM tool should be in host-specific VS LLVM directory '{}' for {} host, but found: {}\",\n expected_host_path,\n host_name,\n path_str\n );\n }\n }\n\n // On CI with VS + Clang installed, we should find at least some LLVM tools\n assert!(\n found_tools_count > 0,\n \"Expected to find at least one LLVM tool on CI with Visual Studio + Clang installed for {} host. Found: {}\",\n host_name,\n found_tools_count\n );\n }\n }\n}\n\n/// Non-Windows Implementation.\n#[cfg(not(windows))]\nmod impl_ {\n use std::{env, ffi::OsStr, path::PathBuf};\n\n use super::{EnvGetter, Sdk, TargetArch};\n use crate::Tool;\n\n /// Finding msbuild.exe tool under unix system is not currently supported.\n /// Maybe can check it using an environment variable looks like `MSBUILD_BIN`.\n #[inline(always)]\n pub(super) fn find_msbuild(_target: TargetArch, _: &dyn EnvGetter) -> Option {\n None\n }\n\n // Finding devenv.exe tool under unix system is not currently supported.\n // Maybe can check it using an environment variable looks like `DEVENV_BIN`.\n #[inline(always)]\n pub(super) fn find_devenv(_target: TargetArch, _: &dyn EnvGetter) -> Option {\n None\n }\n\n // Finding Clang/LLVM-related tools on unix systems is not currently supported.\n #[inline(always)]\n pub(super) fn find_llvm_tool(\n _tool: &str,\n _target: TargetArch,\n _: &dyn EnvGetter,\n ) -> Option {\n None\n }\n\n /// Attempt to find the tool using environment variables set by vcvars.\n pub(super) fn find_msvc_environment(\n tool: &str,\n _target: TargetArch,\n env_getter: &dyn EnvGetter,\n ) -> Option {\n // Early return if the environment doesn't contain a VC install.\n let vc_install_dir = env_getter.get_env(\"VCINSTALLDIR\")?;\n let vs_install_dir = env_getter.get_env(\"VSINSTALLDIR\")?;\n\n let get_tool = |install_dir: &OsStr| {\n env::split_paths(install_dir)\n .map(|p| p.join(tool))\n .find(|p| p.exists())\n .map(|path| Tool {\n tool: path,\n is_clang_cl: false,\n env: Vec::new(),\n })\n };\n\n // Take the path of tool for the vc install directory.\n get_tool(vc_install_dir.as_ref())\n // Take the path of tool for the vs install directory.\n .or_else(|| get_tool(vs_install_dir.as_ref()))\n // Take the path of tool for the current path environment.\n .or_else(|| {\n env_getter\n .get_env(\"PATH\")\n .as_ref()\n .map(|path| path.as_ref())\n .and_then(get_tool)\n })\n }\n\n #[inline(always)]\n pub(super) fn find_msvc_15plus(\n _tool: &str,\n _target: TargetArch,\n _: &dyn EnvGetter,\n ) -> Option {\n None\n }\n\n // For MSVC 14 we need to find the Universal CRT as well as either\n // the Windows 10 SDK or Windows 8.1 SDK.\n #[inline(always)]\n pub(super) fn find_msvc_14(\n _tool: &str,\n _target: TargetArch,\n _: &dyn EnvGetter,\n ) -> Option {\n None\n }\n\n #[inline(always)]\n pub(super) fn has_msbuild_version(_version: &str, _: &dyn EnvGetter) -> bool {\n false\n }\n\n #[inline(always)]\n pub(super) fn get_ucrt_dir() -> Option<(PathBuf, String)> {\n None\n }\n\n #[inline(always)]\n pub(super) fn get_sdks(_target: TargetArch, _env_getter: &dyn EnvGetter) -> Option {\n None\n }\n}\n" }, { "package": "find-msvc-tools@0.1.13", "sourcePath": "src/registry.rs", "sha256": "2c8f7ed919377df32ab5e55c1af438c542da8d1f6a4f7c9ebf337839d3fd96f5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::windows_sys::{\n RegCloseKey, RegEnumKeyExW, RegOpenKeyExW, RegQueryValueExW, ERROR_NO_MORE_ITEMS,\n ERROR_SUCCESS, HKEY, HKEY_LOCAL_MACHINE, KEY_READ, KEY_WOW64_32KEY, REG_SZ,\n};\nuse std::{\n ffi::{OsStr, OsString},\n io,\n ops::RangeFrom,\n os::windows::prelude::*,\n ptr::null_mut,\n};\n\n/// Must never be `HKEY_PERFORMANCE_DATA`.\npub(crate) struct RegistryKey(Repr);\n\n#[allow(clippy::upper_case_acronyms)]\ntype DWORD = u32;\n\nstruct OwnedKey(HKEY);\n\n/// Note: must not encode `HKEY_PERFORMANCE_DATA` or one of its subkeys.\nenum Repr {\n /// `HKEY_LOCAL_MACHINE`.\n LocalMachine,\n /// A subkey of `HKEY_LOCAL_MACHINE`.\n Owned(OwnedKey),\n}\n\npub struct Iter<'a> {\n idx: RangeFrom,\n key: &'a RegistryKey,\n}\n\nunsafe impl Sync for Repr {}\nunsafe impl Send for Repr {}\n\npub(crate) const LOCAL_MACHINE: RegistryKey = RegistryKey(Repr::LocalMachine);\n\nimpl RegistryKey {\n fn raw(&self) -> HKEY {\n match self.0 {\n Repr::LocalMachine => HKEY_LOCAL_MACHINE,\n Repr::Owned(ref val) => val.0,\n }\n }\n\n /// Open a sub-key of `self`.\n pub fn open(&self, key: &OsStr) -> io::Result {\n let key = key.encode_wide().chain(Some(0)).collect::>();\n let mut ret = null_mut();\n let err = unsafe {\n RegOpenKeyExW(\n self.raw(),\n key.as_ptr(),\n 0,\n (KEY_READ | KEY_WOW64_32KEY).try_into().unwrap(),\n &mut ret,\n )\n };\n if err == ERROR_SUCCESS {\n Ok(RegistryKey(Repr::Owned(OwnedKey(ret))))\n } else {\n Err(io::Error::from_raw_os_error(err as i32))\n }\n }\n\n pub fn iter(&self) -> Iter<'_> {\n Iter {\n idx: 0..,\n key: self,\n }\n }\n\n pub fn query_str(&self, name: &str) -> io::Result {\n let name: &OsStr = name.as_ref();\n let name = name.encode_wide().chain(Some(0)).collect::>();\n let mut len = 0;\n let mut kind = 0;\n unsafe {\n let err = RegQueryValueExW(\n self.raw(),\n name.as_ptr(),\n null_mut(),\n &mut kind,\n null_mut(),\n &mut len,\n );\n if err != ERROR_SUCCESS {\n return Err(io::Error::from_raw_os_error(err as i32));\n }\n if kind != REG_SZ {\n return Err(io::Error::new(\n io::ErrorKind::Other,\n \"registry key wasn't a string\",\n ));\n }\n\n // The length here is the length in bytes, but we're using wide\n // characters so we need to be sure to halve it for the length\n // passed in.\n assert!(len % 2 == 0, \"impossible wide string size: {} bytes\", len);\n let vlen = len as usize / 2;\n // Defensively initialized, see comment about\n // `HKEY_PERFORMANCE_DATA` below.\n let mut v = vec![0u16; vlen];\n let err = RegQueryValueExW(\n self.raw(),\n name.as_ptr(),\n null_mut(),\n null_mut(),\n v.as_mut_ptr().cast(),\n &mut len,\n );\n // We don't check for `ERROR_MORE_DATA` (which would if the value\n // grew between the first and second call to `RegQueryValueExW`),\n // both because it's extremely unlikely, and this is a bit more\n // defensive more defensive against weird types of registry keys.\n if err != ERROR_SUCCESS {\n return Err(io::Error::from_raw_os_error(err as i32));\n }\n // The length is allowed to change, but should still be even, as\n // well as smaller.\n assert!(len % 2 == 0, \"impossible wide string size: {} bytes\", len);\n // If the length grew but returned a success code, it *probably*\n // indicates we're `HKEY_PERFORMANCE_DATA` or a subkey(?). We\n // consider this UB, since those keys write \"undefined\" or\n // \"unpredictable\" values to len, and need to use a completely\n // different loop structure. This should be impossible (and enforce\n // it in the API to the best of our ability), but to mitigate the\n // damage we do some smoke-checks on the len, and ensure `v` has\n // been fully initialized (rather than trusting the result of\n // `RegQueryValueExW`).\n let actual_len = len as usize / 2;\n assert!(actual_len <= v.len());\n v.truncate(actual_len);\n // Some registry keys may have a terminating nul character, but\n // we're not interested in that, so chop it off if it's there.\n if !v.is_empty() && v[v.len() - 1] == 0 {\n v.pop();\n }\n Ok(OsString::from_wide(&v))\n }\n }\n}\n\nimpl Drop for OwnedKey {\n fn drop(&mut self) {\n unsafe {\n RegCloseKey(self.0);\n }\n }\n}\n\nimpl<'a> Iterator for Iter<'a> {\n type Item = io::Result;\n\n fn next(&mut self) -> Option> {\n self.idx.next().and_then(|i| unsafe {\n let mut v = Vec::with_capacity(256);\n let mut len = v.capacity() as DWORD;\n let ret = RegEnumKeyExW(\n self.key.raw(),\n i,\n v.as_mut_ptr(),\n &mut len,\n null_mut(),\n null_mut(),\n null_mut(),\n null_mut(),\n );\n if ret == ERROR_NO_MORE_ITEMS {\n None\n } else if ret != ERROR_SUCCESS {\n Some(Err(io::Error::from_raw_os_error(ret as i32)))\n } else {\n v.set_len(len as usize);\n Some(Ok(OsString::from_wide(&v)))\n }\n })\n }\n}\n" }, { "package": "find-msvc-tools@0.1.13", "sourcePath": "src/setup_config.rs", "sha256": "397bd3c353a22edc4dfa8651b97cdf20e36f70526fda78d9bbf5dced3048f6d1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright © 2017 winapi-rs developers\n// Licensed under the Apache License, Version 2.0\n// or the MIT license\n// , at your option.\n// All files in the project carrying such notice may not be copied, modified, or distributed\n// except according to those terms.\n\n#![allow(bad_style)]\n#![allow(unused)]\n\nuse crate::{\n com::{BStr, ComPtr},\n winapi::{\n IUnknown, IUnknownVtbl, Interface, LCID, LPCOLESTR, LPCWSTR, LPFILETIME, LPSAFEARRAY,\n PULONGLONG, ULONG,\n },\n windows_sys::{CoCreateInstance, BSTR, CLSCTX_ALL, HRESULT, S_FALSE},\n};\n\nuse std::{\n ffi::OsString,\n ptr::{null, null_mut},\n};\n\n// Bindings to the Setup.Configuration stuff\npub type InstanceState = u32;\n\npub const eNone: InstanceState = 0;\npub const eLocal: InstanceState = 1;\npub const eRegistered: InstanceState = 2;\npub const eNoRebootRequired: InstanceState = 4;\npub const eComplete: InstanceState = -1i32 as u32;\n\nRIDL! {#[uuid(0xb41463c3, 0x8866, 0x43b5, 0xbc, 0x33, 0x2b, 0x06, 0x76, 0xf7, 0xf4, 0x2e)]\ninterface ISetupInstance(ISetupInstanceVtbl): IUnknown(IUnknownVtbl) {\n fn GetInstanceId(\n pbstrInstanceId: *mut BSTR,\n ) -> HRESULT,\n fn GetInstallDate(\n pInstallDate: LPFILETIME,\n ) -> HRESULT,\n fn GetInstallationName(\n pbstrInstallationName: *mut BSTR,\n ) -> HRESULT,\n fn GetInstallationPath(\n pbstrInstallationPath: *mut BSTR,\n ) -> HRESULT,\n fn GetInstallationVersion(\n pbstrInstallationVersion: *mut BSTR,\n ) -> HRESULT,\n fn GetDisplayName(\n lcid: LCID,\n pbstrDisplayName: *mut BSTR,\n ) -> HRESULT,\n fn GetDescription(\n lcid: LCID,\n pbstrDescription: *mut BSTR,\n ) -> HRESULT,\n fn ResolvePath(\n pwszRelativePath: LPCOLESTR,\n pbstrAbsolutePath: *mut BSTR,\n ) -> HRESULT,\n}}\n\nRIDL! {#[uuid(0x89143c9a, 0x05af, 0x49b0, 0xb7, 0x17, 0x72, 0xe2, 0x18, 0xa2, 0x18, 0x5c)]\ninterface ISetupInstance2(ISetupInstance2Vtbl): ISetupInstance(ISetupInstanceVtbl) {\n fn GetState(\n pState: *mut InstanceState,\n ) -> HRESULT,\n fn GetPackages(\n ppsaPackages: *mut LPSAFEARRAY,\n ) -> HRESULT,\n fn GetProduct(\n ppPackage: *mut *mut ISetupPackageReference,\n ) -> HRESULT,\n fn GetProductPath(\n pbstrProductPath: *mut BSTR,\n ) -> HRESULT,\n}}\n\nRIDL! {#[uuid(0x6380bcff, 0x41d3, 0x4b2e, 0x8b, 0x2e, 0xbf, 0x8a, 0x68, 0x10, 0xc8, 0x48)]\ninterface IEnumSetupInstances(IEnumSetupInstancesVtbl): IUnknown(IUnknownVtbl) {\n fn Next(\n celt: ULONG,\n rgelt: *mut *mut ISetupInstance,\n pceltFetched: *mut ULONG,\n ) -> HRESULT,\n fn Skip(\n celt: ULONG,\n ) -> HRESULT,\n fn Reset() -> HRESULT,\n fn Clone(\n ppenum: *mut *mut IEnumSetupInstances,\n ) -> HRESULT,\n}}\n\nRIDL! {#[uuid(0x42843719, 0xdb4c, 0x46c2, 0x8e, 0x7c, 0x64, 0xf1, 0x81, 0x6e, 0xfd, 0x5b)]\ninterface ISetupConfiguration(ISetupConfigurationVtbl): IUnknown(IUnknownVtbl) {\n fn EnumInstances(\n ppEnumInstances: *mut *mut IEnumSetupInstances,\n ) -> HRESULT,\n fn GetInstanceForCurrentProcess(\n ppInstance: *mut *mut ISetupInstance,\n ) -> HRESULT,\n fn GetInstanceForPath(\n wzPath: LPCWSTR,\n ppInstance: *mut *mut ISetupInstance,\n ) -> HRESULT,\n}}\n\nRIDL! {#[uuid(0x26aab78c, 0x4a60, 0x49d6, 0xaf, 0x3b, 0x3c, 0x35, 0xbc, 0x93, 0x36, 0x5d)]\ninterface ISetupConfiguration2(ISetupConfiguration2Vtbl):\n ISetupConfiguration(ISetupConfigurationVtbl) {\n fn EnumAllInstances(\n ppEnumInstances: *mut *mut IEnumSetupInstances,\n ) -> HRESULT,\n}}\n\nRIDL! {#[uuid(0xda8d8a16, 0xb2b6, 0x4487, 0xa2, 0xf1, 0x59, 0x4c, 0xcc, 0xcd, 0x6b, 0xf5)]\ninterface ISetupPackageReference(ISetupPackageReferenceVtbl): IUnknown(IUnknownVtbl) {\n fn GetId(\n pbstrId: *mut BSTR,\n ) -> HRESULT,\n fn GetVersion(\n pbstrVersion: *mut BSTR,\n ) -> HRESULT,\n fn GetChip(\n pbstrChip: *mut BSTR,\n ) -> HRESULT,\n fn GetLanguage(\n pbstrLanguage: *mut BSTR,\n ) -> HRESULT,\n fn GetBranch(\n pbstrBranch: *mut BSTR,\n ) -> HRESULT,\n fn GetType(\n pbstrType: *mut BSTR,\n ) -> HRESULT,\n fn GetUniqueId(\n pbstrUniqueId: *mut BSTR,\n ) -> HRESULT,\n}}\n\nRIDL! {#[uuid(0x42b21b78, 0x6192, 0x463e, 0x87, 0xbf, 0xd5, 0x77, 0x83, 0x8f, 0x1d, 0x5c)]\ninterface ISetupHelper(ISetupHelperVtbl): IUnknown(IUnknownVtbl) {\n fn ParseVersion(\n pwszVersion: LPCOLESTR,\n pullVersion: PULONGLONG,\n ) -> HRESULT,\n fn ParseVersionRange(\n pwszVersionRange: LPCOLESTR,\n pullMinVersion: PULONGLONG,\n pullMaxVersion: PULONGLONG,\n ) -> HRESULT,\n}}\n\nDEFINE_GUID! {CLSID_SetupConfiguration,\n0x177f0c4a, 0x1cd3, 0x4de7, 0xa3, 0x2c, 0x71, 0xdb, 0xbb, 0x9f, 0xa3, 0x6d}\n\n// Safe wrapper around the COM interfaces\npub struct SetupConfiguration(ComPtr);\n\nimpl SetupConfiguration {\n pub fn new() -> Result {\n let mut obj = null_mut();\n let err = unsafe {\n CoCreateInstance(\n &CLSID_SetupConfiguration,\n null_mut(),\n CLSCTX_ALL.try_into().unwrap(),\n &ISetupConfiguration::uuidof(),\n &mut obj,\n )\n };\n if err < 0 {\n return Err(err);\n }\n let obj = unsafe { ComPtr::from_raw(obj.cast()) };\n Ok(SetupConfiguration(obj))\n }\n pub fn get_instance_for_current_process(&self) -> Result {\n let mut obj = null_mut();\n let err = unsafe { self.0.GetInstanceForCurrentProcess(&mut obj) };\n if err < 0 {\n return Err(err);\n }\n Ok(unsafe { SetupInstance::from_raw(obj) })\n }\n pub fn enum_instances(&self) -> Result {\n let mut obj = null_mut();\n let err = unsafe { self.0.EnumInstances(&mut obj) };\n if err < 0 {\n return Err(err);\n }\n Ok(unsafe { EnumSetupInstances::from_raw(obj) })\n }\n pub fn enum_all_instances(&self) -> Result {\n let mut obj = null_mut();\n let this = self.0.cast::()?;\n let err = unsafe { this.EnumAllInstances(&mut obj) };\n if err < 0 {\n return Err(err);\n }\n Ok(unsafe { EnumSetupInstances::from_raw(obj) })\n }\n}\n\npub struct SetupInstance(ComPtr);\n\nimpl SetupInstance {\n pub unsafe fn from_raw(obj: *mut ISetupInstance) -> SetupInstance {\n SetupInstance(ComPtr::from_raw(obj))\n }\n pub fn instance_id(&self) -> Result {\n let mut s = null();\n let err = unsafe { self.0.GetInstanceId(&mut s) };\n let bstr = unsafe { BStr::from_raw(s) };\n if err < 0 {\n return Err(err);\n }\n Ok(bstr.to_osstring())\n }\n pub fn installation_name(&self) -> Result {\n let mut s = null();\n let err = unsafe { self.0.GetInstallationName(&mut s) };\n let bstr = unsafe { BStr::from_raw(s) };\n if err < 0 {\n return Err(err);\n }\n Ok(bstr.to_osstring())\n }\n pub fn installation_path(&self) -> Result {\n let mut s = null();\n let err = unsafe { self.0.GetInstallationPath(&mut s) };\n let bstr = unsafe { BStr::from_raw(s) };\n if err < 0 {\n return Err(err);\n }\n Ok(bstr.to_osstring())\n }\n pub fn installation_version(&self) -> Result {\n let mut s = null();\n let err = unsafe { self.0.GetInstallationVersion(&mut s) };\n let bstr = unsafe { BStr::from_raw(s) };\n if err < 0 {\n return Err(err);\n }\n Ok(bstr.to_osstring())\n }\n pub fn product_path(&self) -> Result {\n let mut s = null();\n let this = self.0.cast::()?;\n let err = unsafe { this.GetProductPath(&mut s) };\n let bstr = unsafe { BStr::from_raw(s) };\n if err < 0 {\n return Err(err);\n }\n Ok(bstr.to_osstring())\n }\n}\n\npub struct EnumSetupInstances(ComPtr);\n\nimpl EnumSetupInstances {\n pub unsafe fn from_raw(obj: *mut IEnumSetupInstances) -> EnumSetupInstances {\n EnumSetupInstances(ComPtr::from_raw(obj))\n }\n}\n\nimpl Iterator for EnumSetupInstances {\n type Item = Result;\n fn next(&mut self) -> Option> {\n let mut obj = null_mut();\n let err = unsafe { self.0.Next(1, &mut obj, null_mut()) };\n if err < 0 {\n return Some(Err(err));\n }\n if err == S_FALSE {\n return None;\n }\n Some(Ok(unsafe { SetupInstance::from_raw(obj) }))\n }\n}\n" }, { "package": "find-msvc-tools@0.1.13", "sourcePath": "src/winapi.rs", "sha256": "cfb3bae7f2fa9e1d91f813e3b6b66ede7a3822787d10b716189a46f14f31f24a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright © 2015-2017 winapi-rs developers\n// Licensed under the Apache License, Version 2.0\n// or the MIT license\n// , at your option.\n// All files in the project carrying such notice may not be copied, modified, or distributed\n// except according to those terms.\n\n#![allow(bad_style, clippy::upper_case_acronyms)]\n\npub type wchar_t = u16;\n\npub use crate::windows_sys::{FILETIME, GUID, HRESULT, SAFEARRAY};\n\npub type REFIID = *const IID;\npub type IID = GUID;\npub type ULONG = core::ffi::c_ulong;\npub type DWORD = u32;\npub type LPFILETIME = *mut FILETIME;\npub type OLECHAR = WCHAR;\npub type WCHAR = wchar_t;\npub type LPCOLESTR = *const OLECHAR;\npub type LCID = DWORD;\npub type LPCWSTR = *const WCHAR;\npub type PULONGLONG = *mut ULONGLONG;\npub type ULONGLONG = u64;\n\npub trait Interface {\n fn uuidof() -> GUID;\n}\n\npub type LPSAFEARRAY = *mut SAFEARRAY;\n\nmacro_rules! DEFINE_GUID {\n (\n $name:ident, $l:expr, $w1:expr, $w2:expr,\n $b1:expr, $b2:expr, $b3:expr, $b4:expr, $b5:expr, $b6:expr, $b7:expr, $b8:expr\n ) => {\n pub const $name: $crate::winapi::GUID = $crate::winapi::GUID {\n data1: $l,\n data2: $w1,\n data3: $w2,\n data4: [$b1, $b2, $b3, $b4, $b5, $b6, $b7, $b8],\n };\n };\n}\n\nmacro_rules! RIDL {\n (#[uuid($($uuid:expr),+)]\n interface $interface:ident ($vtbl:ident) {$(\n fn $method:ident($($p:ident : $t:ty,)*) -> $rtr:ty,\n )+}) => (\n #[repr(C)]\n pub struct $vtbl {\n $(pub $method: unsafe extern \"system\" fn(\n This: *mut $interface,\n $($p: $t),*\n ) -> $rtr,)+\n }\n #[repr(C)]\n pub struct $interface {\n pub lpVtbl: *const $vtbl,\n }\n RIDL!{@impl $interface {$(fn $method($($p: $t,)*) -> $rtr,)+}}\n RIDL!{@uuid $interface $($uuid),+}\n );\n (#[uuid($($uuid:expr),+)]\n interface $interface:ident ($vtbl:ident) : $pinterface:ident ($pvtbl:ident) {\n }) => (\n #[repr(C)]\n pub struct $vtbl {\n pub parent: $pvtbl,\n }\n #[repr(C)]\n pub struct $interface {\n pub lpVtbl: *const $vtbl,\n }\n RIDL!{@deref $interface $pinterface}\n RIDL!{@uuid $interface $($uuid),+}\n );\n (#[uuid($($uuid:expr),+)]\n interface $interface:ident ($vtbl:ident) : $pinterface:ident ($pvtbl:ident) {$(\n fn $method:ident($($p:ident : $t:ty,)*) -> $rtr:ty,\n )+}) => (\n #[repr(C)]\n pub struct $vtbl {\n pub parent: $pvtbl,\n $(pub $method: unsafe extern \"system\" fn(\n This: *mut $interface,\n $($p: $t,)*\n ) -> $rtr,)+\n }\n #[repr(C)]\n pub struct $interface {\n pub lpVtbl: *const $vtbl,\n }\n RIDL!{@impl $interface {$(fn $method($($p: $t,)*) -> $rtr,)+}}\n RIDL!{@deref $interface $pinterface}\n RIDL!{@uuid $interface $($uuid),+}\n );\n (@deref $interface:ident $pinterface:ident) => (\n impl ::std::ops::Deref for $interface {\n type Target = $pinterface;\n #[inline]\n fn deref(&self) -> &$pinterface {\n unsafe { &*(self as *const Self).cast() }\n }\n }\n );\n (@impl $interface:ident {$(\n fn $method:ident($($p:ident : $t:ty,)*) -> $rtr:ty,\n )+}) => (\n impl $interface {\n $(#[inline] pub unsafe fn $method(&self, $($p: $t,)*) -> $rtr {\n ((*self.lpVtbl).$method)(\n (self as *const Self).cast_mut(),\n $($p,)*\n )\n })+\n }\n );\n (@uuid $interface:ident\n $l:expr, $w1:expr, $w2:expr,\n $b1:expr, $b2:expr, $b3:expr, $b4:expr, $b5:expr, $b6:expr, $b7:expr, $b8:expr\n ) => (\n impl $crate::winapi::Interface for $interface {\n #[inline]\n fn uuidof() -> $crate::winapi::GUID {\n $crate::winapi::GUID {\n data1: $l,\n data2: $w1,\n data3: $w2,\n data4: [$b1, $b2, $b3, $b4, $b5, $b6, $b7, $b8],\n }\n }\n }\n );\n}\n\nRIDL! {#[uuid(0x00000000, 0x0000, 0x0000, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46)]\ninterface IUnknown(IUnknownVtbl) {\n fn QueryInterface(\n riid: REFIID,\n ppvObject: *mut *mut core::ffi::c_void,\n ) -> HRESULT,\n fn AddRef() -> ULONG,\n fn Release() -> ULONG,\n}}\n" }, { "package": "flate2@1.1.10", "sourcePath": "README.md", "sha256": "f14e438f37a22eb8b81a1160b7dd15b8e37a2fdbcb32e729e00b9e639b370a66", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# flate2\n\n[![Crates.io](https://img.shields.io/crates/v/flate2.svg?maxAge=2592000)](https://crates.io/crates/flate2)\n[![Documentation](https://docs.rs/flate2/badge.svg)](https://docs.rs/flate2)\n\nA streaming compression/decompression library DEFLATE-based streams in Rust.\n\nThis crate by default uses the `miniz_oxide` crate, a port of `miniz.c` to pure\nRust. This crate also supports other [backends](#backends), such as the widely\navailable zlib library or the high-performance zlib-ng library.\n\nSupported formats:\n\n* deflate\n* zlib\n* gzip\n\n```toml\n# Cargo.toml\n[dependencies]\nflate2 = \"1.0\"\n```\n\n## MSRV (Minimum Supported Rust Version) Policy\n\nThis crate supports the current and previous stable versions of the Rust compiler.\nFor example, if the current stable is 1.80, this crate supports 1.80 and 1.79.\n\nOther compiler versions may work, but failures may not be treated as a `flate2` bug.\n\nThe `Cargo.toml` file specifies a `rust-version` for which builds of the current version\npassed at some point. This value is indicative only, and may change at any time.\n\nThe `rust-version` is a best-effort measured value and is different to the MSRV. The\n`rust-version` can be incremented by a PR in order to pass tests, as long as the MSRV\ncontinues to hold. When the `rust-version` increases, the next release should be a minor\nversion, to allow any affected users to pin to a previous minor version.\n\n## Compression\n\n```rust\nuse std::io::prelude::*;\nuse flate2::Compression;\nuse flate2::write::ZlibEncoder;\n\nfn main() {\n let mut e = ZlibEncoder::new(Vec::new(), Compression::default());\n e.write_all(b\"foo\");\n e.write_all(b\"bar\");\n let compressed_bytes = e.finish();\n}\n```\n\n## Decompression\n\n```rust,no_run\nuse std::io::prelude::*;\nuse flate2::read::GzDecoder;\n\nfn main() {\n let mut d = GzDecoder::new(\"...\".as_bytes());\n let mut s = String::new();\n d.read_to_string(&mut s).unwrap();\n println!(\"{}\", s);\n}\n```\n\n## Backends\n\nThe default `miniz_oxide` backend has the advantage of only using safe Rust.\n\nIf you want maximum performance while still benefiting from a Rust\nimplementation at the cost of some `unsafe`, you can use `zlib-rs`:\n\n```toml\n[dependencies]\nflate2 = { version = \"1.0.17\", features = [\"zlib-rs\"], default-features = false }\n```\n\n### C backends\n\nWhile zlib-rs is [the fastest overall](https://trifectatech.org/blog/zlib-rs-is-faster-than-c/),\nthe zlib-ng C library can be slightly faster in certain cases:\n\n```toml\n[dependencies]\nflate2 = { version = \"1.0.17\", features = [\"zlib-ng\"], default-features = false }\n```\n\nNote that the `\"zlib-ng\"` feature works even if some other part of your crate\ngraph depends on zlib.\n\nHowever, if you're already using another C or Rust library that depends on\nzlib, and you want to avoid including both zlib and zlib-ng, you can use that\nfor Rust code as well:\n\n```toml\n[dependencies]\nflate2 = { version = \"1.0.17\", features = [\"zlib\"], default-features = false }\n```\n\nOr, if you have C or Rust code that depends on zlib and you want to use zlib-ng\nvia libz-sys in zlib-compat mode, use:\n\n```toml\n[dependencies]\nflate2 = { version = \"1.0.17\", features = [\"zlib-ng-compat\"], default-features = false }\n```\n\nNote that when using the `\"zlib-ng-compat\"` feature, if any crate in your\ndependency graph explicitly requests stock zlib, or uses libz-sys directly\nwithout `default-features = false`, you'll get stock zlib rather than zlib-ng.\nSee [the libz-sys\nREADME](https://github.com/rust-lang/libz-sys/blob/main/README.md) for details.\nTo avoid that, use the `\"zlib-ng\"` feature instead.\n\n# License\n\nThis project is licensed under either of\n\n * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or\n https://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or\n https://opensource.org/licenses/MIT)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this project by you, as defined in the Apache-2.0 license,\nshall be dual licensed as above, without any additional terms or conditions.\n" }, { "package": "flate2@1.1.10", "sourcePath": "src/bufreader.rs", "sha256": "20ffc9cb16f0b01ae50be49aaf33f3592e2805b7d6bd76b7ea04da72c428d281", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2013 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// .\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::io;\nuse crate::io::{BufRead, Read};\nuse alloc::boxed::Box;\nuse alloc::vec::Vec;\nuse core::cmp;\nuse core::mem;\n\npub struct BufReader {\n inner: R,\n buf: Box<[u8]>,\n pos: usize,\n cap: usize,\n}\n\nimpl ::core::fmt::Debug for BufReader\nwhere\n R: ::core::fmt::Debug,\n{\n fn fmt(&self, fmt: &mut ::core::fmt::Formatter) -> Result<(), ::core::fmt::Error> {\n fmt.debug_struct(\"BufReader\")\n .field(\"reader\", &self.inner)\n .field(\n \"buffer\",\n &format_args!(\"{}/{}\", self.cap - self.pos, self.buf.len()),\n )\n .finish()\n }\n}\n\nimpl BufReader {\n pub fn new(inner: R) -> BufReader {\n BufReader::with_buf(vec![0; 32 * 1024], inner)\n }\n\n pub fn with_buf(buf: Vec, inner: R) -> BufReader {\n BufReader {\n inner,\n buf: buf.into_boxed_slice(),\n pos: 0,\n cap: 0,\n }\n }\n}\n\nimpl BufReader {\n pub fn get_ref(&self) -> &R {\n &self.inner\n }\n\n pub fn get_mut(&mut self) -> &mut R {\n &mut self.inner\n }\n\n pub fn into_inner(self) -> R {\n self.inner\n }\n\n pub fn reset(&mut self, inner: R) -> R {\n self.pos = 0;\n self.cap = 0;\n mem::replace(&mut self.inner, inner)\n }\n}\n\nimpl Read for BufReader {\n fn read(&mut self, buf: &mut [u8]) -> io::Result {\n // If we don't have any buffered data and we're doing a massive read\n // (larger than our internal buffer), bypass our internal buffer\n // entirely.\n if self.pos == self.cap && buf.len() >= self.buf.len() {\n return self.inner.read(buf);\n }\n let nread = {\n let mut rem = self.fill_buf()?;\n rem.read(buf)?\n };\n self.consume(nread);\n Ok(nread)\n }\n}\n\nimpl BufRead for BufReader {\n fn fill_buf(&mut self) -> io::Result<&[u8]> {\n // If we've reached the end of our internal buffer then we need to fetch\n // some more data from the underlying reader.\n if self.pos == self.cap {\n self.cap = self.inner.read(&mut self.buf)?;\n self.pos = 0;\n }\n Ok(&self.buf[self.pos..self.cap])\n }\n\n fn consume(&mut self, amt: usize) {\n self.pos = cmp::min(self.pos + amt, self.cap);\n }\n}\n" }, { "package": "float-cmp@0.9.0", "sourcePath": "src/eq.rs", "sha256": "63978ad73d7144c3371491fd459d7e2ce9bdb672d121d9cb900556c5efd4cf92", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014-2020 Optimal Computing (NZ) Ltd.\n// Licensed under the MIT license. See LICENSE for details.\n\nuse core::{f32, f64};\n#[cfg(feature = \"num-traits\")]\n#[allow(unused_imports)]\nuse num_traits::float::FloatCore;\nuse super::Ulps;\n\n/// A trait for approximate equality comparisons.\npub trait ApproxEq: Sized {\n /// This type type defines a margin within which two values are to be\n /// considered approximately equal. It must implement `Default` so that\n /// `approx_eq()` can be called on unknown types.\n type Margin: Copy + Default;\n\n /// This method tests that the `self` and `other` values are equal within `margin`\n /// of each other.\n fn approx_eq>(self, other: Self, margin: M) -> bool;\n\n /// This method tests that the `self` and `other` values are not within `margin`\n /// of each other.\n fn approx_ne>(self, other: Self, margin: M) -> bool {\n !self.approx_eq(other, margin)\n }\n}\n\n/// This type defines a margin within two `f32` values might be considered equal,\n/// and is intended as the associated type for the `ApproxEq` trait.\n///\n/// Two tests are used to determine approximate equality.\n///\n/// The first test considers two values approximately equal if they differ by <=\n/// `epsilon`. This will only succeed for very small numbers. Note that it may\n/// succeed even if the parameters are of differing signs, straddling zero.\n///\n/// The second test considers how many ULPs (units of least precision, units in\n/// the last place, which is the integer number of floating-point representations\n/// that the parameters are separated by) different the parameters are and considers\n/// them approximately equal if this is <= `ulps`. For large floating-point numbers,\n/// an ULP can be a rather large gap, but this kind of comparison is necessary\n/// because floating-point operations must round to the nearest representable value\n/// and so larger floating-point values accumulate larger errors.\n#[repr(C)]\n#[derive(Debug, Clone, Copy)]\npub struct F32Margin {\n pub epsilon: f32,\n pub ulps: i32\n}\nimpl Default for F32Margin {\n #[inline]\n fn default() -> F32Margin {\n F32Margin {\n epsilon: f32::EPSILON,\n ulps: 4\n }\n }\n}\nimpl F32Margin {\n #[inline]\n pub fn zero() -> F32Margin {\n F32Margin {\n epsilon: 0.0,\n ulps: 0\n }\n }\n pub fn epsilon(self, epsilon: f32) -> Self {\n F32Margin {\n epsilon,\n ..self\n }\n }\n pub fn ulps(self, ulps: i32) -> Self {\n F32Margin {\n ulps,\n ..self\n }\n }\n}\nimpl From<(f32, i32)> for F32Margin {\n fn from(m: (f32, i32)) -> F32Margin {\n F32Margin {\n epsilon: m.0,\n ulps: m.1\n }\n }\n}\n\nimpl ApproxEq for f32 {\n type Margin = F32Margin;\n\n fn approx_eq>(self, other: f32, margin: M) -> bool {\n let margin = margin.into();\n\n // Check for exact equality first. This is often true, and so we get the\n // performance benefit of only doing one compare in most cases.\n self==other ||\n\n // Perform epsilon comparison next\n ((self - other).abs() <= margin.epsilon) ||\n\n {\n // Perform ulps comparion last\n let diff: i32 = self.ulps(&other);\n saturating_abs_i32!(diff) <= margin.ulps\n }\n }\n}\n\n#[test]\nfn f32_approx_eq_test1() {\n let f: f32 = 0.0_f32;\n let g: f32 = -0.0000000000000005551115123125783_f32;\n assert!(f != g); // Should not be directly equal\n assert!(f.approx_eq(g, (f32::EPSILON, 0)) == true);\n}\n#[test]\nfn f32_approx_eq_test2() {\n let f: f32 = 0.0_f32;\n let g: f32 = -0.0_f32;\n assert!(f.approx_eq(g, (f32::EPSILON, 0)) == true);\n}\n#[test]\nfn f32_approx_eq_test3() {\n let f: f32 = 0.0_f32;\n let g: f32 = 0.00000000000000001_f32;\n assert!(f.approx_eq(g, (f32::EPSILON, 0)) == true);\n}\n#[test]\nfn f32_approx_eq_test4() {\n let f: f32 = 0.00001_f32;\n let g: f32 = 0.00000000000000001_f32;\n assert!(f.approx_eq(g, (f32::EPSILON, 0)) == false);\n}\n#[test]\nfn f32_approx_eq_test5() {\n let f: f32 = 0.1_f32;\n let mut sum: f32 = 0.0_f32;\n for _ in 0_isize..10_isize { sum += f; }\n let product: f32 = f * 10.0_f32;\n assert!(sum != product); // Should not be directly equal:\n assert!(sum.approx_eq(product, (f32::EPSILON, 1)) == true);\n assert!(sum.approx_eq(product, F32Margin::zero()) == false);\n}\n#[test]\nfn f32_approx_eq_test6() {\n let x: f32 = 1000000_f32;\n let y: f32 = 1000000.1_f32;\n assert!(x != y); // Should not be directly equal\n assert!(x.approx_eq(y, (0.0, 2)) == true); // 2 ulps does it\n // epsilon method no good here:\n assert!(x.approx_eq(y, (1000.0 * f32::EPSILON, 0)) == false);\n}\n\n/// This type defines a margin within two `f64` values might be considered equal,\n/// and is intended as the associated type for the `ApproxEq` trait.\n///\n/// Two tests are used to determine approximate equality.\n///\n/// The first test considers two values approximately equal if they differ by <=\n/// `epsilon`. This will only succeed for very small numbers. Note that it may\n/// succeed even if the parameters are of differing signs, straddling zero.\n///\n/// The second test considers how many ULPs (units of least precision, units in\n/// the last place, which is the integer number of floating-point representations\n/// that the parameters are separated by) different the parameters are and considers\n/// them approximately equal if this is <= `ulps`. For large floating-point numbers,\n/// an ULP can be a rather large gap, but this kind of comparison is necessary\n/// because floating-point operations must round to the nearest representable value\n/// and so larger floating-point values accumulate larger errors.\n#[derive(Debug, Clone, Copy)]\npub struct F64Margin {\n pub epsilon: f64,\n pub ulps: i64\n}\nimpl Default for F64Margin {\n #[inline]\n fn default() -> F64Margin {\n F64Margin {\n epsilon: f64::EPSILON,\n ulps: 4\n }\n }\n}\nimpl F64Margin {\n #[inline]\n pub fn zero() -> F64Margin {\n F64Margin {\n epsilon: 0.0,\n ulps: 0\n }\n }\n pub fn epsilon(self, epsilon: f64) -> Self {\n F64Margin {\n epsilon,\n ..self\n }\n }\n pub fn ulps(self, ulps: i64) -> Self {\n F64Margin {\n ulps,\n ..self\n }\n }\n}\nimpl From<(f64, i64)> for F64Margin {\n fn from(m: (f64, i64)) -> F64Margin {\n F64Margin {\n epsilon: m.0,\n ulps: m.1\n }\n }\n}\n\nimpl ApproxEq for f64 {\n type Margin = F64Margin;\n\n fn approx_eq>(self, other: f64, margin: M) -> bool {\n let margin = margin.into();\n\n // Check for exact equality first. This is often true, and so we get the\n // performance benefit of only doing one compare in most cases.\n self == other ||\n\n // Perform epsilon comparison next\n ((self - other).abs() <= margin.epsilon) ||\n\n {\n // Perform ulps comparion last\n let diff: i64 = self.ulps(&other);\n saturating_abs_i64!(diff) <= margin.ulps\n }\n }\n}\n\n#[test]\nfn f64_approx_eq_test1() {\n let f: f64 = 0.0_f64;\n let g: f64 = -0.0000000000000005551115123125783_f64;\n assert!(f != g); // Should not be precisely equal.\n assert!(f.approx_eq(g, (3.0 * f64::EPSILON, 0)) == true); // 3e is enough.\n // ULPs test won't ever call these equal.\n}\n#[test]\nfn f64_approx_eq_test2() {\n let f: f64 = 0.0_f64;\n let g: f64 = -0.0_f64;\n assert!(f.approx_eq(g, (f64::EPSILON, 0)) == true);\n}\n#[test]\nfn f64_approx_eq_test3() {\n let f: f64 = 0.0_f64;\n let g: f64 = 1e-17_f64;\n assert!(f.approx_eq(g, (f64::EPSILON, 0)) == true);\n}\n#[test]\nfn f64_approx_eq_test4() {\n let f: f64 = 0.00001_f64;\n let g: f64 = 0.00000000000000001_f64;\n assert!(f.approx_eq(g, (f64::EPSILON, 0)) == false);\n}\n#[test]\nfn f64_approx_eq_test5() {\n let f: f64 = 0.1_f64;\n let mut sum: f64 = 0.0_f64;\n for _ in 0_isize..10_isize { sum += f; }\n let product: f64 = f * 10.0_f64;\n assert!(sum != product); // Should not be precisely equaly.\n assert!(sum.approx_eq(product, (f64::EPSILON, 0)) == true);\n assert!(sum.approx_eq(product, (0.0, 1)) == true);\n}\n#[test]\nfn f64_approx_eq_test6() {\n let x: f64 = 1000000_f64;\n let y: f64 = 1000000.0000000003_f64;\n assert!(x != y); // Should not be precisely equal.\n assert!(x.approx_eq(y, (0.0, 3)) == true);\n}\n#[test]\nfn f64_code_triggering_issue_20() {\n assert_eq!((-25.0f64).approx_eq(25.0, (0.00390625, 1)), false);\n}\n\nimpl ApproxEq for &[T]\nwhere T: Copy + ApproxEq {\n type Margin = ::Margin;\n\n fn approx_eq>(self, other: Self, margin: M) -> bool {\n let margin = margin.into();\n if self.len() != other.len() { return false; }\n self.iter().zip(other.iter()).all(|(a,b)| {\n a.approx_eq(*b, margin)\n })\n }\n}\n\n#[test]\nfn test_slices() {\n assert!( [1.33, 2.4, 2.5].approx_eq(&[1.33, 2.4, 2.5], (0.0, 0_i64)) );\n assert!( ! [1.33, 2.4, 2.6].approx_eq(&[1.33, 2.4, 2.5], (0.0, 0_i64)) );\n assert!( ! [1.33, 2.4].approx_eq(&[1.33, 2.4, 2.5], (0.0, 0_i64)) );\n assert!( ! [1.33, 2.4, 2.5].approx_eq(&[1.33, 2.4], (0.0, 0_i64)) );\n}\n\n" }, { "package": "float-cmp@0.9.0", "sourcePath": "src/lib.rs", "sha256": "0be2c16ab1ca344bf8448e983e91b3ae41507d2e90e390433832ea83fdafb404", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014-2020 Optimal Computing (NZ) Ltd.\n// Licensed under the MIT license. See LICENSE for details.\n\n//! # float-cmp\n//!\n//! float-cmp defines and implements traits for approximate comparison of floating point types\n//! which have fallen away from exact equality due to the limited precision available within\n//! floating point representations. Implementations of these traits are provided for `f32`\n//! and `f64` types.\n//!\n//! When I was a kid in the '80s, the programming rule was \"Never compare floating point\n//! numbers\". If you can follow that rule and still get the outcome you desire, then more\n//! power to you. However, if you really do need to compare them, this crate provides a\n//! reasonable way to do so.\n//!\n//! Another crate `efloat` offers another solution by providing a floating point type that\n//! tracks its error bounds as operations are performed on it, and thus can implement the\n//! `ApproxEq` trait in this crate more accurately, without specifying a `Margin`.\n//!\n//! The recommended go-to solution (although it may not be appropriate in all cases) is the\n//! `approx_eq()` function in the `ApproxEq` trait (or better yet, the macros). For `f32`\n//! and `f64`, the `F32Margin` and `F64Margin` types are provided for specifying margins as\n//! both an epsilon value and an ULPs value, and defaults are provided via `Default`\n//! (although there is no perfect default value that is always appropriate, so beware).\n//!\n//! Several other traits are provided including `Ulps`, `ApproxEqUlps`, `ApproxOrdUlps`, and\n//! `ApproxEqRatio`.\n//!\n//! ## The problem\n//!\n//! Floating point operations must round answers to the nearest representable number. Multiple\n//! operations may result in an answer different from what you expect. In the following example,\n//! the assert will fail, even though the printed output says \"0.45 == 0.45\":\n//!\n//! ```should_panic\n//! # extern crate float_cmp;\n//! # use float_cmp::ApproxEq;\n//! # fn main() {\n//! let a: f32 = 0.15 + 0.15 + 0.15;\n//! let b: f32 = 0.1 + 0.1 + 0.25;\n//! println!(\"{} == {}\", a, b);\n//! assert!(a==b) // Fails, because they are not exactly equal\n//! # }\n//! ```\n//!\n//! This fails because the correct answer to most operations isn't exactly representable, and so\n//! your computer's processor chooses to represent the answer with the closest value it has\n//! available. This introduces error, and this error can accumulate as multiple operations are\n//! performed.\n//!\n//! ## The solution\n//!\n//! With `ApproxEq`, we can get the answer we intend:\n//!\n//! ```\n//! # #[macro_use]\n//! # extern crate float_cmp;\n//! # use float_cmp::{ApproxEq, F32Margin};\n//! # fn main() {\n//! let a: f32 = 0.15 + 0.15 + 0.15;\n//! let b: f32 = 0.1 + 0.1 + 0.25;\n//! println!(\"{} == {}\", a, b);\n//! // They are equal, within 2 ulps\n//! assert!( approx_eq!(f32, a, b, ulps = 2) );\n//! # }\n//! ```\n//!\n//! ## Some explanation\n//!\n//! We use the term ULP (units of least precision, or units in the last place) to mean the\n//! difference between two adjacent floating point representations (adjacent meaning that there is\n//! no floating point number between them). This term is borrowed from prior work (personally I\n//! would have chosen \"quanta\"). The size of an ULP (measured as a float) varies\n//! depending on the exponents of the floating point numbers in question. That is a good thing,\n//! because as numbers fall away from equality due to the imprecise nature of their representation,\n//! they fall away in ULPs terms, not in absolute terms. Pure epsilon-based comparisons are\n//! absolute and thus don't map well to the nature of the additive error issue. They work fine\n//! for many ranges of numbers, but not for others (consider comparing -0.0000000028\n//! to +0.00000097).\n//!\n//! ## Using this crate\n//!\n//! By default this crate enables the `ratio` module providing the `ApproxEqRatio` trait. This\n//! feature pulls in `num-traits`. If you disable this feature, you'll need to either enable\n//! `num-traits` directly or else enable the `std` feature; otherwise it won't compile. This crate\n//! is `#![no_std]` unless you enable the `std` feature.\n//!\n//! You can use the `ApproxEq` trait directly like so:\n//!\n//! ```\n//! # extern crate float_cmp;\n//! # use float_cmp::{ApproxEq, F32Margin};\n//! # fn main() {\n//! # let a: f32 = 0.15 + 0.15 + 0.15;\n//! # let b: f32 = 0.1 + 0.1 + 0.25;\n//! assert!( a.approx_eq(b, F32Margin { ulps: 2, epsilon: 0.0 }) );\n//! # }\n//! ```\n//!\n//! We have implemented `From<(f32,i32)>` for `F32Margin` (and similarly for `F64Margin`)\n//! so you can use this shorthand:\n//!\n//! ```\n//! # extern crate float_cmp;\n//! # use float_cmp::{ApproxEq, F32Margin};\n//! # fn main() {\n//! # let a: f32 = 0.15 + 0.15 + 0.15;\n//! # let b: f32 = 0.1 + 0.1 + 0.25;\n//! assert!( a.approx_eq(b, (0.0, 2)) );\n//! # }\n//! ```\n//!\n//! With macros, it is easier to be explicit about which type of margin you wish to set,\n//! without mentioning the other one (the other one will be zero). But the downside is\n//! that you have to specify the type you are dealing with:\n//!\n//! ```\n//! # #[macro_use]\n//! # extern crate float_cmp;\n//! # use float_cmp::{ApproxEq, F32Margin};\n//! # fn main() {\n//! # let a: f32 = 0.15 + 0.15 + 0.15;\n//! # let b: f32 = 0.1 + 0.1 + 0.25;\n//! assert!( approx_eq!(f32, a, b, ulps = 2) );\n//! assert!( approx_eq!(f32, a, b, epsilon = 0.00000003) );\n//! assert!( approx_eq!(f32, a, b, epsilon = 0.00000003, ulps = 2) );\n//! assert!( approx_eq!(f32, a, b, (0.0, 2)) );\n//! assert!( approx_eq!(f32, a, b, F32Margin { epsilon: 0.0, ulps: 2 }) );\n//! assert!( approx_eq!(f32, a, b, F32Margin::default()) );\n//! assert!( approx_eq!(f32, a, b) ); // uses the default\n//! # }\n//! ```\n//!\n//! For most cases, I recommend you use a smallish integer for the `ulps` parameter (1 to 5\n//! or so), and a similar small multiple of the floating point's EPSILON constant (1.0 to 5.0\n//! or so), but there are *plenty* of cases where this is insufficient.\n//!\n//! ## Implementing these traits\n//!\n//! You can implement `ApproxEq` for your own complex types like shown below.\n//! The floating point type `F` must be `Copy`, but for large types you can implement\n//! it for references to your type as shown.\n//!\n//! ```\n//! use float_cmp::ApproxEq;\n//!\n//! pub struct Vec2 {\n//! pub x: F,\n//! pub y: F,\n//! }\n//!\n//! impl<'a, M: Copy + Default, F: Copy + ApproxEq> ApproxEq for &'a Vec2 {\n//! type Margin = M;\n//!\n//! fn approx_eq>(self, other: Self, margin: T) -> bool {\n//! let margin = margin.into();\n//! self.x.approx_eq(other.x, margin)\n//! && self.y.approx_eq(other.y, margin)\n//! }\n//! }\n//! ```\n//!\n//! ## Non floating-point types\n//!\n//! `ApproxEq` can be implemented for non floating-point types as well, since `Margin` is\n//! an associated type.\n//!\n//! The `efloat` crate implements (or soon will implement) `ApproxEq` for a compound type\n//! that tracks floating point error bounds by checking if the error bounds overlap.\n//! In that case `type Margin = ()`.\n//!\n//! ## Inspiration\n//!\n//! This crate was inspired by this Random ASCII blog post:\n//!\n//! [https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/](https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/)\n\n#![cfg_attr(not(feature = \"std\"), no_std)]\n\n#[macro_use]\nmod macros;\n\nmod ulps;\npub use self::ulps::Ulps;\n\nmod ulps_eq;\npub use self::ulps_eq::ApproxEqUlps;\n\nmod eq;\npub use self::eq::{ApproxEq, F32Margin, F64Margin};\n\n#[cfg(feature=\"ratio\")]\nmod ratio;\n#[cfg(feature=\"ratio\")]\npub use self::ratio::ApproxEqRatio;\n" }, { "package": "float-cmp@0.9.0", "sourcePath": "src/ratio.rs", "sha256": "ec6c8338c1c45d49b9df2748069dba883105730a7bb6542dfeaac1b778c16feb", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014-2020 Optimal Computing (NZ) Ltd.\n// Licensed under the MIT license. See LICENSE for details.\n\nuse core::cmp::PartialOrd;\nuse core::ops::{Sub, Div, Neg};\nuse num_traits::Zero;\n\n/// ApproxEqRatio is a trait for approximate equality comparisons bounding the ratio\n/// of the difference to the larger.\npub trait ApproxEqRatio : Div + Sub + Neg\n + PartialOrd + Zero + Sized + Copy\n{\n /// This method tests if `self` and `other` are nearly equal by bounding the\n /// difference between them to some number much less than the larger of the two.\n /// This bound is set as the ratio of the difference to the larger.\n fn approx_eq_ratio(&self, other: &Self, ratio: Self) -> bool {\n\n // Not equal if signs are not equal\n if *self < Self::zero() && *other > Self::zero() { return false; }\n if *self > Self::zero() && *other < Self::zero() { return false; }\n\n // Handle all zero cases\n match (*self == Self::zero(), *other == Self::zero()) {\n (true,true) => return true,\n (true,false) => return false,\n (false,true) => return false,\n _ => { }\n }\n\n // abs\n let (s,o) = if *self < Self::zero() {\n (-*self, -*other)\n } else {\n (*self, *other)\n };\n\n let (smaller,larger) = if s < o {\n (s,o)\n } else {\n (o,s)\n };\n let difference: Self = larger.sub(smaller);\n let actual_ratio: Self = difference.div(larger);\n actual_ratio < ratio\n }\n\n /// This method tests if `self` and `other` are not nearly equal by bounding the\n /// difference between them to some number much less than the larger of the two.\n /// This bound is set as the ratio of the difference to the larger.\n #[inline]\n fn approx_ne_ratio(&self, other: &Self, ratio: Self) -> bool {\n !self.approx_eq_ratio(other, ratio)\n }\n}\n\nimpl ApproxEqRatio for f32 { }\n\n#[test]\nfn f32_approx_eq_ratio_test1() {\n let x: f32 = 0.00004_f32;\n let y: f32 = 0.00004001_f32;\n assert!(x.approx_eq_ratio(&y, 0.00025));\n assert!(y.approx_eq_ratio(&x, 0.00025));\n assert!(x.approx_ne_ratio(&y, 0.00024));\n assert!(y.approx_ne_ratio(&x, 0.00024));\n}\n\n#[test]\nfn f32_approx_eq_ratio_test2() {\n let x: f32 = 0.00000000001_f32;\n let y: f32 = 0.00000000005_f32;\n assert!(x.approx_eq_ratio(&y, 0.81));\n assert!(y.approx_ne_ratio(&x, 0.79));\n}\n\n#[test]\nfn f32_approx_eq_ratio_test_zero_eq_zero_returns_true() {\n let x: f32 = 0.0_f32;\n assert!(x.approx_eq_ratio(&x,0.1) == true);\n}\n\n#[test]\nfn f32_approx_eq_ratio_test_zero_ne_zero_returns_false() {\n let x: f32 = 0.0_f32;\n assert!(x.approx_ne_ratio(&x,0.1) == false);\n}\n\n#[test]\nfn f32_approx_eq_ratio_test_against_a_zero_is_false() {\n let x: f32 = 0.0_f32;\n let y: f32 = 0.1_f32;\n assert!(x.approx_eq_ratio(&y,0.1) == false);\n assert!(y.approx_eq_ratio(&x,0.1) == false);\n}\n\n#[test]\nfn f32_approx_eq_ratio_test_negative_numbers() {\n let x: f32 = -3.0_f32;\n let y: f32 = -4.0_f32;\n // -3 and -4 should not be equal at a ratio of 0.1\n assert!(x.approx_eq_ratio(&y,0.1) == false);\n}\n\nimpl ApproxEqRatio for f64 { }\n\n#[test]\nfn f64_approx_eq_ratio_test1() {\n let x: f64 = 0.000000004_f64;\n let y: f64 = 0.000000004001_f64;\n assert!(x.approx_eq_ratio(&y, 0.00025));\n assert!(y.approx_eq_ratio(&x, 0.00025));\n assert!(x.approx_ne_ratio(&y, 0.00024));\n assert!(y.approx_ne_ratio(&x, 0.00024));\n}\n\n#[test]\nfn f64_approx_eq_ratio_test2() {\n let x: f64 = 0.0000000000000001_f64;\n let y: f64 = 0.0000000000000005_f64;\n assert!(x.approx_eq_ratio(&y, 0.81));\n assert!(y.approx_ne_ratio(&x, 0.79));\n}\n\n#[test]\nfn f64_approx_eq_ratio_test_zero_eq_zero_returns_true() {\n let x: f64 = 0.0_f64;\n assert!(x.approx_eq_ratio(&x,0.1) == true);\n}\n\n#[test]\nfn f64_approx_eq_ratio_test_zero_ne_zero_returns_false() {\n let x: f64 = 0.0_f64;\n assert!(x.approx_ne_ratio(&x,0.1) == false);\n}\n\n#[test]\nfn f64_approx_eq_ratio_test_negative_numbers() {\n let x: f64 = -3.0_f64;\n let y: f64 = -4.0_f64;\n // -3 and -4 should not be equal at a ratio of 0.1\n assert!(x.approx_eq_ratio(&y,0.1) == false);\n}\n" }, { "package": "float-cmp@0.9.0", "sourcePath": "src/ulps.rs", "sha256": "15528c3b0007b9f8b588f9113040973e819adf6245fa99da1497ec153e62c7a0", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014-2020 Optimal Computing (NZ) Ltd.\n// Licensed under the MIT license. See LICENSE for details.\n\n#[cfg(feature=\"num_traits\")]\nuse num_traits::NumCast;\n\n/// A trait for floating point numbers which computes the number of representable\n/// values or ULPs (Units of Least Precision) that separate the two given values.\n#[cfg(feature=\"num_traits\")]\npub trait Ulps {\n type U: Copy + NumCast;\n\n /// The number of representable values or ULPs (Units of Least Precision) that\n /// separate `self` and `other`. The result `U` is an integral value, and will\n /// be zero if `self` and `other` are exactly equal.\n fn ulps(&self, other: &Self) -> ::U;\n\n /// The next representable number above this one\n fn next(&self) -> Self;\n\n /// The previous representable number below this one\n fn prev(&self) -> Self;\n}\n\n#[cfg(not(feature=\"num_traits\"))]\npub trait Ulps {\n type U: Copy;\n\n /// The number of representable values or ULPs (Units of Least Precision) that\n /// separate `self` and `other`. The result `U` is an integral value, and will\n /// be zero if `self` and `other` are exactly equal.\n fn ulps(&self, other: &Self) -> ::U;\n\n /// The next representable number above this one\n fn next(&self) -> Self;\n\n /// The previous representable number below this one\n fn prev(&self) -> Self;\n}\n\nimpl Ulps for f32 {\n type U = i32;\n\n fn ulps(&self, other: &f32) -> i32 {\n\n // IEEE754 defined floating point storage representation to\n // maintain their order when their bit patterns are interpreted as\n // integers. This is a huge boon to the task at hand, as we can\n // reinterpret them as integers to find out how many ULPs apart any\n // two floats are\n\n // Setup integer representations of the input\n let ai32: i32 = self.to_bits() as i32;\n let bi32: i32 = other.to_bits() as i32;\n\n ai32.wrapping_sub(bi32)\n }\n\n fn next(&self) -> Self {\n if self.is_infinite() && *self > 0.0 {\n *self\n } else if *self == -0.0 && self.is_sign_negative() {\n 0.0\n } else {\n let mut u = self.to_bits();\n if *self >= 0.0 {\n u += 1;\n } else {\n u -= 1;\n }\n f32::from_bits(u)\n }\n }\n\n fn prev(&self) -> Self {\n if self.is_infinite() && *self < 0.0 {\n *self\n } else if *self == 0.0 && self.is_sign_positive() {\n -0.0\n } else {\n let mut u = self.to_bits();\n if *self <= -0.0 {\n u += 1;\n } else {\n u -= 1;\n }\n f32::from_bits(u)\n }\n }\n}\n\n#[test]\nfn f32_ulps_test1() {\n let x: f32 = 1000000_f32;\n let y: f32 = 1000000.1_f32;\n assert!(x.ulps(&y) == -2);\n}\n\n#[test]\nfn f32_ulps_test2() {\n let pzero: f32 = f32::from_bits(0x00000000_u32);\n let nzero: f32 = f32::from_bits(0x80000000_u32);\n assert!(pzero.ulps(&nzero) == -2147483648);\n}\n#[test]\nfn f32_ulps_test3() {\n let pinf: f32 = f32::from_bits(0x7f800000_u32);\n let ninf: f32 = f32::from_bits(0xff800000_u32);\n assert!(pinf.ulps(&ninf) == -2147483648);\n}\n\n#[test]\nfn f32_ulps_test4() {\n let x: f32 = f32::from_bits(0x63a7f026_u32);\n let y: f32 = f32::from_bits(0x63a7f023_u32);\n assert!(x.ulps(&y) == 3);\n}\n\n#[test]\nfn f32_ulps_test5() {\n let x: f32 = 2.0;\n let ulps: i32 = x.to_bits() as i32;\n let x2: f32 = ::from_bits(ulps as u32);\n assert_eq!(x, x2);\n}\n\n#[test]\nfn f32_ulps_test6() {\n let negzero: f32 = -0.;\n let zero: f32 = 0.;\n assert_eq!(negzero.next(), zero);\n assert_eq!(zero.prev(), negzero);\n assert!(negzero.prev() < 0.0);\n assert!(zero.next() > 0.0);\n}\n\nimpl Ulps for f64 {\n type U = i64;\n\n fn ulps(&self, other: &f64) -> i64 {\n\n // IEEE754 defined floating point storage representation to\n // maintain their order when their bit patterns are interpreted as\n // integers. This is a huge boon to the task at hand, as we can\n // reinterpret them as integers to find out how many ULPs apart any\n // two floats are\n\n // Setup integer representations of the input\n let ai64: i64 = self.to_bits() as i64;\n let bi64: i64 = other.to_bits() as i64;\n\n ai64.wrapping_sub(bi64)\n }\n\n fn next(&self) -> Self {\n if self.is_infinite() && *self > 0.0 {\n *self\n } else if *self == -0.0 && self.is_sign_negative() {\n 0.0\n } else {\n let mut u = self.to_bits();\n if *self >= 0.0 {\n u += 1;\n } else {\n u -= 1;\n }\n f64::from_bits(u)\n }\n }\n\n fn prev(&self) -> Self {\n if self.is_infinite() && *self < 0.0 {\n *self\n } else if *self == 0.0 && self.is_sign_positive() {\n -0.0\n } else {\n let mut u = self.to_bits();\n if *self <= -0.0 {\n u += 1;\n } else {\n u -= 1;\n }\n f64::from_bits(u)\n }\n }\n}\n\n#[test]\nfn f64_ulps_test1() {\n let x: f64 = 1000000_f64;\n let y: f64 = 1000000.00000001_f64;\n assert!(x.ulps(&y) == -86);\n}\n\n#[test]\nfn f64_ulps_test2() {\n let pzero: f64 = f64::from_bits(0x0000000000000000_u64);\n let nzero: f64 = f64::from_bits(0x8000000000000000_u64);\n assert!(pzero.ulps(&nzero) == -9223372036854775808i64);\n}\n#[test]\nfn f64_ulps_test3() {\n let pinf: f64 = f64::from_bits(0x7f80000000000000_u64);\n let ninf: f64 = f64::from_bits(0xff80000000000000_u64);\n assert!(pinf.ulps(&ninf) == -9223372036854775808i64);\n}\n\n#[test]\nfn f64_ulps_test4() {\n let x: f64 = f64::from_bits(0xd017f6cc63a7f026_u64);\n let y: f64 = f64::from_bits(0xd017f6cc63a7f023_u64);\n assert!(x.ulps(&y) == 3);\n}\n\n#[test]\nfn f64_ulps_test5() {\n let x: f64 = 2.0;\n let ulps: i64 = x.to_bits() as i64;\n let x2: f64 = ::from_bits(ulps as u64);\n assert_eq!(x, x2);\n}\n\n#[test]\nfn f64_ulps_test6() {\n let negzero: f64 = -0.;\n let zero: f64 = 0.;\n assert_eq!(negzero.next(), zero);\n assert_eq!(zero.prev(), negzero);\n assert!(negzero.prev() < 0.0);\n assert!(zero.next() > 0.0);\n}\n\n" }, { "package": "float-cmp@0.9.0", "sourcePath": "src/ulps_eq.rs", "sha256": "87ad38eb13c31304962b29511d6f8215ab0b9f3b2b51545844678f81f83fb688", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014-2020 Optimal Computing (NZ) Ltd.\n// Licensed under the MIT license. See LICENSE for details.\n\n#[cfg(feature = \"num-traits\")]\n#[allow(unused_imports)]\nuse num_traits::float::FloatCore;\nuse super::Ulps;\n\n/// ApproxEqUlps is a trait for approximate equality comparisons.\n/// The associated type Flt is a floating point type which implements Ulps, and is\n/// required so that this trait can be implemented for compound types (e.g. vectors),\n/// not just for the floats themselves.\npub trait ApproxEqUlps {\n type Flt: Ulps;\n\n /// This method tests for `self` and `other` values to be approximately equal\n /// within ULPs (Units of Least Precision) floating point representations.\n /// Differing signs are always unequal with this method, and zeroes are only\n /// equal to zeroes. Use approx_eq() from the ApproxEq trait if that is more\n /// appropriate.\n fn approx_eq_ulps(&self, other: &Self, ulps: ::U) -> bool;\n\n /// This method tests for `self` and `other` values to be not approximately\n /// equal within ULPs (Units of Least Precision) floating point representations.\n /// Differing signs are always unequal with this method, and zeroes are only\n /// equal to zeroes. Use approx_eq() from the ApproxEq trait if that is more\n /// appropriate.\n #[inline]\n fn approx_ne_ulps(&self, other: &Self, ulps: ::U) -> bool {\n !self.approx_eq_ulps(other, ulps)\n }\n}\n\nimpl ApproxEqUlps for f32 {\n type Flt = f32;\n\n fn approx_eq_ulps(&self, other: &f32, ulps: i32) -> bool {\n // -0 and +0 are drastically far in ulps terms, so\n // we need a special case for that.\n if *self==*other { return true; }\n\n // Handle differing signs as a special case, even if\n // they are very close, most people consider them\n // unequal.\n if self.is_sign_positive() != other.is_sign_positive() { return false; }\n\n let diff: i32 = self.ulps(other);\n diff >= -ulps && diff <= ulps\n }\n}\n\n#[test]\nfn f32_approx_eq_ulps_test1() {\n let f: f32 = 0.1_f32;\n let mut sum: f32 = 0.0_f32;\n for _ in 0_isize..10_isize { sum += f; }\n let product: f32 = f * 10.0_f32;\n assert!(sum != product); // Should not be directly equal:\n assert!(sum.approx_eq_ulps(&product,1) == true); // But should be close\n assert!(sum.approx_eq_ulps(&product,0) == false);\n}\n#[test]\nfn f32_approx_eq_ulps_test2() {\n let x: f32 = 1000000_f32;\n let y: f32 = 1000000.1_f32;\n assert!(x != y); // Should not be directly equal\n assert!(x.approx_eq_ulps(&y,2) == true);\n assert!(x.approx_eq_ulps(&y,1) == false);\n}\n#[test]\nfn f32_approx_eq_ulps_test_zeroes() {\n let x: f32 = 0.0_f32;\n let y: f32 = -0.0_f32;\n assert!(x.approx_eq_ulps(&y,0) == true);\n}\n\nimpl ApproxEqUlps for f64 {\n type Flt = f64;\n\n fn approx_eq_ulps(&self, other: &f64, ulps: i64) -> bool {\n // -0 and +0 are drastically far in ulps terms, so\n // we need a special case for that.\n if *self==*other { return true; }\n\n // Handle differing signs as a special case, even if\n // they are very close, most people consider them\n // unequal.\n if self.is_sign_positive() != other.is_sign_positive() { return false; }\n\n let diff: i64 = self.ulps(other);\n diff >= -ulps && diff <= ulps\n }\n}\n\n#[test]\nfn f64_approx_eq_ulps_test1() {\n let f: f64 = 0.1_f64;\n let mut sum: f64 = 0.0_f64;\n for _ in 0_isize..10_isize { sum += f; }\n let product: f64 = f * 10.0_f64;\n assert!(sum != product); // Should not be directly equal:\n assert!(sum.approx_eq_ulps(&product,1) == true); // But should be close\n assert!(sum.approx_eq_ulps(&product,0) == false);\n}\n#[test]\nfn f64_approx_eq_ulps_test2() {\n let x: f64 = 1000000_f64;\n let y: f64 = 1000000.0000000003_f64;\n assert!(x != y); // Should not be directly equal\n assert!(x.approx_eq_ulps(&y,3) == true);\n assert!(x.approx_eq_ulps(&y,2) == false);\n}\n#[test]\nfn f64_approx_eq_ulps_test_zeroes() {\n let x: f64 = 0.0_f64;\n let y: f64 = -0.0_f64;\n assert!(x.approx_eq_ulps(&y,0) == true);\n}\n" }, { "package": "fontdb@0.14.1", "sourcePath": "tests/fonts/LICENSE.txt", "sha256": "b84fdd2c3da5db56385cdbb639795e90aa3e035c53bc3591135f18df3331451f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "We, the copyright holders of this work, hereby release it into the\npublic domain. This applies worldwide.\n\nIn case this is not legally possible,\n\nWe grant any entity the right to use this work for any purpose, without\nany conditions, unless such conditions are required by law.\n\nThatcher Ulrich http://tulrich.com\nKaroly Barta bartakarcsi@gmail.com\nMichael Evans http://www.evertype.com\n" }, { "package": "funty@2.0.0", "sourcePath": "LICENSE.txt", "sha256": "f790cc576999f5998c766d3d26d7d64dc368e805a98461484f65e8d961ec6d9f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "MIT License\n\nCopyright (c) 2020 myrrlyn (Alexander Payne)\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "package": "futures@0.3.34", "sourcePath": "README.md", "sha256": "260fb7aba3cb6a6410da78f717172a96ab7b03ddc99beb8d78b107ffc3e5a6ae", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "

\n \"futures-rs\"\n

\n\n

\n Zero-cost asynchronous programming in Rust\n

\n\n

\n \n \"Build\n \n\n \n \"crates.io\"\n \n

\n\n

\n \n Documentation\n | \n Website\n \n

\n\n`futures-rs` is a library providing the foundations for asynchronous programming in Rust.\nIt includes key trait definitions like `Stream`, as well as utilities like `join!`,\n`select!`, and various futures combinator methods which enable expressive asynchronous\ncontrol flow.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nfutures = \"0.3\"\n```\n\nThe current `futures` requires Rust 1.71 or later.\n\n### Feature `std`\n\nFutures-rs works without the standard library, such as in bare metal environments.\nHowever, it has a significantly reduced API surface. To use futures-rs in\na `#[no_std]` environment, use:\n\n```toml\n[dependencies]\nfutures = { version = \"0.3\", default-features = false }\n```\n\n## License\n\nLicensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or\n[MIT license](LICENSE-MIT) at your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "futures-channel@0.3.34", "sourcePath": "README.md", "sha256": "76d57b08b50a525e00265477b582a6bbcf82ec17bdce3a808f55f70358a83552", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# futures-channel\n\nChannels for asynchronous communication using futures-rs.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nfutures-channel = \"0.3\"\n```\n\nThe current `futures-channel` requires Rust 1.71 or later.\n\n## License\n\nLicensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or\n[MIT license](LICENSE-MIT) at your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "futures-channel@0.3.34", "sourcePath": "src/mpsc/queue.rs", "sha256": "22034085dc22050b708a37854e215fc7cbb16d65edc60370cb5d8f4b7faca18e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved.\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *\n * 1. Redistributions of source code must retain the above copyright notice,\n * this list of conditions and the following disclaimer.\n *\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n *\n * THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV \"AS IS\" AND ANY EXPRESS OR IMPLIED\n * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT\n * SHALL DMITRY VYUKOV OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\n * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF\n * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE\n * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF\n * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n *\n * The views and conclusions contained in the software and documentation are\n * those of the authors and should not be interpreted as representing official\n * policies, either expressed or implied, of Dmitry Vyukov.\n */\n\n//! A mostly lock-free multi-producer, single consumer queue for sending\n//! messages between asynchronous tasks.\n//!\n//! The queue implementation is essentially the same one used for mpsc channels\n//! in the standard library.\n//!\n//! Note that the current implementation of this queue has a caveat of the `pop`\n//! method, and see the method for more information about it. Due to this\n//! caveat, this queue may not be appropriate for all use-cases.\n\n// http://www.1024cores.net/home/lock-free-algorithms\n// /queues/non-intrusive-mpsc-node-based-queue\n\n// NOTE: this implementation is lifted from the standard library and only\n// slightly modified\n\npub(super) use self::PopResult::*;\n\nuse std::boxed::Box;\nuse std::cell::UnsafeCell;\nuse std::ptr;\nuse std::sync::atomic::{AtomicPtr, Ordering};\nuse std::thread;\n\n/// A result of the `pop` function.\npub(super) enum PopResult {\n /// Some data has been popped\n Data(T),\n /// The queue is empty\n Empty,\n /// The queue is in an inconsistent state. Popping data should succeed, but\n /// some pushers have yet to make enough progress in order allow a pop to\n /// succeed. It is recommended that a pop() occur \"in the near future\" in\n /// order to see if the sender has made progress or not\n Inconsistent,\n}\n\nstruct Node {\n next: AtomicPtr,\n value: Option,\n}\n\n/// The multi-producer single-consumer structure. This is not cloneable, but it\n/// may be safely shared so long as it is guaranteed that there is only one\n/// popper at a time (many pushers are allowed).\npub(super) struct Queue {\n head: AtomicPtr>,\n tail: UnsafeCell<*mut Node>,\n}\n\nunsafe impl Send for Queue {}\nunsafe impl Sync for Queue {}\n\nimpl Node {\n unsafe fn new(v: Option) -> *mut Self {\n Box::into_raw(Box::new(Self { next: AtomicPtr::new(ptr::null_mut()), value: v }))\n }\n}\n\nimpl Queue {\n /// Creates a new queue that is safe to share among multiple producers and\n /// one consumer.\n pub(super) fn new() -> Self {\n let stub = unsafe { Node::new(None) };\n Self { head: AtomicPtr::new(stub), tail: UnsafeCell::new(stub) }\n }\n\n /// Pushes a new value onto this queue.\n pub(super) fn push(&self, t: T) {\n unsafe {\n let n = Node::new(Some(t));\n let prev = self.head.swap(n, Ordering::AcqRel);\n (*prev).next.store(n, Ordering::Release);\n }\n }\n\n /// Pops some data from this queue.\n ///\n /// Note that the current implementation means that this function cannot\n /// return `Option`. It is possible for this queue to be in an\n /// inconsistent state where many pushes have succeeded and completely\n /// finished, but pops cannot return `Some(t)`. This inconsistent state\n /// happens when a pusher is preempted at an inopportune moment.\n ///\n /// This inconsistent state means that this queue does indeed have data, but\n /// it does not currently have access to it at this time.\n ///\n /// This function is unsafe because only one thread can call it at a time.\n pub(super) unsafe fn pop(&self) -> PopResult {\n unsafe {\n let tail = *self.tail.get();\n let next = (*tail).next.load(Ordering::Acquire);\n\n if !next.is_null() {\n *self.tail.get() = next;\n assert!((*tail).value.is_none());\n assert!((*next).value.is_some());\n let ret = (*next).value.take().unwrap();\n drop(Box::from_raw(tail));\n return Data(ret);\n }\n\n if self.head.load(Ordering::Acquire) == tail {\n Empty\n } else {\n Inconsistent\n }\n }\n }\n\n /// Pop an element similarly to `pop` function, but spin-wait on inconsistent\n /// queue state instead of returning `Inconsistent`.\n ///\n /// This function is unsafe because only one thread can call it at a time.\n pub(super) unsafe fn pop_spin(&self) -> Option {\n loop {\n match unsafe { self.pop() } {\n Empty => return None,\n Data(t) => return Some(t),\n // Inconsistent means that there will be a message to pop\n // in a short time. This branch can only be reached if\n // values are being produced from another thread, so there\n // are a few ways that we can deal with this:\n //\n // 1) Spin\n // 2) thread::yield_now()\n // 3) task::current().unwrap() & return Pending\n //\n // For now, thread::yield_now() is used, but it would\n // probably be better to spin a few times then yield.\n Inconsistent => {\n thread::yield_now();\n }\n }\n }\n }\n}\n\nimpl Drop for Queue {\n fn drop(&mut self) {\n unsafe {\n let mut cur = *self.tail.get();\n while !cur.is_null() {\n let next = (*cur).next.load(Ordering::Relaxed);\n drop(Box::from_raw(cur));\n cur = next;\n }\n }\n }\n}\n" }, { "package": "futures-core@0.3.34", "sourcePath": "README.md", "sha256": "e8258273fed6f1796485777655118f2369fd3f000191e9d8cdbd10bf052946a9", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# futures-core\n\nThe core traits and types in for the `futures` library.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nfutures-core = \"0.3\"\n```\n\nThe current `futures-core` requires Rust 1.36 or later.\n\n## License\n\nLicensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or\n[MIT license](LICENSE-MIT) at your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "futures-executor@0.3.34", "sourcePath": "README.md", "sha256": "c245377709f81e8d92d5e162c3b28131294c0c0452bd69bee8df68777d23b2ac", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# futures-executor\n\nExecutors for asynchronous tasks based on the futures-rs library.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nfutures-executor = \"0.3\"\n```\n\nThe current `futures-executor` requires Rust 1.71 or later.\n\n## License\n\nLicensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or\n[MIT license](LICENSE-MIT) at your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "futures-io@0.3.34", "sourcePath": "README.md", "sha256": "575430be5c47352d85f36b44dcc2c2851a6a19e2384593415c4af22c6654cee7", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# futures-io\n\nThe `AsyncRead`, `AsyncWrite`, `AsyncSeek`, and `AsyncBufRead` traits for the futures-rs library.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nfutures-io = \"0.3\"\n```\n\nThe current `futures-io` requires Rust 1.36 or later.\n\n## License\n\nLicensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or\n[MIT license](LICENSE-MIT) at your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "futures-sink@0.3.34", "sourcePath": "README.md", "sha256": "a509e1ce84f285190130def6d2b9e3861988f9be725f7697f09fba347601d86f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# futures-sink\n\nThe asynchronous `Sink` trait for the futures-rs library.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nfutures-sink = \"0.3\"\n```\n\nThe current `futures-sink` requires Rust 1.36 or later.\n\n## License\n\nLicensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or\n[MIT license](LICENSE-MIT) at your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "futures-task@0.3.34", "sourcePath": "README.md", "sha256": "7a458e69ff008c35cffbbbcd2a14ad3ba7ad75201ce74df2d8f8d0fdcf6c4a89", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# futures-task\n\nTools for working with tasks.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nfutures-task = \"0.3\"\n```\n\nThe current `futures-task` requires Rust 1.71 or later.\n\n## License\n\nLicensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or\n[MIT license](LICENSE-MIT) at your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "futures-util@0.3.34", "sourcePath": "README.md", "sha256": "00510d047b36feb45db9034bf5f07a049a99b51d6b8c40d028cb35977baeb723", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# futures-util\n\nCommon utilities and extension traits for the futures-rs library.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nfutures-util = \"0.3\"\n```\n\nThe current `futures-util` requires Rust 1.71 or later.\n\n## License\n\nLicensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or\n[MIT license](LICENSE-MIT) at your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "is_terminal_polyfill@1.70.2", "sourcePath": "README.md", "sha256": "edcbd294340511763ef2f1d7b96434e822afa48fcb6dd712aaf9498afe06ea3f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# is_terminal_polyfill\n\n> Polyfill for `is_terminal` stdlib feature for use with older MSRVs\n\n[![Documentation](https://img.shields.io/badge/docs-master-blue.svg)][Documentation]\n![License](https://img.shields.io/crates/l/is_terminal_polyfill.svg)\n[![Crates Status](https://img.shields.io/crates/v/is_terminal_polyfill.svg)][Crates.io]\n\n## License\n\nLicensed under either of\n\n* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or )\n* MIT license ([LICENSE-MIT](LICENSE-MIT) or )\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally\nsubmitted for inclusion in the work by you, as defined in the Apache-2.0\nlicense, shall be dual licensed as above, without any additional terms or\nconditions.\n\n[Crates.io]: https://crates.io/crates/is_terminal_polyfill\n[Documentation]: https://docs.rs/is_terminal_polyfill\n" }, { "package": "itoa@1.0.18", "sourcePath": "README.md", "sha256": "1305bd79675db0194915d9bc91bd0b3cf9487bc1126e05bf46b48ab43edce44b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "itoa\n====\n\n[\"github\"](https://github.com/dtolnay/itoa)\n[\"crates.io\"](https://crates.io/crates/itoa)\n[\"docs.rs\"](https://docs.rs/itoa)\n[\"build](https://github.com/dtolnay/itoa/actions?query=branch%3Amaster)\n\nThis crate provides a fast conversion of integer primitives to decimal strings.\nThe implementation comes straight from [libcore] but avoids the performance\npenalty of going through [`core::fmt::Formatter`].\n\nSee also [`zmij`] for printing floating point primitives.\n\n[libcore]: https://github.com/rust-lang/rust/blob/1.92.0/library/core/src/fmt/num.rs#L190-L253\n[`core::fmt::Formatter`]: https://doc.rust-lang.org/std/fmt/struct.Formatter.html\n[`zmij`]: https://github.com/dtolnay/zmij\n\n```toml\n[dependencies]\nitoa = \"1.0\"\n```\n\n
\n\n## Example\n\n```rust\nfn main() {\n let mut buffer = itoa::Buffer::new();\n let printed = buffer.format(128u64);\n assert_eq!(printed, \"128\");\n}\n```\n\n
\n\n## Performance\n\nThe [itoa-benchmark] compares this library and other Rust integer formatting\nimplementations across a range of integer sizes. The vertical axis in this chart\nshows nanoseconds taken by a single execution of\n`itoa::Buffer::new().format(value)` so a lower result indicates a faster\nlibrary.\n\n[itoa-benchmark]: https://github.com/dtolnay/itoa-benchmark\n\n![performance](https://raw.githubusercontent.com/dtolnay/itoa/master/itoa-benchmark.png)\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this crate by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "kurbo@0.9.5", "sourcePath": "benches/cubic_arclen.rs", "sha256": "edb2707abacaf15261cfa6b546c2e35c57b805b2d9ff96b464cceff98ca40e05", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Benchmarks of cubic arclength approaches.\r\n\r\n#![cfg(nightly)]\r\n#![feature(test)]\r\nextern crate test;\r\nuse test::Bencher;\r\n\r\nuse kurbo::{CubicBez, ParamCurveArclen};\r\n\r\n#[bench]\r\nfn bench_cubic_arclen_1e_4(b: &mut Bencher) {\r\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\r\n b.iter(|| test::black_box(c).arclen(1e-4))\r\n}\r\n\r\n#[bench]\r\nfn bench_cubic_arclen_1e_5(b: &mut Bencher) {\r\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\r\n b.iter(|| test::black_box(c).arclen(1e-5))\r\n}\r\n\r\n#[bench]\r\nfn bench_cubic_arclen_1e_6(b: &mut Bencher) {\r\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\r\n b.iter(|| test::black_box(c).arclen(1e-6))\r\n}\r\n\r\n#[bench]\r\nfn bench_cubic_arclen_degenerate(b: &mut Bencher) {\r\n let c = CubicBez::new((0.0, 0.0), (0.0, 0.0), (0.0, 0.0), (0.0, 0.0));\r\n b.iter(|| test::black_box(c).arclen(1e-6))\r\n}\r\n\r\n#[bench]\r\nfn bench_cubic_arclen_1e_7(b: &mut Bencher) {\r\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\r\n b.iter(|| test::black_box(c).arclen(1e-7))\r\n}\r\n\r\n#[bench]\r\nfn bench_cubic_arclen_1e_8(b: &mut Bencher) {\r\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\r\n b.iter(|| test::black_box(c).arclen(1e-8))\r\n}\r\n\r\n#[bench]\r\nfn bench_cubic_arclen_1e_9(b: &mut Bencher) {\r\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\r\n b.iter(|| test::black_box(c).arclen(1e-9))\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "benches/quad_arclen.rs", "sha256": "17be81efcdf2beb5d117de12babc046f25cf89ca835def28c240e79f4f008aaf", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Benchmarks of quadratic arclength approaches.\r\n\r\n// TODO: organize so there's less cut'n'paste from arclen_accuracy example.\r\n\r\n#![cfg(nightly)]\r\n#![feature(test)]\r\nextern crate test;\r\nuse test::Bencher;\r\n\r\nuse kurbo::{ParamCurve, ParamCurveArclen, ParamCurveDeriv, QuadBez};\r\n\r\n// Based on http://www.malczak.linuxpl.com/blog/quadratic-bezier-curve-length/\r\nfn quad_arclen_analytical(q: QuadBez) -> f64 {\r\n let d2 = q.p0.to_vec2() - 2.0 * q.p1.to_vec2() + q.p2.to_vec2();\r\n let a = d2.hypot2();\r\n let d1 = q.p1 - q.p0;\r\n let b = 2.0 * d2.dot(d1);\r\n let c = d1.hypot2();\r\n\r\n let sabc = (a + b + c).sqrt();\r\n let a2 = a.powf(-0.5);\r\n let a32 = a2.powi(3);\r\n let c2 = 2.0 * c.sqrt();\r\n let ba = b * a2;\r\n\r\n sabc + 0.25\r\n * (a2 * a2 * b * (2.0 * sabc - c2)\r\n + a32 * (4.0 * c * a - b * b) * (((2.0 * a + b) * a2 + 2.0 * sabc) / (ba + c2)).ln())\r\n}\r\n\r\n/// Calculate arclength using Gauss-Legendre quadrature using formula from Behdad\r\n/// in https://github.com/Pomax/BezierInfo-2/issues/77\r\nfn gauss_arclen_3(q: QuadBez) -> f64 {\r\n let v0 = (-0.492943519233745 * q.p0.to_vec2()\r\n + 0.430331482911935 * q.p1.to_vec2()\r\n + 0.0626120363218102 * q.p2.to_vec2())\r\n .hypot();\r\n let v1 = ((q.p2 - q.p0) * 0.4444444444444444).hypot();\r\n let v2 = (-0.0626120363218102 * q.p0.to_vec2() - 0.430331482911935 * q.p1.to_vec2()\r\n + 0.492943519233745 * q.p2.to_vec2())\r\n .hypot();\r\n v0 + v1 + v2\r\n}\r\n\r\nfn awesome_quad_arclen3(q: QuadBez, accuracy: f64, depth: usize) -> f64 {\r\n let pm = q.p0.midpoint(q.p2);\r\n let d1 = q.p1 - pm;\r\n let d = q.p2 - q.p0;\r\n let dhypot2 = d.hypot2();\r\n let x = 2.0 * d.dot(d1) / dhypot2;\r\n let y = 2.0 * d.cross(d1) / dhypot2;\r\n let lc = (q.p2 - q.p0).hypot();\r\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\r\n let est_err = 0.06 * (lp - lc) * (x * x + y * y).powf(2.0);\r\n if est_err < accuracy || depth == 16 {\r\n gauss_arclen_3(q)\r\n } else {\r\n let (q0, q1) = q.subdivide();\r\n awesome_quad_arclen3(q0, accuracy * 0.5, depth + 1)\r\n + awesome_quad_arclen3(q1, accuracy * 0.5, depth + 1)\r\n }\r\n}\r\n\r\n/// Another implementation, using weights from\r\n/// https://pomax.github.io/bezierinfo/legendre-gauss.html\r\nfn gauss_arclen_n(q: QuadBez, coeffs: &[(f64, f64)]) -> f64 {\r\n let d = q.deriv();\r\n coeffs\r\n .iter()\r\n .map(|(wi, xi)| wi * d.eval(0.5 * (xi + 1.0)).to_vec2().hypot())\r\n .sum::()\r\n * 0.5\r\n}\r\n\r\nfn gauss_arclen_7(q: QuadBez) -> f64 {\r\n gauss_arclen_n(\r\n q,\r\n &[\r\n (0.4179591836734694, 0.0000000000000000),\r\n (0.3818300505051189, 0.4058451513773972),\r\n (0.3818300505051189, -0.4058451513773972),\r\n (0.2797053914892766, -0.7415311855993945),\r\n (0.2797053914892766, 0.7415311855993945),\r\n (0.1294849661688697, -0.9491079123427585),\r\n (0.1294849661688697, 0.9491079123427585),\r\n ],\r\n )\r\n}\r\n\r\nfn gauss_arclen_24(q: QuadBez) -> f64 {\r\n gauss_arclen_n(\r\n q,\r\n &[\r\n (0.1279381953467522, -0.0640568928626056),\r\n (0.1279381953467522, 0.0640568928626056),\r\n (0.1258374563468283, -0.1911188674736163),\r\n (0.1258374563468283, 0.1911188674736163),\r\n (0.1216704729278034, -0.3150426796961634),\r\n (0.1216704729278034, 0.3150426796961634),\r\n (0.1155056680537256, -0.4337935076260451),\r\n (0.1155056680537256, 0.4337935076260451),\r\n (0.1074442701159656, -0.5454214713888396),\r\n (0.1074442701159656, 0.5454214713888396),\r\n (0.0976186521041139, -0.6480936519369755),\r\n (0.0976186521041139, 0.6480936519369755),\r\n (0.0861901615319533, -0.7401241915785544),\r\n (0.0861901615319533, 0.7401241915785544),\r\n (0.0733464814110803, -0.8200019859739029),\r\n (0.0733464814110803, 0.8200019859739029),\r\n (0.0592985849154368, -0.8864155270044011),\r\n (0.0592985849154368, 0.8864155270044011),\r\n (0.0442774388174198, -0.9382745520027328),\r\n (0.0442774388174198, 0.9382745520027328),\r\n (0.0285313886289337, -0.9747285559713095),\r\n (0.0285313886289337, 0.9747285559713095),\r\n (0.0123412297999872, -0.9951872199970213),\r\n (0.0123412297999872, 0.9951872199970213),\r\n ],\r\n )\r\n}\r\n\r\nfn awesome_quad_arclen7(q: QuadBez, accuracy: f64, depth: usize) -> f64 {\r\n let pm = q.p0.midpoint(q.p2);\r\n let d1 = q.p1 - pm;\r\n let d = q.p2 - q.p0;\r\n let dhypot2 = d.hypot2();\r\n let x = 2.0 * d.dot(d1) / dhypot2;\r\n let y = 2.0 * d.cross(d1) / dhypot2;\r\n let lc = dhypot2.sqrt();\r\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\r\n let est_err = 2.5e-2 * (lp - lc) * (x * x + y * y).powf(8.0).tanh();\r\n if est_err < accuracy || depth == 16 {\r\n gauss_arclen_7(q)\r\n } else {\r\n let (q0, q1) = q.subdivide();\r\n awesome_quad_arclen7(q0, accuracy * 0.5, depth + 1)\r\n + awesome_quad_arclen7(q1, accuracy * 0.5, depth + 1)\r\n }\r\n}\r\n\r\n#[bench]\r\nfn bench_quad_arclen_analytical(b: &mut Bencher) {\r\n // Analytical solution is not sensitive to exact shape.\r\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\r\n b.iter(|| quad_arclen_analytical(test::black_box(q)))\r\n}\r\n\r\nconst ACCURACY: f64 = 1e-6;\r\n\r\n#[bench]\r\nfn bench_quad_arclen(b: &mut Bencher) {\r\n // This is a pretty easy case.\r\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\r\n b.iter(|| (test::black_box(q).arclen(ACCURACY)))\r\n}\r\n\r\n#[bench]\r\nfn bench_quad_arclen_gauss3(b: &mut Bencher) {\r\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\r\n b.iter(|| awesome_quad_arclen3(test::black_box(q), ACCURACY, 0))\r\n}\r\n\r\n#[bench]\r\nfn bench_quad_arclen_gauss7(b: &mut Bencher) {\r\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\r\n b.iter(|| awesome_quad_arclen7(test::black_box(q), ACCURACY, 0))\r\n}\r\n\r\n#[bench]\r\nfn bench_quad_arclen_gauss3_one(b: &mut Bencher) {\r\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\r\n b.iter(|| gauss_arclen_3(test::black_box(q)))\r\n}\r\n\r\n#[bench]\r\nfn bench_quad_arclen_gauss7_one(b: &mut Bencher) {\r\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\r\n b.iter(|| gauss_arclen_7(test::black_box(q)))\r\n}\r\n\r\n#[bench]\r\nfn bench_quad_arclen_gauss24_one(b: &mut Bencher) {\r\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\r\n b.iter(|| gauss_arclen_24(test::black_box(q)))\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "benches/quartic.rs", "sha256": "d660a15c24cb93fe2761d9faf138a8b37ef4e0de9415d95682b1df900078da1c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Benchmarks of the quartic equation solver.\r\n\r\n#![cfg(nightly)]\r\n#![feature(test)]\r\nextern crate test;\r\nuse kurbo::common::solve_quartic;\r\nuse test::Bencher;\r\n\r\n#[bench]\r\nfn bench_quartic(bb: &mut Bencher) {\r\n let (x1, x2, x3, x4) = (1.0, 2.0, 3.0, 4.0);\r\n let a = -(x1 + x2 + x3 + x4);\r\n let b = x1 * (x2 + x3) + x2 * (x3 + x4) + x4 * (x1 + x3);\r\n let c = -x1 * x2 * (x3 + x4) - x3 * x4 * (x1 + x2);\r\n let d = x1 * x2 * x3 * x4;\r\n\r\n bb.iter(|| {\r\n solve_quartic(\r\n test::black_box(d),\r\n test::black_box(c),\r\n test::black_box(b),\r\n test::black_box(a),\r\n 1.0,\r\n )\r\n })\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "benches/rect_expand.rs", "sha256": "deb2fa69284b7895722bf4001bef7603e4ebdb5f6cbb76b8c8e1cd205673b5fd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n#![cfg(nightly)]\r\n#![feature(test)]\r\nextern crate test;\r\nuse test::Bencher;\r\n\r\nuse kurbo::Rect;\r\n\r\n// In positive space\r\nstatic RECT_POS: Rect = Rect::new(3.3, 3.6, 5.6, 4.1);\r\n// In both positive and negative space\r\nstatic RECT_PAN: Rect = Rect::new(-3.3, -3.6, 5.6, 4.1);\r\n// In negative space\r\nstatic RECT_NEG: Rect = Rect::new(-5.6, -4.1, -3.3, -3.6);\r\n// In positive space reverse\r\nstatic RECT_POR: Rect = Rect::new(5.6, 4.1, 3.3, 3.6);\r\n// In both positive and negative space reverse\r\nstatic RECT_PNR: Rect = Rect::new(5.6, 4.1, -3.3, -3.6);\r\n// In negative space reverse\r\nstatic RECT_NER: Rect = Rect::new(-3.3, -3.6, -5.6, -4.1);\r\n// In positive space mixed\r\nstatic RECT_POM: Rect = Rect::new(3.3, 4.1, 5.6, 3.6);\r\n// In both positive and negative space mixed\r\nstatic RECT_PNM: Rect = Rect::new(-3.3, 4.1, 5.6, -3.6);\r\n// In negative space mixed\r\nstatic RECT_NEM: Rect = Rect::new(-5.6, -3.6, -3.3, -4.1);\r\n\r\n#[inline]\r\nfn expand(rects: &[Rect; 9]) {\r\n test::black_box(rects[0].expand());\r\n test::black_box(rects[1].expand());\r\n test::black_box(rects[2].expand());\r\n\r\n test::black_box(rects[3].expand());\r\n test::black_box(rects[4].expand());\r\n test::black_box(rects[5].expand());\r\n\r\n test::black_box(rects[6].expand());\r\n test::black_box(rects[7].expand());\r\n test::black_box(rects[8].expand());\r\n}\r\n\r\n#[bench]\r\nfn bench_expand(b: &mut Bencher) {\r\n // Creating the array here to prevent the compiler from optimizing all of it to NOP.\r\n let rects: [Rect; 9] = [\r\n RECT_POS, RECT_PAN, RECT_NEG, RECT_POR, RECT_PNR, RECT_NER, RECT_POM, RECT_PNM, RECT_NEM,\r\n ];\r\n b.iter(|| expand(&rects));\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "examples/arclen_accuracy.rs", "sha256": "2e7bd497685563b67363e53fe788391f4b48b40edd067951c0447a5ded76daf7", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A test program to plot the error of arclength approximation.\r\n\r\n// Lots of stuff is commented out or was just something to try.\r\n#![allow(unused)]\r\n#![allow(clippy::unreadable_literal)]\r\n#![allow(clippy::many_single_char_names)]\r\n\r\n// TODO: make more functionality accessible from command line rather than uncommenting.\r\n\r\nuse std::env;\r\nuse std::time::{Duration, Instant};\r\n\r\nuse kurbo::{ParamCurve, ParamCurveArclen, ParamCurveDeriv, QuadBez};\r\n\r\nuse kurbo::common::{GAUSS_LEGENDRE_COEFFS_24, GAUSS_LEGENDRE_COEFFS_5, GAUSS_LEGENDRE_COEFFS_7};\r\n\r\n/// Calculate arclength using Gauss-Legendre quadrature using formula from Behdad\r\n/// in https://github.com/Pomax/BezierInfo-2/issues/77\r\nfn gauss_arclen_3(q: QuadBez) -> f64 {\r\n let v0 = (-0.492943519233745 * q.p0.to_vec2()\r\n + 0.430331482911935 * q.p1.to_vec2()\r\n + 0.0626120363218102 * q.p2.to_vec2())\r\n .hypot();\r\n let v1 = ((q.p2 - q.p0) * 0.4444444444444444).hypot();\r\n let v2 = (-0.0626120363218102 * q.p0.to_vec2() - 0.430331482911935 * q.p1.to_vec2()\r\n + 0.492943519233745 * q.p2.to_vec2())\r\n .hypot();\r\n v0 + v1 + v2\r\n}\r\n\r\nfn awesome_quad_arclen(q: QuadBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\r\n let pm = q.p0.midpoint(q.p2);\r\n let d1 = q.p1 - pm;\r\n let d = q.p2 - q.p0;\r\n let dhypot2 = d.hypot2();\r\n let x = 2.0 * d.dot(d1) / dhypot2;\r\n let y = 2.0 * d.cross(d1) / dhypot2;\r\n let lc = (q.p2 - q.p0).hypot();\r\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\r\n let est_err = 0.06 * (lp - lc) * (x * x + y * y).powf(2.0);\r\n if est_err < accuracy || depth == 16 {\r\n *count += 1;\r\n gauss_arclen_3(q)\r\n } else {\r\n let (q0, q1) = q.subdivide();\r\n awesome_quad_arclen(q0, accuracy * 0.5, depth + 1, count)\r\n + awesome_quad_arclen(q1, accuracy * 0.5, depth + 1, count)\r\n }\r\n}\r\n\r\nconst MAX_DEPTH: usize = 16;\r\nfn gravesen_rec(q: &QuadBez, l0: f64, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\r\n let (q0, q1) = q.subdivide();\r\n let l0_q0 = calc_l0(q0);\r\n let l0_q1 = calc_l0(q1);\r\n let l1 = l0_q0 + l0_q1;\r\n let error = (l0 - l1) * (1.0 / 15.0);\r\n if error.abs() < accuracy || depth == MAX_DEPTH {\r\n *count += 1;\r\n l1 - error\r\n } else {\r\n gravesen_rec(&q0, l0_q0, accuracy * 0.5, depth + 1, count)\r\n + gravesen_rec(&q1, l0_q1, accuracy * 0.5, depth + 1, count)\r\n }\r\n}\r\n\r\nfn gauss_arclen_5(q: QuadBez) -> f64 {\r\n q.gauss_arclen(GAUSS_LEGENDRE_COEFFS_5)\r\n}\r\n\r\nfn gauss_arclen_7(q: QuadBez) -> f64 {\r\n q.gauss_arclen(GAUSS_LEGENDRE_COEFFS_7)\r\n}\r\n\r\nfn gauss_arclen_24(q: QuadBez) -> f64 {\r\n q.gauss_arclen(GAUSS_LEGENDRE_COEFFS_24)\r\n}\r\n\r\nfn awesome_quad_arclen7(q: QuadBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\r\n let pm = q.p0.midpoint(q.p2);\r\n let d1 = q.p1 - pm;\r\n let d = q.p2 - q.p0;\r\n let dhypot2 = d.hypot2();\r\n let x = 2.0 * d.dot(d1) / dhypot2;\r\n let y = 2.0 * d.cross(d1) / dhypot2;\r\n let lc = (q.p2 - q.p0).hypot();\r\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\r\n let est_err = 2.5e-2 * (lp - lc) * (x * x + y * y).powf(8.0).tanh();\r\n // Increase depth, so we can be an accurate baseline for comparison.\r\n if est_err < accuracy || depth == 22 {\r\n *count += 1;\r\n gauss_arclen_7(q)\r\n } else {\r\n let (q0, q1) = q.subdivide();\r\n awesome_quad_arclen7(q0, accuracy * 0.5, depth + 1, count)\r\n + awesome_quad_arclen7(q1, accuracy * 0.5, depth + 1, count)\r\n }\r\n}\r\n\r\nfn awesome_quad_arclen24(q: QuadBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\r\n let pm = q.p0.midpoint(q.p2);\r\n let d1 = q.p1 - pm;\r\n let d = q.p2 - q.p0;\r\n let dhypot2 = d.hypot2();\r\n let x = 2.0 * d.dot(d1) / dhypot2;\r\n let y = 2.0 * d.cross(d1) / dhypot2;\r\n let lc = (q.p2 - q.p0).hypot();\r\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\r\n // This isn't quite right near (1.1, 0)\r\n let est_err = 1.0 * (lp - lc) * (0.5 * x * x + 0.1 * y * y).powf(16.0).tanh();\r\n if est_err < accuracy || depth == 16 {\r\n *count += 1;\r\n gauss_arclen_24(q)\r\n } else {\r\n let (q0, q1) = q.subdivide();\r\n awesome_quad_arclen24(q0, accuracy * 0.5, depth + 1, count)\r\n + awesome_quad_arclen24(q1, accuracy * 0.5, depth + 1, count)\r\n }\r\n}\r\n\r\n// Based on http://www.malczak.linuxpl.com/blog/quadratic-bezier-curve-length/\r\nfn quad_arclen_analytical(q: QuadBez) -> f64 {\r\n let d2 = q.p0.to_vec2() - 2.0 * q.p1.to_vec2() + q.p2.to_vec2();\r\n let a = d2.hypot2();\r\n let d1 = q.p1 - q.p0;\r\n let c = d1.hypot2();\r\n if a < 5e-4 * c {\r\n return gauss_arclen_3(q);\r\n }\r\n let b = 2.0 * d2.dot(d1);\r\n\r\n let sabc = (a + b + c).sqrt();\r\n let a2 = a.powf(-0.5);\r\n let a32 = a2.powi(3);\r\n let c2 = 2.0 * c.sqrt();\r\n let ba_c2 = b * a2 + c2;\r\n\r\n let v0 = 0.25 * a2 * a2 * b * (2.0 * sabc - c2) + sabc;\r\n // TODO: justify and fine-tune this exact constant.\r\n if ba_c2 < 1e-13 {\r\n // This case happens for Béziers with a sharp kink.\r\n v0\r\n } else {\r\n v0 + 0.25 * a32 * (4.0 * c * a - b * b) * (((2.0 * a + b) * a2 + 2.0 * sabc) / ba_c2).ln()\r\n }\r\n}\r\n\r\n/// Calculate the L0 metric from \"Adaptive subdivision and the length and\r\n/// energy of Bézier curves\" by Jens Gravesen.\r\nfn calc_l0(q: QuadBez) -> f64 {\r\n let lc = (q.p2 - q.p0).hypot();\r\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\r\n (2.0 / 3.0) * lc + (1.0 / 3.0) * lp\r\n}\r\n\r\nfn with_subdiv(q: QuadBez, f: &dyn Fn(QuadBez) -> f64, depth: usize) -> f64 {\r\n if depth == 0 {\r\n f(q)\r\n } else {\r\n let (q0, q1) = q.subdivide();\r\n with_subdiv(q0, f, depth - 1) + with_subdiv(q1, f, depth - 1)\r\n }\r\n}\r\n\r\nfn duration_to_time(d: Duration) -> f64 {\r\n 1e-9 * (d.subsec_nanos() as f64) + (d.as_secs() as f64)\r\n}\r\n\r\nfn run_simple() {\r\n let q = QuadBez::new((0.0, 0.0), (0.0, 0.5), (1.0, 1.0));\r\n let true_len = q.arclen(1e-13);\r\n for i in 0..20 {\r\n let n = 1 << i;\r\n let start_time = Instant::now();\r\n let mut est = 0.0;\r\n let mut last = q.start();\r\n let dt = (n as f64).recip();\r\n for j in 0..n {\r\n let t = ((j + 1) as f64) * dt;\r\n let p = q.eval(t);\r\n est += (p - last).hypot();\r\n last = p;\r\n }\r\n let elapsed = duration_to_time(start_time.elapsed());\r\n let err = true_len - est;\r\n if i > 0 {\r\n println!(\"{elapsed} {err}\");\r\n }\r\n }\r\n}\r\n\r\n/// Generate map data suitable for plotting in Gnuplot.\r\nfn main() {\r\n let mut n_subdiv = 0;\r\n let mut func: &dyn Fn(QuadBez) -> f64 = &gauss_arclen_3;\r\n for arg in env::args().skip(1) {\r\n if arg == \"gauss3\" {\r\n func = &gauss_arclen_3;\r\n } else if arg == \"gauss5\" {\r\n func = &gauss_arclen_5;\r\n } else if arg == \"gauss7\" {\r\n func = &gauss_arclen_7;\r\n } else if arg == \"gauss24\" {\r\n func = &gauss_arclen_24;\r\n } else if arg == \"l0\" {\r\n func = &calc_l0;\r\n } else if arg == \"simple\" {\r\n run_simple();\r\n return;\r\n } else if let Ok(n) = arg.parse() {\r\n n_subdiv = n;\r\n } else {\r\n println!(\"usage: arclen_accuracy [func] n_subdiv\");\r\n std::process::exit(1);\r\n }\r\n }\r\n let n = 400;\r\n let accuracy = 1e-6;\r\n for i in 0..=n {\r\n let x = 2.0 * (i as f64) * (n as f64).recip();\r\n for j in 0..=n {\r\n let y = 2.0 * (j as f64) * (n as f64).recip();\r\n let q = QuadBez::new((-1.0, 0.0), (x, y), (1.0, 0.0));\r\n let mut count = 0;\r\n let accurate_arclen = awesome_quad_arclen7(q, 1e-15, 0, &mut count);\r\n //count = 0;\r\n let start_time = Instant::now();\r\n //let est = awesome_quad_arclen7(q, accuracy, 0, &mut count);\r\n let est = quad_arclen_analytical(q);\r\n let elapsed = start_time.elapsed();\r\n //let est = gravesen_rec(&q, calc_l0(q), accuracy, 0, &mut count);\r\n //let est = with_subdiv(q, func, n_subdiv);\r\n let error = est - accurate_arclen;\r\n println!(\"{} {} {}\", x, y, (error.abs() + 1e-18).log10());\r\n //println!(\"{} {} {}\", x, y, count);\r\n //println!(\"{} {} {}\", x, y, elapsed.subsec_nanos());\r\n //let accurate_arclen = with_subdiv(q, &gauss_arclen_5, 8);\r\n //let error = est - accurate_arclen;\r\n /*\r\n let lc = (q.p2 - q.p0).hypot();\r\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\r\n let est_err = 1.0 * (lp - lc) * (0.5 * x * x + 0.1 * y * y).powf(16.0).tanh();\r\n println!(\"{} {} {}\", x, y, (est_err/error.abs() + 1e-15).log10());\r\n */\r\n }\r\n println!();\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "examples/circle.rs", "sha256": "509aeddeeb953066afe1219f683b9efe72d94109b3a29739e6b00b9ccc6c012a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Example of circle\r\n\r\n#[cfg(feature = \"std\")]\r\nfn main() {\r\n use kurbo::{Circle, Shape};\r\n\r\n let circle = Circle::new((400.0, 400.0), 380.0);\r\n println!(\"\");\r\n println!(\"\");\r\n println!(\"\");\r\n println!(\"\");\r\n let path = circle.to_path(1e-3).to_svg();\r\n println!(\" \", path);\r\n let path = circle.to_path(1.0).to_svg();\r\n println!(\" \", path);\r\n println!(\"\");\r\n println!(\"\");\r\n println!(\"\");\r\n}\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nfn main() {\r\n println!(\"This example requires the standard library\");\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "examples/cubic_arclen.rs", "sha256": "c25b3359ecebe925c10817c7a3a2d059e35f7ccb6b5ddf8fb12660a6b6e2f4d1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Research testbed for arclengths of cubic Bézier segments.\r\n\r\n// Lots of stuff is commented out or was just something to try.\r\n#![allow(unused)]\r\n#![allow(clippy::unreadable_literal)]\r\n#![allow(clippy::many_single_char_names)]\r\n\r\nuse kurbo::common::*;\r\nuse kurbo::{\r\n CubicBez, ParamCurve, ParamCurveArclen, ParamCurveCurvature, ParamCurveDeriv, Point, Vec2,\r\n};\r\n\r\n/// Calculate arclength using Gauss-Legendre quadrature using formula from Behdad\r\n/// in https://github.com/Pomax/BezierInfo-2/issues/77\r\nfn gauss_arclen_5(c: CubicBez) -> f64 {\r\n let v0 = (c.p1 - c.p0).hypot() * 0.15;\r\n let v1 = (-0.558983582205757 * c.p0.to_vec2()\r\n + 0.325650248872424 * c.p1.to_vec2()\r\n + 0.208983582205757 * c.p2.to_vec2()\r\n + 0.024349751127576 * c.p3.to_vec2())\r\n .hypot();\r\n let v2 = (c.p3 - c.p0 + (c.p2 - c.p1)).hypot() * 0.26666666666666666;\r\n let v3 = (-0.024349751127576 * c.p0.to_vec2()\r\n - 0.208983582205757 * c.p1.to_vec2()\r\n - 0.325650248872424 * c.p2.to_vec2()\r\n + 0.558983582205757 * c.p3.to_vec2())\r\n .hypot();\r\n let v4 = (c.p3 - c.p2).hypot() * 0.15;\r\n\r\n v0 + v1 + v2 + v3 + v4\r\n}\r\n\r\nfn gauss_arclen_7(c: C) -> f64 {\r\n c.gauss_arclen(GAUSS_LEGENDRE_COEFFS_7)\r\n}\r\n\r\nfn est_gauss5_error(c: CubicBez) -> f64 {\r\n let lc = (c.p3 - c.p0).hypot();\r\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\r\n\r\n let d2 = c.deriv().deriv();\r\n let d3 = d2.deriv();\r\n let lmi = 2.0 / (lp + lc);\r\n 7e-8 * (d3.eval(0.5).to_vec2().hypot() * lmi + 5.0 * d2.eval(0.5).to_vec2().hypot() * lmi)\r\n .powi(5)\r\n * lp\r\n}\r\n\r\nfn gauss_errnorm_n(c: C, coeffs: &[(f64, f64)]) -> f64\r\nwhere\r\n C::DerivResult: ParamCurveDeriv,\r\n{\r\n let d = c.deriv().deriv();\r\n coeffs\r\n .iter()\r\n .map(|(wi, xi)| wi * d.eval(0.5 * (xi + 1.0)).to_vec2().hypot2())\r\n .sum::()\r\n}\r\n\r\n// Squared L2 norm of the second derivative of the cubic.\r\nfn cubic_errnorm(c: CubicBez) -> f64 {\r\n let d = c.deriv().deriv();\r\n let dd = d.end() - d.start();\r\n d.start().to_vec2().hypot2() + d.start().to_vec2().dot(dd) + dd.hypot2() * (1.0 / 3.0)\r\n}\r\n\r\nfn est_gauss7_error(c: CubicBez) -> f64 {\r\n let lc = (c.p3 - c.p0).hypot();\r\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\r\n\r\n 8e-9 * (2.0 * cubic_errnorm(c) / lc.powi(2)).powi(6) * lp\r\n}\r\n\r\nfn gauss_arclen_9(c: C) -> f64 {\r\n c.gauss_arclen(GAUSS_LEGENDRE_COEFFS_9)\r\n}\r\n\r\nfn gauss_arclen_11(c: C) -> f64 {\r\n c.gauss_arclen(GAUSS_LEGENDRE_COEFFS_11)\r\n}\r\n\r\nfn est_gauss9_error(c: CubicBez) -> f64 {\r\n let lc = (c.p3 - c.p0).hypot();\r\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\r\n\r\n 1e-10 * (2.0 * cubic_errnorm(c) / lc.powi(2)).powi(8) * lp\r\n}\r\n\r\nfn est_gauss11_error(c: CubicBez) -> f64 {\r\n let lc = (c.p3 - c.p0).hypot();\r\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\r\n\r\n 1e-12 * (2.0 * cubic_errnorm(c) / lc.powi(2)).powi(11) * lp\r\n}\r\n\r\n// A new approach based on integrating local error.\r\nfn est_gauss11_error_2(c: CubicBez) -> f64 {\r\n let d = c.deriv();\r\n let d2 = d.deriv();\r\n GAUSS_LEGENDRE_COEFFS_11\r\n .iter()\r\n .map(|(wi, xi)| {\r\n wi * {\r\n let t = 0.5 * (xi + 1.0);\r\n let v = d.eval(t).to_vec2().hypot();\r\n let a2 = d2.eval(t).to_vec2().hypot2();\r\n a2.powi(3) / v.powi(5)\r\n }\r\n })\r\n .sum::()\r\n}\r\n\r\n#[allow(clippy::neg_cmp_op_on_partial_ord)]\r\nfn est_max_curvature(c: CubicBez) -> f64 {\r\n let n = 100;\r\n let mut max = 0.0;\r\n for i in 0..=n {\r\n let t = (i as f64) * (n as f64).recip();\r\n let k = c.curvature(t).abs();\r\n if !(k < max) {\r\n max = k;\r\n }\r\n }\r\n max\r\n}\r\n\r\nfn est_min_deriv_norm2(c: CubicBez) -> f64 {\r\n let d = c.deriv();\r\n let n = 100;\r\n let mut min = d.eval(1.0).to_vec2().hypot2();\r\n for i in 0..n {\r\n let t = (i as f64) * (n as f64).recip();\r\n min = min.min(d.eval(t).to_vec2().hypot2())\r\n }\r\n min\r\n}\r\n\r\nfn est_gauss11_error_3(c: CubicBez) -> f64 {\r\n let lc = (c.p3 - c.p0).hypot();\r\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\r\n let pc_err = (lp - lc) * 0.02;\r\n let ks = est_max_curvature(c) * lp;\r\n let est = ks.powi(3) * lp * 8e-9;\r\n if est < pc_err {\r\n est\r\n } else {\r\n pc_err\r\n }\r\n}\r\n\r\nfn est_gauss9_error_3(c: CubicBez) -> f64 {\r\n let lc = (c.p3 - c.p0).hypot();\r\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\r\n let pc_err = (lp - lc) * 0.02;\r\n let ks = est_max_curvature(c) * lp;\r\n let est = ks.powi(3) * lp * 5e-8;\r\n if est < pc_err {\r\n est\r\n } else {\r\n pc_err\r\n }\r\n}\r\n\r\n// A new approach based on integrating local error; the cost of evaluating the\r\n// error metric is likely to dominate unless the accuracy buys a lot of subdivisions.\r\nfn est_gauss9_error_2(c: CubicBez) -> f64 {\r\n let d = c.deriv();\r\n let d2 = d.deriv();\r\n let p = 10;\r\n GAUSS_LEGENDRE_COEFFS_9\r\n .iter()\r\n .map(|(wi, xi)| {\r\n wi * {\r\n let t = 0.5 * (xi + 1.0);\r\n let v = d.eval(t).to_vec2().hypot();\r\n let a = d2.eval(t).to_vec2().hypot();\r\n (1.0e-1 * a / v).tanh().powi(p) * v\r\n }\r\n })\r\n .sum::()\r\n * 3.0\r\n}\r\n\r\nfn my_arclen(c: CubicBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\r\n if depth == 16 || est_gauss5_error(c) < accuracy {\r\n *count += 1;\r\n gauss_arclen_5(c)\r\n } else {\r\n let (c0, c1) = c.subdivide();\r\n my_arclen(c0, accuracy * 0.5, depth + 1, count)\r\n + my_arclen(c1, accuracy * 0.5, depth + 1, count)\r\n }\r\n}\r\n\r\nfn my_arclen7(c: CubicBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\r\n if depth == 16 || est_gauss7_error(c) < accuracy {\r\n *count += 1;\r\n gauss_arclen_7(c)\r\n } else {\r\n let (c0, c1) = c.subdivide();\r\n my_arclen7(c0, accuracy * 0.5, depth + 1, count)\r\n + my_arclen7(c1, accuracy * 0.5, depth + 1, count)\r\n }\r\n}\r\n\r\n// Should make this generic instead of copy+paste, but we need only one when we're done.\r\nfn my_arclen9(c: CubicBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\r\n if depth == 16 || est_gauss9_error(c) < accuracy {\r\n *count += 1;\r\n gauss_arclen_9(c)\r\n } else {\r\n let (c0, c1) = c.subdivide();\r\n my_arclen9(c0, accuracy * 0.5, depth + 1, count)\r\n + my_arclen9(c1, accuracy * 0.5, depth + 1, count)\r\n }\r\n}\r\n\r\n// This doesn't help; we can't really get a more accurate error bound, so all this\r\n// does is overkill the accuracy.\r\nfn my_arclen11(c: CubicBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\r\n if depth == 16 || est_gauss9_error(c) < accuracy {\r\n *count += 1;\r\n gauss_arclen_11(c)\r\n } else {\r\n let (c0, c1) = c.subdivide();\r\n my_arclen11(c0, accuracy * 0.5, depth + 1, count)\r\n + my_arclen11(c1, accuracy * 0.5, depth + 1, count)\r\n }\r\n}\r\n\r\nfn randpt() -> Point {\r\n Point::new(rand::random(), rand::random())\r\n}\r\n\r\nfn randbez() -> CubicBez {\r\n CubicBez::new(randpt(), randpt(), randpt(), randpt())\r\n}\r\n\r\nfn main() {\r\n let accuracy = 1e-4;\r\n for _ in 0..10_000 {\r\n let c = randbez();\r\n let t: f64 = rand::random();\r\n let c = c.subsegment(0.0..t);\r\n //let accurate_arclen = c.arclen(1e-12);\r\n let mut count = 0;\r\n let accurate_arclen = my_arclen9(c, 1e-15, 0, &mut count);\r\n\r\n let est = gauss_arclen_9(c);\r\n let est_err = est_gauss9_error_3(c);\r\n let err = (accurate_arclen - est).abs();\r\n println!(\"{est_err} {err}\");\r\n\r\n /*\r\n let mut count = 0;\r\n let est = my_arclen9(c, accuracy, 0, &mut count);\r\n let err = (accurate_arclen - est).abs();\r\n println!(\"{err} {count}\");\r\n */\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "examples/ellipse.rs", "sha256": "be0cd0a6291a9600c98f25195597da74fa623a77f6c45433c9e68005835c3cc9", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Example of ellipse\r\n\r\n#[cfg(feature = \"std\")]\r\nfn main() {\r\n use kurbo::{Ellipse, Shape};\r\n use std::f64::consts::PI;\r\n\r\n let ellipse = Ellipse::new((400.0, 400.0), (200.0, 100.0), 0.25 * PI);\r\n println!(\"\");\r\n println!(\"\");\r\n println!(\"\");\r\n println!(\"\");\r\n let path = ellipse.to_path(1e-3).to_svg();\r\n println!(\" \", path);\r\n let path = ellipse.to_path(1.0).to_svg();\r\n println!(\" \", path);\r\n println!(\"\");\r\n println!(\"\");\r\n println!(\"\");\r\n}\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nfn main() {\r\n println!(\"This example requires the standard library\");\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "examples/fit_poly.rs", "sha256": "19cf29a6a9720c0c88320ee0abd3fbf3e4d83834433b103b8b5833dd76589548", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A test case for the cubic Bézier path simplifier, generating a very\r\n//! efficient and accurate SVG file to plot a quartic polynomial. This\r\n//! file was used to generate the image for the Wikipedia article on\r\n//! quartic polynomials.\r\n\r\nuse std::{io::Write, ops::Range};\r\n\r\nuse kurbo::{BezPath, CurveFitSample, ParamCurveFit, PathEl, Point, Vec2};\r\n\r\nstruct MyPoly;\r\n\r\nfn eval_mypoly(x: f64) -> f64 {\r\n (x + 4.) * (x + 1.) * (x - 1.) * (x - 3.) / 14. + 0.5\r\n}\r\n\r\nfn eval_mypoly_deriv(x: f64) -> f64 {\r\n (((4. * x + 3.) * x - 26.) * x - 1.) / 14.\r\n}\r\n\r\nimpl ParamCurveFit for MyPoly {\r\n fn sample_pt_deriv(&self, t: f64) -> (Point, Vec2) {\r\n const S: f64 = 336.;\r\n let x = 37. + t * S;\r\n let math_x = (x - 220.) / 40.;\r\n let y = -eval_mypoly(math_x) * 40. + 260.;\r\n let dx = S;\r\n let dy = -dx * eval_mypoly_deriv(math_x);\r\n (Point::new(x, y), Vec2::new(dx, dy))\r\n }\r\n\r\n fn sample_pt_tangent(&self, t: f64, _: f64) -> CurveFitSample {\r\n let (p, tangent) = self.sample_pt_deriv(t);\r\n CurveFitSample { p, tangent }\r\n }\r\n\r\n fn break_cusp(&self, _: Range) -> Option {\r\n None\r\n }\r\n}\r\n\r\npub fn to_svg_economical(path: &BezPath) -> String {\r\n let mut buffer = Vec::new();\r\n write_to(path, &mut buffer).unwrap();\r\n String::from_utf8(buffer).unwrap()\r\n}\r\n\r\n/// Write the SVG representation of this path to the provided buffer.\r\npub fn write_to(path: &BezPath, mut writer: W) -> std::io::Result<()> {\r\n for el in path.elements() {\r\n match *el {\r\n PathEl::MoveTo(p) => write!(writer, \"M{:.2} {:.2}\", p.x, p.y)?,\r\n PathEl::LineTo(p) => write!(writer, \"L{} {}\", p.x, p.y)?,\r\n PathEl::QuadTo(p1, p2) => write!(writer, \"Q{} {} {} {}\", p1.x, p1.y, p2.x, p2.y)?,\r\n PathEl::CurveTo(p1, p2, p3) => write!(\r\n writer,\r\n \"C{:.2} {:.2} {:.2} {:.2} {:.2} {:.2}\",\r\n p1.x, p1.y, p2.x, p2.y, p3.x, p3.y\r\n )?,\r\n PathEl::ClosePath => write!(writer, \"Z\")?,\r\n }\r\n }\r\n Ok(())\r\n}\r\n\r\nfn main() {\r\n println!(\"\");\r\n let path = kurbo::fit_to_bezpath_opt(&MyPoly, 0.1);\r\n println!(\r\n \" \",\r\n to_svg_economical(&path)\r\n );\r\n println!(\"\");\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "examples/offset.rs", "sha256": "450e161a17ece9269a8a6828c7e986b5bfbce2487511491d0b9e4ca90e4c80d6", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A simple example to show an offset curve of a cubic Bézier segment.\r\n\r\nuse kurbo::{offset::CubicOffset, CubicBez, Shape};\r\n\r\nfn main() {\r\n println!(\"\");\r\n let c = CubicBez::new((100., 100.), (150., 75.), (300., 50.), (400., 200.));\r\n println!(\r\n \" \",\r\n c.to_path(1e-9).to_svg()\r\n );\r\n for i in 1..=80 {\r\n let co = CubicOffset::new(c, i as f64 * 4.0);\r\n let path = kurbo::fit_to_bezpath_opt(&co, 1e-3);\r\n println!(\r\n \" \",\r\n path.to_svg()\r\n );\r\n }\r\n println!(\"\");\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "examples/quad_intersect.rs", "sha256": "1232d45f11b3bcabe836670bee040e8aaf21ac376d64c2a330361d14f3119899", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Test case that demonstrates quadratic/quadratic intersection using\r\n//! new quartic solver.\r\n\r\nuse arrayvec::ArrayVec;\r\nuse kurbo::{common::solve_quartic, ParamCurve, Point, QuadBez, Shape};\r\nuse rand::{thread_rng, Rng};\r\n\r\nfn rand_point() -> Point {\r\n let mut rng = thread_rng();\r\n Point::new(rng.gen_range(0.0..500.0), rng.gen_range(0.0..500.0))\r\n}\r\n\r\nfn rand_quad() -> QuadBez {\r\n QuadBez::new(rand_point(), rand_point(), rand_point())\r\n}\r\n\r\nstruct ImplicitQuad {\r\n x2: f64,\r\n xy: f64,\r\n y2: f64,\r\n x: f64,\r\n y: f64,\r\n c: f64,\r\n}\r\n\r\nimpl ImplicitQuad {\r\n fn from_quad(q: QuadBez) -> Self {\r\n let Point { x: ax, y: ay } = q.p0;\r\n let Point { x: bx, y: by } = q.p1;\r\n let Point { x: cx, y: cy } = q.p2;\r\n let (u0, u1, u2) = (by - cy, cx - bx, bx * cy - by * cx);\r\n let (v0, v1, v2) = (cy - ay, ax - cx, cx * ay - cy * ax);\r\n let (w0, w1, w2) = (ay - by, bx - ax, ax * by - ay * bx);\r\n ImplicitQuad {\r\n x2: 4. * u0 * w0 - v0 * v0,\r\n xy: 4. * (u0 * w1 + u1 * w0) - 2. * v0 * v1,\r\n y2: 4. * u1 * w1 - v1 * v1,\r\n x: 4. * (u0 * w2 + u2 * w0) - 2. * v0 * v2,\r\n y: 4. * (u1 * w2 + u2 * w1) - 2. * v1 * v2,\r\n c: 4. * u2 * w2 - v2 * v2,\r\n }\r\n }\r\n\r\n fn eval(&self, x: f64, y: f64) -> f64 {\r\n self.x2 * x * x + self.xy * x * y + self.y2 * y * y + self.x * x + self.y * y + self.c\r\n }\r\n}\r\n\r\nfn intersect_quads(q0: QuadBez, q1: QuadBez) -> ArrayVec {\r\n let iq = ImplicitQuad::from_quad(q0);\r\n let c = q1.p0.to_vec2();\r\n let b = (q1.p1 - q1.p0) * 2.0;\r\n let a = c - q1.p1.to_vec2() * 2.0 + q1.p2.to_vec2();\r\n let c0 = iq.eval(c.x, c.y);\r\n let c1 = iq.x * b.x\r\n + iq.y * b.y\r\n + 2. * iq.x2 * (b.x * c.x)\r\n + 2. * iq.y2 * (b.y * c.y)\r\n + iq.xy * (b.x * c.y + b.y * c.x);\r\n let c2 = iq.x * a.x\r\n + iq.y * a.y\r\n + iq.x2 * (2. * a.x * c.x + b.x * b.x)\r\n + iq.xy * (a.x * c.y + b.x * b.y + a.y * c.x)\r\n + iq.y2 * (2. * a.y * c.y + b.y * b.y);\r\n let c3 = iq.x2 * 2. * a.x * b.x + iq.xy * (a.x * b.y + b.x * a.y) + iq.y2 * 2. * a.y * b.y;\r\n let c4 = iq.x2 * a.x * a.x + iq.xy * a.x * a.y + iq.y2 * a.y * a.y;\r\n let ts = solve_quartic(c0, c1, c2, c3, c4);\r\n println!(\"ts: {:?}\", ts);\r\n ts\r\n}\r\n\r\nfn main() {\r\n let q0 = rand_quad();\r\n let q1 = rand_quad();\r\n println!(\r\n \" \",\r\n q0.to_path(1e-9).to_svg()\r\n );\r\n println!(\r\n \" \",\r\n q1.to_path(1e-9).to_svg()\r\n );\r\n for t in intersect_quads(q0, q1) {\r\n if (0.0..=1.0).contains(&t) {\r\n let p = q1.eval(t);\r\n println!(\" \", p.x, p.y);\r\n }\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "examples/simplify.rs", "sha256": "87b273a11bb4a4128f36fe50e18c38afeeef62d8691ab038af147af1db276ccc", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Test case showing simplification of a Bézier path. In this case, the\r\n//! source path is a simple mathematical function, but it should work for\r\n//! more general cases as well.\r\n\r\nuse kurbo::{BezPath, Point};\r\n\r\nfn plot_fn(f: &dyn Fn(f64) -> f64, d: &dyn Fn(f64) -> f64, xa: f64, xb: f64, n: usize) -> BezPath {\r\n let width = 800.;\r\n let dx = (xb - xa) / n as f64;\r\n let xs = width / (xb - xa);\r\n let ys = 250.;\r\n let y_origin = 300.;\r\n let plot = |x, y| Point::new((x - xa) * xs, y_origin - y * ys);\r\n let mut x0 = xa;\r\n let mut y0 = f(xa);\r\n let mut d0 = d(xa);\r\n let mut path = BezPath::new();\r\n path.move_to(plot(x0, y0));\r\n for i in 0..n {\r\n let x3 = xa + dx * (i + 1) as f64;\r\n let y3 = f(x3);\r\n let d3 = d(x3);\r\n let x1 = x0 + (1. / 3.) * dx;\r\n let x2 = x3 - (1. / 3.) * dx;\r\n let y1 = y0 + d0 * (1. / 3.) * dx;\r\n let y2 = y3 - d3 * (1. / 3.) * dx;\r\n path.curve_to(plot(x1, y1), plot(x2, y2), plot(x3, y3));\r\n x0 = x3;\r\n y0 = y3;\r\n d0 = d3;\r\n }\r\n path\r\n}\r\n\r\nfn main() {\r\n println!(\"\");\r\n let path = plot_fn(&|x| x.sin(), &|x| x.cos(), -8., 8., 20);\r\n println!(\r\n \" \",\r\n path.to_svg()\r\n );\r\n let simpl = kurbo::simplify::SimplifyBezPath::new(path);\r\n let simplified_path = kurbo::fit_to_bezpath_opt(&simpl, 0.1);\r\n println!(\r\n \" \",\r\n simplified_path.to_svg()\r\n );\r\n println!(\"\");\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/affine.rs", "sha256": "7d9dfd98195bb4561a6aa85df1f7d39468b69bba67710ebb3402bc89556ef4d5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Affine transforms.\r\n\r\nuse core::ops::{Mul, MulAssign};\r\n\r\nuse crate::{Point, Rect, Vec2};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// A 2D affine transform.\r\n#[derive(Clone, Copy, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct Affine([f64; 6]);\r\n\r\nimpl Affine {\r\n /// The identity transform.\r\n pub const IDENTITY: Affine = Affine::scale(1.0);\r\n\r\n /// A transform that is flipped on the y-axis. Useful for converting between\r\n /// y-up and y-down spaces.\r\n pub const FLIP_Y: Affine = Affine::new([1.0, 0., 0., -1.0, 0., 0.]);\r\n\r\n /// A transform that is flipped on the x-axis.\r\n pub const FLIP_X: Affine = Affine::new([-1.0, 0., 0., 1.0, 0., 0.]);\r\n\r\n /// Construct an affine transform from coefficients.\r\n ///\r\n /// If the coefficients are `(a, b, c, d, e, f)`, then the resulting\r\n /// transformation represents this augmented matrix:\r\n ///\r\n /// ```text\r\n /// | a c e |\r\n /// | b d f |\r\n /// | 0 0 1 |\r\n /// ```\r\n ///\r\n /// Note that this convention is transposed from PostScript and\r\n /// Direct2D, but is consistent with the\r\n /// [Wikipedia](https://en.wikipedia.org/wiki/Affine_transformation)\r\n /// formulation of affine transformation as augmented matrix. The\r\n /// idea is that `(A * B) * v == A * (B * v)`, where `*` is the\r\n /// [`Mul`](std::ops::Mul) trait.\r\n #[inline]\r\n pub const fn new(c: [f64; 6]) -> Affine {\r\n Affine(c)\r\n }\r\n\r\n /// An affine transform representing uniform scaling.\r\n #[inline]\r\n pub const fn scale(s: f64) -> Affine {\r\n Affine([s, 0.0, 0.0, s, 0.0, 0.0])\r\n }\r\n\r\n /// An affine transform representing non-uniform scaling\r\n /// with different scale values for x and y\r\n #[inline]\r\n pub const fn scale_non_uniform(s_x: f64, s_y: f64) -> Affine {\r\n Affine([s_x, 0.0, 0.0, s_y, 0.0, 0.0])\r\n }\r\n\r\n /// An affine transform representing rotation.\r\n ///\r\n /// The convention for rotation is that a positive angle rotates a\r\n /// positive X direction into positive Y. Thus, in a Y-down coordinate\r\n /// system (as is common for graphics), it is a clockwise rotation, and\r\n /// in Y-up (traditional for math), it is anti-clockwise.\r\n ///\r\n /// The angle, `th`, is expressed in radians.\r\n #[inline]\r\n pub fn rotate(th: f64) -> Affine {\r\n let (s, c) = th.sin_cos();\r\n Affine([c, s, -s, c, 0.0, 0.0])\r\n }\r\n\r\n /// An affine transform representing a rotation of `th` radians about `center`.\r\n ///\r\n /// See [Affine::rotate] for more info.\r\n #[inline]\r\n pub fn rotate_about(th: f64, center: Point) -> Affine {\r\n let center = center.to_vec2();\r\n Self::translate(-center)\r\n .then_rotate(th)\r\n .then_translate(center)\r\n }\r\n\r\n /// An affine transform representing translation.\r\n #[inline]\r\n pub fn translate>(p: V) -> Affine {\r\n let p = p.into();\r\n Affine([1.0, 0.0, 0.0, 1.0, p.x, p.y])\r\n }\r\n\r\n /// An affine transformation representing a skew.\r\n ///\r\n /// The skew_x and skew_y parameters represent skew factors for the\r\n /// horizontal and vertical directions, respectively.\r\n ///\r\n /// This is commonly used to generate a faux oblique transform for\r\n /// font rendering. In this case, you can slant the glyph 20 degrees\r\n /// clockwise in the horizontal direction (assuming a Y-up coordinate\r\n /// system):\r\n ///\r\n /// ```\r\n /// let oblique_transform = kurbo::Affine::skew(20f64.to_radians().tan(), 0.0);\r\n /// ```\r\n #[inline]\r\n pub fn skew(skew_x: f64, skew_y: f64) -> Affine {\r\n Affine([1.0, skew_y, skew_x, 1.0, 0.0, 0.0])\r\n }\r\n\r\n /// A rotation by `th` followed by `self`.\r\n ///\r\n /// Equivalent to `self * Affine::rotate(th)`\r\n #[inline]\r\n #[must_use]\r\n pub fn pre_rotate(self, th: f64) -> Self {\r\n self * Affine::rotate(th)\r\n }\r\n\r\n /// A rotation by `th` about `center` followed by `self`.\r\n ///\r\n /// Equivalent to `self * Affine::rotate_about(th)`\r\n #[inline]\r\n #[must_use]\r\n pub fn pre_rotate_about(self, th: f64, center: Point) -> Self {\r\n Affine::rotate_about(th, center) * self\r\n }\r\n\r\n /// A scale by `scale` followed by `self`.\r\n ///\r\n /// Equivalent to `self * Affine::scale(scale)`\r\n #[inline]\r\n #[must_use]\r\n pub fn pre_scale(self, scale: f64) -> Self {\r\n self * Affine::scale(scale)\r\n }\r\n\r\n /// A scale by `(scale_x, scale_y)` followed by `self`.\r\n ///\r\n /// Equivalent to `self * Affine::scale_non_uniform(scale_x, scale_y)`\r\n #[inline]\r\n #[must_use]\r\n pub fn pre_scale_non_uniform(self, scale_x: f64, scale_y: f64) -> Self {\r\n self * Affine::scale_non_uniform(scale_x, scale_y)\r\n }\r\n\r\n /// A translation of `trans` followed by `self`.\r\n ///\r\n /// Equivalent to `self * Affine::translate(trans)`\r\n #[inline]\r\n #[must_use]\r\n pub fn pre_translate(self, trans: Vec2) -> Self {\r\n self * Affine::translate(trans)\r\n }\r\n\r\n /// `self` followed by a rotation of `th`.\r\n ///\r\n /// Equivalent to `Affine::rotate(th) * self`\r\n #[inline]\r\n #[must_use]\r\n pub fn then_rotate(self, th: f64) -> Self {\r\n Affine::rotate(th) * self\r\n }\r\n\r\n /// `self` followed by a rotation of `th` about `center.\r\n ///\r\n /// Equivalent to `Affine::rotate_about(th, center) * self`\r\n #[inline]\r\n #[must_use]\r\n pub fn then_rotate_about(self, th: f64, center: Point) -> Self {\r\n Affine::rotate_about(th, center) * self\r\n }\r\n\r\n /// `self` followed by a scale of `scale`.\r\n ///\r\n /// Equivalent to `Affine::scale(scale) * self`\r\n #[inline]\r\n #[must_use]\r\n pub fn then_scale(self, scale: f64) -> Self {\r\n Affine::scale(scale) * self\r\n }\r\n\r\n /// `self` followed by a scale of `(scale_x, scale_y)`.\r\n ///\r\n /// Equivalent to `Affine::scale_non_uniform(scale_x, scale_y) * self`\r\n #[inline]\r\n #[must_use]\r\n pub fn then_scale_non_uniform(self, scale_x: f64, scale_y: f64) -> Self {\r\n Affine::scale_non_uniform(scale_x, scale_y) * self\r\n }\r\n\r\n /// `self` followed by a translation of `trans`.\r\n ///\r\n /// Equivalent to `Affine::translate(trans) * self`\r\n #[inline]\r\n #[must_use]\r\n pub fn then_translate(mut self, trans: Vec2) -> Self {\r\n self.0[4] += trans.x;\r\n self.0[5] += trans.y;\r\n self\r\n }\r\n\r\n /// Creates an affine transformation that takes the unit square to the given rectangle.\r\n ///\r\n /// Useful when you want to draw into the unit square but have your output fill any rectangle.\r\n /// In this case push the `Affine` onto the transform stack.\r\n pub fn map_unit_square(rect: Rect) -> Affine {\r\n Affine([rect.width(), 0., 0., rect.height(), rect.x0, rect.y0])\r\n }\r\n\r\n /// Get the coefficients of the transform.\r\n #[inline]\r\n pub fn as_coeffs(self) -> [f64; 6] {\r\n self.0\r\n }\r\n\r\n /// Compute the determinant of this transform.\r\n pub fn determinant(self) -> f64 {\r\n self.0[0] * self.0[3] - self.0[1] * self.0[2]\r\n }\r\n\r\n /// Compute the inverse transform.\r\n ///\r\n /// Produces NaN values when the determinant is zero.\r\n pub fn inverse(self) -> Affine {\r\n let inv_det = self.determinant().recip();\r\n Affine([\r\n inv_det * self.0[3],\r\n -inv_det * self.0[1],\r\n -inv_det * self.0[2],\r\n inv_det * self.0[0],\r\n inv_det * (self.0[2] * self.0[5] - self.0[3] * self.0[4]),\r\n inv_det * (self.0[1] * self.0[4] - self.0[0] * self.0[5]),\r\n ])\r\n }\r\n\r\n /// Compute the bounding box of a transformed rectangle.\r\n ///\r\n /// Returns the minimal `Rect` that encloses the given `Rect` after affine transformation.\r\n /// If the transform is axis-aligned, then this bounding box is \"tight\", in other words the\r\n /// returned `Rect` is the transformed rectangle.\r\n ///\r\n /// The returned rectangle always has non-negative width and height.\r\n pub fn transform_rect_bbox(self, rect: Rect) -> Rect {\r\n let p00 = self * Point::new(rect.x0, rect.y0);\r\n let p01 = self * Point::new(rect.x0, rect.y1);\r\n let p10 = self * Point::new(rect.x1, rect.y0);\r\n let p11 = self * Point::new(rect.x1, rect.y1);\r\n Rect::from_points(p00, p01).union(Rect::from_points(p10, p11))\r\n }\r\n\r\n /// Is this map finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n self.0[0].is_finite()\r\n && self.0[1].is_finite()\r\n && self.0[2].is_finite()\r\n && self.0[3].is_finite()\r\n && self.0[4].is_finite()\r\n && self.0[5].is_finite()\r\n }\r\n\r\n /// Is this map NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n self.0[0].is_nan()\r\n || self.0[1].is_nan()\r\n || self.0[2].is_nan()\r\n || self.0[3].is_nan()\r\n || self.0[4].is_nan()\r\n || self.0[5].is_nan()\r\n }\r\n\r\n /// Compute the singular value decomposition of the linear transformation (ignoring the\r\n /// translation).\r\n ///\r\n /// All non-degenerate linear transformations can be represented as\r\n ///\r\n /// 1. a rotation about the origin.\r\n /// 2. a scaling along the x and y axes\r\n /// 3. another rotation about the origin\r\n ///\r\n /// composed together. Decomposing a 2x2 matrix in this way is called a \"singular value\r\n /// decomposition\" and is written `U Σ V^T`, where U and V^T are orthogonal (rotations) and Σ\r\n /// is a diagonal matrix (a scaling).\r\n ///\r\n /// Since currently this function is used to calculate ellipse radii and rotation from an\r\n /// affine map on the unit circle, we don't calculate V^T, since a rotation of the unit (or\r\n /// any) circle about its center always results in the same circle. This is the reason that an\r\n /// ellipse mapped using an affine map is always an ellipse.\r\n ///\r\n /// Will return NaNs if the matrix (or equivalently the linear map) is singular.\r\n ///\r\n /// First part of the return tuple is the scaling, second part is the angle of rotation (in\r\n /// radians)\r\n #[inline]\r\n pub(crate) fn svd(self) -> (Vec2, f64) {\r\n let a = self.0[0];\r\n let a2 = a * a;\r\n let b = self.0[1];\r\n let b2 = b * b;\r\n let c = self.0[2];\r\n let c2 = c * c;\r\n let d = self.0[3];\r\n let d2 = d * d;\r\n let ab = a * b;\r\n let cd = c * d;\r\n let angle = 0.5 * (2.0 * (ab + cd)).atan2(a2 - b2 + c2 - d2);\r\n let s1 = a2 + b2 + c2 + d2;\r\n let s2 = ((a2 - b2 + c2 - d2).powi(2) + 4.0 * (ab + cd).powi(2)).sqrt();\r\n (\r\n Vec2 {\r\n x: (0.5 * (s1 + s2)).sqrt(),\r\n y: (0.5 * (s1 - s2)).sqrt(),\r\n },\r\n angle,\r\n )\r\n }\r\n\r\n /// Returns the translation part of this affine map (`(self.0[4], self.0[5])`).\r\n #[inline]\r\n pub fn translation(self) -> Vec2 {\r\n Vec2 {\r\n x: self.0[4],\r\n y: self.0[5],\r\n }\r\n }\r\n\r\n /// Replaces the translation portion of this affine map\r\n ///\r\n /// The translation can be seen as being applied after the linear part of the map.\r\n #[must_use]\r\n #[inline]\r\n pub fn with_translation(mut self, trans: Vec2) -> Affine {\r\n self.0[4] = trans.x;\r\n self.0[5] = trans.y;\r\n self\r\n }\r\n}\r\n\r\nimpl Default for Affine {\r\n #[inline]\r\n fn default() -> Affine {\r\n Affine::IDENTITY\r\n }\r\n}\r\n\r\nimpl Mul for Affine {\r\n type Output = Point;\r\n\r\n #[inline]\r\n fn mul(self, other: Point) -> Point {\r\n Point::new(\r\n self.0[0] * other.x + self.0[2] * other.y + self.0[4],\r\n self.0[1] * other.x + self.0[3] * other.y + self.0[5],\r\n )\r\n }\r\n}\r\n\r\nimpl Mul for Affine {\r\n type Output = Affine;\r\n\r\n #[inline]\r\n fn mul(self, other: Affine) -> Affine {\r\n Affine([\r\n self.0[0] * other.0[0] + self.0[2] * other.0[1],\r\n self.0[1] * other.0[0] + self.0[3] * other.0[1],\r\n self.0[0] * other.0[2] + self.0[2] * other.0[3],\r\n self.0[1] * other.0[2] + self.0[3] * other.0[3],\r\n self.0[0] * other.0[4] + self.0[2] * other.0[5] + self.0[4],\r\n self.0[1] * other.0[4] + self.0[3] * other.0[5] + self.0[5],\r\n ])\r\n }\r\n}\r\n\r\nimpl MulAssign for Affine {\r\n #[inline]\r\n fn mul_assign(&mut self, other: Affine) {\r\n *self = self.mul(other);\r\n }\r\n}\r\n\r\nimpl Mul for f64 {\r\n type Output = Affine;\r\n\r\n #[inline]\r\n fn mul(self, other: Affine) -> Affine {\r\n Affine([\r\n self * other.0[0],\r\n self * other.0[1],\r\n self * other.0[2],\r\n self * other.0[3],\r\n self * other.0[4],\r\n self * other.0[5],\r\n ])\r\n }\r\n}\r\n\r\n// Conversions to and from mint\r\n#[cfg(feature = \"mint\")]\r\nimpl From for mint::ColumnMatrix2x3 {\r\n #[inline]\r\n fn from(a: Affine) -> mint::ColumnMatrix2x3 {\r\n mint::ColumnMatrix2x3 {\r\n x: mint::Vector2 {\r\n x: a.0[0],\r\n y: a.0[1],\r\n },\r\n y: mint::Vector2 {\r\n x: a.0[2],\r\n y: a.0[3],\r\n },\r\n z: mint::Vector2 {\r\n x: a.0[4],\r\n y: a.0[5],\r\n },\r\n }\r\n }\r\n}\r\n\r\n#[cfg(feature = \"mint\")]\r\nimpl From> for Affine {\r\n #[inline]\r\n fn from(m: mint::ColumnMatrix2x3) -> Affine {\r\n Affine([m.x.x, m.x.y, m.y.x, m.y.y, m.z.x, m.z.y])\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{Affine, Point};\r\n use std::f64::consts::PI;\r\n\r\n fn assert_near(p0: Point, p1: Point) {\r\n assert!((p1 - p0).hypot() < 1e-9, \"{p0:?} != {p1:?}\");\r\n }\r\n\r\n #[test]\r\n fn affine_basic() {\r\n let p = Point::new(3.0, 4.0);\r\n\r\n assert_near(Affine::default() * p, p);\r\n assert_near(Affine::scale(2.0) * p, Point::new(6.0, 8.0));\r\n assert_near(Affine::rotate(0.0) * p, p);\r\n assert_near(Affine::rotate(PI / 2.0) * p, Point::new(-4.0, 3.0));\r\n assert_near(Affine::translate((5.0, 6.0)) * p, Point::new(8.0, 10.0));\r\n assert_near(Affine::skew(0.0, 0.0) * p, p);\r\n assert_near(Affine::skew(2.0, 4.0) * p, Point::new(11.0, 16.0));\r\n }\r\n\r\n #[test]\r\n fn affine_mul() {\r\n let a1 = Affine::new([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);\r\n let a2 = Affine::new([0.1, 1.2, 2.3, 3.4, 4.5, 5.6]);\r\n\r\n let px = Point::new(1.0, 0.0);\r\n let py = Point::new(0.0, 1.0);\r\n let pxy = Point::new(1.0, 1.0);\r\n assert_near(a1 * (a2 * px), (a1 * a2) * px);\r\n assert_near(a1 * (a2 * py), (a1 * a2) * py);\r\n assert_near(a1 * (a2 * pxy), (a1 * a2) * pxy);\r\n }\r\n\r\n #[test]\r\n fn affine_inv() {\r\n let a = Affine::new([0.1, 1.2, 2.3, 3.4, 4.5, 5.6]);\r\n let a_inv = a.inverse();\r\n\r\n let px = Point::new(1.0, 0.0);\r\n let py = Point::new(0.0, 1.0);\r\n let pxy = Point::new(1.0, 1.0);\r\n assert_near(a * (a_inv * px), px);\r\n assert_near(a * (a_inv * py), py);\r\n assert_near(a * (a_inv * pxy), pxy);\r\n assert_near(a_inv * (a * px), px);\r\n assert_near(a_inv * (a * py), py);\r\n assert_near(a_inv * (a * pxy), pxy);\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/arc.rs", "sha256": "0810fdbce31e6647603415622b017b4b0c5661d225d89eee9282f9f63a162e02", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! An ellipse arc.\r\n\r\nuse crate::{Affine, Ellipse, PathEl, Point, Rect, Shape, Vec2};\r\nuse core::{\r\n f64::consts::{FRAC_PI_2, PI},\r\n iter,\r\n ops::Mul,\r\n};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// A single arc segment.\r\n#[derive(Clone, Copy, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct Arc {\r\n /// The arc's centre point.\r\n pub center: Point,\r\n /// The arc's radii, where the vector's x-component is the radius in the\r\n /// positive x direction after applying `x_rotation`.\r\n pub radii: Vec2,\r\n /// The start angle in radians.\r\n pub start_angle: f64,\r\n /// The angle between the start and end of the arc, in radians.\r\n pub sweep_angle: f64,\r\n /// How much the arc is rotated, in radians.\r\n pub x_rotation: f64,\r\n}\r\n\r\nimpl Arc {\r\n /// Create a new `Arc`.\r\n pub fn new(\r\n center: impl Into,\r\n radii: impl Into,\r\n start_angle: f64,\r\n sweep_angle: f64,\r\n x_rotation: f64,\r\n ) -> Self {\r\n Self {\r\n center: center.into(),\r\n radii: radii.into(),\r\n start_angle,\r\n sweep_angle,\r\n x_rotation,\r\n }\r\n }\r\n\r\n /// Create an iterator generating Bezier path elements.\r\n ///\r\n /// The generated elements can be appended to an existing bezier path.\r\n pub fn append_iter(&self, tolerance: f64) -> ArcAppendIter {\r\n let sign = self.sweep_angle.signum();\r\n let scaled_err = self.radii.x.max(self.radii.y) / tolerance;\r\n // Number of subdivisions per ellipse based on error tolerance.\r\n // Note: this may slightly underestimate the error for quadrants.\r\n let n_err = (1.1163 * scaled_err).powf(1.0 / 6.0).max(3.999_999);\r\n let n = (n_err * self.sweep_angle.abs() * (1.0 / (2.0 * PI))).ceil();\r\n let angle_step = self.sweep_angle / n;\r\n let n = n as usize;\r\n let arm_len = (4.0 / 3.0) * (0.25 * angle_step).abs().tan() * sign;\r\n let angle0 = self.start_angle;\r\n let p0 = sample_ellipse(self.radii, self.x_rotation, angle0);\r\n\r\n ArcAppendIter {\r\n idx: 0,\r\n\r\n center: self.center,\r\n radii: self.radii,\r\n x_rotation: self.x_rotation,\r\n n,\r\n arm_len,\r\n angle_step,\r\n\r\n p0,\r\n angle0,\r\n }\r\n }\r\n\r\n /// Converts an Arc into a series of cubic bezier segments.\r\n ///\r\n /// Closure will be invoked for each segment.\r\n pub fn to_cubic_beziers

(self, tolerance: f64, mut p: P)\r\n where\r\n P: FnMut(Point, Point, Point),\r\n {\r\n let mut path = self.append_iter(tolerance);\r\n while let Some(PathEl::CurveTo(p1, p2, p3)) = path.next() {\r\n p(p1, p2, p3);\r\n }\r\n }\r\n}\r\n\r\n#[doc(hidden)]\r\npub struct ArcAppendIter {\r\n idx: usize,\r\n\r\n center: Point,\r\n radii: Vec2,\r\n x_rotation: f64,\r\n n: usize,\r\n arm_len: f64,\r\n angle_step: f64,\r\n\r\n p0: Vec2,\r\n angle0: f64,\r\n}\r\n\r\nimpl Iterator for ArcAppendIter {\r\n type Item = PathEl;\r\n\r\n fn next(&mut self) -> Option {\r\n if self.idx >= self.n {\r\n return None;\r\n }\r\n\r\n let angle1 = self.angle0 + self.angle_step;\r\n let p0 = self.p0;\r\n let p1 = p0\r\n + self.arm_len * sample_ellipse(self.radii, self.x_rotation, self.angle0 + FRAC_PI_2);\r\n let p3 = sample_ellipse(self.radii, self.x_rotation, angle1);\r\n let p2 =\r\n p3 - self.arm_len * sample_ellipse(self.radii, self.x_rotation, angle1 + FRAC_PI_2);\r\n\r\n self.angle0 = angle1;\r\n self.p0 = p3;\r\n self.idx += 1;\r\n\r\n Some(PathEl::CurveTo(\r\n self.center + p1,\r\n self.center + p2,\r\n self.center + p3,\r\n ))\r\n }\r\n}\r\n\r\n/// Take the ellipse radii, how the radii are rotated, and the sweep angle, and return a point on\r\n/// the ellipse.\r\nfn sample_ellipse(radii: Vec2, x_rotation: f64, angle: f64) -> Vec2 {\r\n let (angle_sin, angle_cos) = angle.sin_cos();\r\n let u = radii.x * angle_cos;\r\n let v = radii.y * angle_sin;\r\n rotate_pt(Vec2::new(u, v), x_rotation)\r\n}\r\n\r\n/// Rotate `pt` about the origin by `angle` radians.\r\nfn rotate_pt(pt: Vec2, angle: f64) -> Vec2 {\r\n let (angle_sin, angle_cos) = angle.sin_cos();\r\n Vec2::new(\r\n pt.x * angle_cos - pt.y * angle_sin,\r\n pt.x * angle_sin + pt.y * angle_cos,\r\n )\r\n}\r\n\r\nimpl Shape for Arc {\r\n type PathElementsIter<'iter> = iter::Chain, ArcAppendIter>;\r\n\r\n fn path_elements(&self, tolerance: f64) -> Self::PathElementsIter<'_> {\r\n let p0 = sample_ellipse(self.radii, self.x_rotation, self.start_angle);\r\n iter::once(PathEl::MoveTo(self.center + p0)).chain(self.append_iter(tolerance))\r\n }\r\n\r\n /// Note: shape isn't closed so area is not well defined.\r\n #[inline]\r\n fn area(&self) -> f64 {\r\n let Vec2 { x, y } = self.radii;\r\n PI * x * y\r\n }\r\n\r\n /// The perimeter of the ellipse.\r\n ///\r\n /// Note: Finding the perimeter of an ellipse is [fairly involved][wikipedia],\r\n /// so for now we just approximate by using the bezier curve representation.\r\n ///\r\n /// [wikipedia]: https://en.wikipedia.org/wiki/Ellipse#Circumference\r\n #[inline]\r\n fn perimeter(&self, accuracy: f64) -> f64 {\r\n self.path_segments(0.1).perimeter(accuracy)\r\n }\r\n\r\n /// Note: shape isn't closed, so a point's winding number is not well defined.\r\n #[inline]\r\n fn winding(&self, pt: Point) -> i32 {\r\n self.path_segments(0.1).winding(pt)\r\n }\r\n\r\n #[inline]\r\n fn bounding_box(&self) -> Rect {\r\n self.path_segments(0.1).bounding_box()\r\n }\r\n}\r\n\r\nimpl Mul for Affine {\r\n type Output = Arc;\r\n\r\n fn mul(self, arc: Arc) -> Self::Output {\r\n let ellipse = self * Ellipse::new(arc.center, arc.radii, arc.x_rotation);\r\n let center = ellipse.center();\r\n let (radii, rotation) = ellipse.radii_and_rotation();\r\n Arc {\r\n center,\r\n radii,\r\n x_rotation: rotation,\r\n start_angle: arc.start_angle,\r\n sweep_angle: arc.sweep_angle,\r\n }\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/bezpath.rs", "sha256": "35012ab6b710873b0f9f70cbaac470f080da518548b943387c5db0d7c9537d29", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Bézier paths (up to cubic).\r\n\r\n#![allow(clippy::many_single_char_names)]\r\n\r\nuse core::iter::{Extend, FromIterator};\r\nuse core::mem;\r\nuse core::ops::{Mul, Range};\r\n\r\nuse alloc::vec::Vec;\r\n\r\nuse arrayvec::ArrayVec;\r\n\r\nuse crate::common::{solve_cubic, solve_quadratic};\r\nuse crate::MAX_EXTREMA;\r\nuse crate::{\r\n Affine, CubicBez, Line, Nearest, ParamCurve, ParamCurveArclen, ParamCurveArea,\r\n ParamCurveExtrema, ParamCurveNearest, Point, QuadBez, Rect, Shape, TranslateScale, Vec2,\r\n};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// A Bézier path.\r\n///\r\n/// These docs assume basic familiarity with Bézier curves; for an introduction,\r\n/// see Pomax's wonderful [A Primer on Bézier Curves].\r\n///\r\n/// This path can contain lines, quadratics ([`QuadBez`]) and cubics\r\n/// ([`CubicBez`]), and may contain multiple subpaths.\r\n///\r\n/// # Elements and Segments\r\n///\r\n/// A Bézier path can be represented in terms of either 'elements' ([`PathEl`])\r\n/// or 'segments' ([`PathSeg`]). Elements map closely to how Béziers are\r\n/// generally used in PostScript-style drawing APIs; they can be thought of as\r\n/// instructions for drawing the path. Segments more directly describe the\r\n/// path itself, with each segment being an independent line or curve.\r\n///\r\n/// These different representations are useful in different contexts.\r\n/// For tasks like drawing, elements are a natural fit, but when doing\r\n/// hit-testing or subdividing, we need to have access to the segments.\r\n///\r\n/// Conceptually, a `BezPath` contains zero or more subpaths. Each subpath\r\n/// *always* begins with a `MoveTo`, then has zero or more `LineTo`, `QuadTo`,\r\n/// and `CurveTo` elements, and optionally ends with a `ClosePath`.\r\n///\r\n/// Internally, a `BezPath` is a list of [`PathEl`]s; as such it implements\r\n/// [`FromIterator`] and [`Extend`]:\r\n///\r\n/// ```\r\n/// use kurbo::{BezPath, Rect, Shape, Vec2};\r\n/// let accuracy = 0.1;\r\n/// let rect = Rect::from_origin_size((0., 0.,), (10., 10.));\r\n/// // these are equivalent\r\n/// let path1 = rect.to_path(accuracy);\r\n/// let path2: BezPath = rect.path_elements(accuracy).collect();\r\n///\r\n/// // extend a path with another path:\r\n/// let mut path = rect.to_path(accuracy);\r\n/// let shifted_rect = rect + Vec2::new(5.0, 10.0);\r\n/// path.extend(shifted_rect.to_path(accuracy));\r\n/// ```\r\n///\r\n/// You can iterate the elements of a `BezPath` with the [`iter`] method,\r\n/// and the segments with the [`segments`] method:\r\n///\r\n/// ```\r\n/// use kurbo::{BezPath, Line, PathEl, PathSeg, Point, Rect, Shape};\r\n/// let accuracy = 0.1;\r\n/// let rect = Rect::from_origin_size((0., 0.,), (10., 10.));\r\n/// // these are equivalent\r\n/// let path = rect.to_path(accuracy);\r\n/// let first_el = PathEl::MoveTo(Point::ZERO);\r\n/// let first_seg = PathSeg::Line(Line::new((0., 0.), (10., 0.)));\r\n/// assert_eq!(path.iter().next(), Some(first_el));\r\n/// assert_eq!(path.segments().next(), Some(first_seg));\r\n/// ```\r\n/// In addition, if you have some other type that implements\r\n/// `Iterator`, you can adapt that to an iterator of segments with\r\n/// the [`segments` free function].\r\n///\r\n/// # Advanced functionality\r\n///\r\n/// In addition to the basic API, there are several useful pieces of advanced\r\n/// functionality available on `BezPath`:\r\n///\r\n/// - [`flatten`] does Bézier flattening, converting a curve to a series of\r\n/// line segments\r\n/// - [`intersect_line`] computes intersections of a path with a line, useful\r\n/// for things like subdividing\r\n///\r\n/// [A Primer on Bézier Curves]: https://pomax.github.io/bezierinfo/\r\n/// [`iter`]: BezPath::iter\r\n/// [`segments`]: BezPath::segments\r\n/// [`flatten`]: BezPath::flatten\r\n/// [`intersect_line`]: PathSeg::intersect_line\r\n/// [`segments` free function]: segments\r\n/// [`FromIterator`]: std::iter::FromIterator\r\n/// [`Extend`]: std::iter::Extend\r\n#[derive(Clone, Default, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct BezPath(Vec);\r\n\r\n/// The element of a Bézier path.\r\n///\r\n/// A valid path has `MoveTo` at the beginning of each subpath.\r\n#[derive(Clone, Copy, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub enum PathEl {\r\n /// Move directly to the point without drawing anything, starting a new\r\n /// subpath.\r\n MoveTo(Point),\r\n /// Draw a line from the current location to the point.\r\n LineTo(Point),\r\n /// Draw a quadratic bezier using the current location and the two points.\r\n QuadTo(Point, Point),\r\n /// Draw a cubic bezier using the current location and the three points.\r\n CurveTo(Point, Point, Point),\r\n /// Close off the path.\r\n ClosePath,\r\n}\r\n\r\n/// A segment of a Bézier path.\r\n#[derive(Clone, Copy, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub enum PathSeg {\r\n /// A line segment.\r\n Line(Line),\r\n /// A quadratic bezier segment.\r\n Quad(QuadBez),\r\n /// A cubic bezier segment.\r\n Cubic(CubicBez),\r\n}\r\n\r\n/// An intersection of a [`Line`] and a [`PathSeg`].\r\n///\r\n/// This can be generated with the [`PathSeg::intersect_line`] method.\r\n#[derive(Debug, Clone, Copy)]\r\npub struct LineIntersection {\r\n /// The 'time' that the intersection occurs, on the line.\r\n ///\r\n /// This value is in the range 0..1.\r\n pub line_t: f64,\r\n\r\n /// The 'time' that the intersection occurs, on the path segment.\r\n ///\r\n /// This value is nominally in the range 0..1, although it may slightly exceed\r\n /// that range at the boundaries of segments.\r\n pub segment_t: f64,\r\n}\r\n\r\n/// The minimum distance between two Bézier curves.\r\npub struct MinDistance {\r\n /// The shortest distance between any two points on the two curves.\r\n pub distance: f64,\r\n /// The position of the nearest point on the first curve, as a parameter.\r\n ///\r\n /// To resolve this to a [`Point`], use [`ParamCurve::eval`].\r\n ///\r\n /// [`ParamCurve::eval`]: crate::ParamCurve::eval\r\n pub t1: f64,\r\n /// The position of the nearest point on the second curve, as a parameter.\r\n ///\r\n /// To resolve this to a [`Point`], use [`ParamCurve::eval`].\r\n ///\r\n /// [`ParamCurve::eval`]: crate::ParamCurve::eval\r\n pub t2: f64,\r\n}\r\n\r\nimpl BezPath {\r\n /// Create a new path.\r\n pub fn new() -> BezPath {\r\n Default::default()\r\n }\r\n\r\n /// Create a path from a vector of path elements.\r\n ///\r\n /// `BezPath` also implements `FromIterator`, so it works with `collect`:\r\n ///\r\n /// ```\r\n /// // a very contrived example:\r\n /// use kurbo::{BezPath, PathEl};\r\n ///\r\n /// let path = BezPath::new();\r\n /// let as_vec: Vec = path.into_iter().collect();\r\n /// let back_to_path: BezPath = as_vec.into_iter().collect();\r\n /// ```\r\n pub fn from_vec(v: Vec) -> BezPath {\r\n debug_assert!(\r\n v.first().is_none() || matches!(v.first(), Some(PathEl::MoveTo(_))),\r\n \"BezPath must begin with MoveTo\"\r\n );\r\n BezPath(v)\r\n }\r\n\r\n /// Removes the last [`PathEl`] from the path and returns it, or `None` if the path is empty.\r\n pub fn pop(&mut self) -> Option {\r\n self.0.pop()\r\n }\r\n\r\n /// Push a generic path element onto the path.\r\n pub fn push(&mut self, el: PathEl) {\r\n self.0.push(el);\r\n debug_assert!(\r\n matches!(self.0.first(), Some(PathEl::MoveTo(_))),\r\n \"BezPath must begin with MoveTo\"\r\n )\r\n }\r\n\r\n /// Push a \"move to\" element onto the path.\r\n pub fn move_to>(&mut self, p: P) {\r\n self.push(PathEl::MoveTo(p.into()));\r\n }\r\n\r\n /// Push a \"line to\" element onto the path.\r\n ///\r\n /// Will panic with a debug assert when the current subpath does not\r\n /// start with `move_to`.\r\n pub fn line_to>(&mut self, p: P) {\r\n debug_assert!(self.is_open_subpath(), \"no open subpath (missing MoveTo)\");\r\n self.push(PathEl::LineTo(p.into()));\r\n }\r\n\r\n /// Push a \"quad to\" element onto the path.\r\n ///\r\n /// Will panic with a debug assert when the current subpath does not\r\n /// start with `move_to`.\r\n pub fn quad_to>(&mut self, p1: P, p2: P) {\r\n debug_assert!(self.is_open_subpath(), \"no open subpath (missing MoveTo)\");\r\n self.push(PathEl::QuadTo(p1.into(), p2.into()));\r\n }\r\n\r\n /// Push a \"curve to\" element onto the path.\r\n ///\r\n /// Will panic with a debug assert when the current subpath does not\r\n /// start with `move_to`.\r\n pub fn curve_to>(&mut self, p1: P, p2: P, p3: P) {\r\n debug_assert!(self.is_open_subpath(), \"no open subpath (missing MoveTo)\");\r\n self.push(PathEl::CurveTo(p1.into(), p2.into(), p3.into()));\r\n }\r\n\r\n /// Push a \"close path\" element onto the path.\r\n ///\r\n /// Will panic with a debug assert when the current subpath does not\r\n /// start with `move_to`.\r\n pub fn close_path(&mut self) {\r\n debug_assert!(self.is_open_subpath(), \"no open subpath (missing MoveTo)\");\r\n self.push(PathEl::ClosePath);\r\n }\r\n\r\n fn is_open_subpath(&self) -> bool {\r\n !self.0.is_empty() && self.0.last() != Some(&PathEl::ClosePath)\r\n }\r\n\r\n /// Get the path elements.\r\n pub fn elements(&self) -> &[PathEl] {\r\n &self.0\r\n }\r\n\r\n /// Get the path elements (mut version).\r\n pub fn elements_mut(&mut self) -> &mut [PathEl] {\r\n &mut self.0\r\n }\r\n\r\n /// Returns an iterator over the path's elements.\r\n pub fn iter(&self) -> impl Iterator + Clone + '_ {\r\n self.0.iter().copied()\r\n }\r\n\r\n /// Iterate over the path segments.\r\n pub fn segments(&self) -> impl Iterator + Clone + '_ {\r\n segments(self.iter())\r\n }\r\n\r\n /// Shorten the path, keeping the first `len` elements.\r\n pub fn truncate(&mut self, len: usize) {\r\n self.0.truncate(len);\r\n }\r\n\r\n /// Flatten the path, invoking the callback repeatedly.\r\n ///\r\n /// Flattening is the action of approximating a curve with a succession of line segments.\r\n ///\r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// →\r\n /// \r\n /// \r\n ///\r\n /// The tolerance value controls the maximum distance between the curved input\r\n /// segments and their polyline approximations. (In technical terms, this is the\r\n /// Hausdorff distance). The algorithm attempts to bound this distance between\r\n /// by `tolerance` but this is not absolutely guaranteed. The appropriate value\r\n /// depends on the use, but for antialiased rendering, a value of 0.25 has been\r\n /// determined to give good results. The number of segments tends to scale as the\r\n /// inverse square root of tolerance.\r\n ///\r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n /// \r\n ///\r\n /// The callback will be called in order with each element of the generated\r\n /// path. Because the result is made of polylines, these will be straight-line\r\n /// path elements only, no curves.\r\n ///\r\n /// This algorithm is based on the blog post [Flattening quadratic Béziers]\r\n /// but with some refinements. For one, there is a more careful approximation\r\n /// at cusps. For two, the algorithm is extended to work with cubic Béziers\r\n /// as well, by first subdividing into quadratics and then computing the\r\n /// subdivision of each quadratic. However, as a clever trick, these quadratics\r\n /// are subdivided fractionally, and their endpoints are not included.\r\n ///\r\n /// TODO: write a paper explaining this in more detail.\r\n ///\r\n /// Note: the [`flatten`] function provides the same\r\n /// functionality but works with slices and other [`PathEl`] iterators.\r\n ///\r\n /// [Flattening quadratic Béziers]: https://raphlinus.github.io/graphics/curves/2019/12/23/flatten-quadbez.html\r\n pub fn flatten(&self, tolerance: f64, callback: impl FnMut(PathEl)) {\r\n flatten(self, tolerance, callback);\r\n }\r\n\r\n /// Get the segment at the given element index.\r\n ///\r\n /// If you need to access all segments, [`segments`] provides a better\r\n /// API. This is intended for random access of specific elements, for clients\r\n /// that require this specifically.\r\n ///\r\n /// **note**: This returns the segment that ends at the provided element\r\n /// index. In effect this means it is *1-indexed*: since no segment ends at\r\n /// the first element (which is presumed to be a `MoveTo`) `get_seg(0)` will\r\n /// always return `None`.\r\n pub fn get_seg(&self, ix: usize) -> Option {\r\n if ix == 0 || ix >= self.0.len() {\r\n return None;\r\n }\r\n let last = match self.0[ix - 1] {\r\n PathEl::MoveTo(p) => p,\r\n PathEl::LineTo(p) => p,\r\n PathEl::QuadTo(_, p2) => p2,\r\n PathEl::CurveTo(_, _, p3) => p3,\r\n _ => return None,\r\n };\r\n match self.0[ix] {\r\n PathEl::LineTo(p) => Some(PathSeg::Line(Line::new(last, p))),\r\n PathEl::QuadTo(p1, p2) => Some(PathSeg::Quad(QuadBez::new(last, p1, p2))),\r\n PathEl::CurveTo(p1, p2, p3) => Some(PathSeg::Cubic(CubicBez::new(last, p1, p2, p3))),\r\n PathEl::ClosePath => self.0[..ix].iter().rev().find_map(|el| match *el {\r\n PathEl::MoveTo(start) if start != last => {\r\n Some(PathSeg::Line(Line::new(last, start)))\r\n }\r\n _ => None,\r\n }),\r\n _ => None,\r\n }\r\n }\r\n\r\n /// Returns `true` if the path contains no segments.\r\n pub fn is_empty(&self) -> bool {\r\n self.0\r\n .iter()\r\n .all(|el| matches!(el, PathEl::MoveTo(..) | PathEl::ClosePath))\r\n }\r\n\r\n /// Apply an affine transform to the path.\r\n pub fn apply_affine(&mut self, affine: Affine) {\r\n for el in self.0.iter_mut() {\r\n *el = affine * (*el);\r\n }\r\n }\r\n\r\n /// Is this path finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n self.0.iter().all(|v| v.is_finite())\r\n }\r\n\r\n /// Is this path NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n self.0.iter().any(|v| v.is_nan())\r\n }\r\n}\r\n\r\nimpl FromIterator for BezPath {\r\n fn from_iter>(iter: T) -> Self {\r\n let el_vec: Vec<_> = iter.into_iter().collect();\r\n BezPath::from_vec(el_vec)\r\n }\r\n}\r\n\r\n/// Allow iteration over references to `BezPath`.\r\n///\r\n/// Note: the semantics are slightly different from simply iterating over the\r\n/// slice, as it returns `PathEl` items, rather than references.\r\nimpl<'a> IntoIterator for &'a BezPath {\r\n type Item = PathEl;\r\n type IntoIter = core::iter::Cloned>;\r\n\r\n fn into_iter(self) -> Self::IntoIter {\r\n self.elements().iter().cloned()\r\n }\r\n}\r\n\r\nimpl IntoIterator for BezPath {\r\n type Item = PathEl;\r\n type IntoIter = alloc::vec::IntoIter;\r\n\r\n fn into_iter(self) -> Self::IntoIter {\r\n self.0.into_iter()\r\n }\r\n}\r\n\r\nimpl Extend for BezPath {\r\n fn extend>(&mut self, iter: I) {\r\n self.0.extend(iter);\r\n }\r\n}\r\n\r\n/// Proportion of tolerance budget that goes to cubic to quadratic conversion.\r\nconst TO_QUAD_TOL: f64 = 0.1;\r\n\r\n/// Flatten the path, invoking the callback repeatedly.\r\n///\r\n/// See [`BezPath::flatten`] for more discussion.\r\n/// This signature is a bit more general, allowing flattening of `&[PathEl]` slices\r\n/// and other iterators yielding `PathEl`.\r\npub fn flatten(\r\n path: impl IntoIterator,\r\n tolerance: f64,\r\n mut callback: impl FnMut(PathEl),\r\n) {\r\n let sqrt_tol = tolerance.sqrt();\r\n let mut last_pt = None;\r\n let mut quad_buf = Vec::new();\r\n for el in path {\r\n match el {\r\n PathEl::MoveTo(p) => {\r\n last_pt = Some(p);\r\n callback(PathEl::MoveTo(p));\r\n }\r\n PathEl::LineTo(p) => {\r\n last_pt = Some(p);\r\n callback(PathEl::LineTo(p));\r\n }\r\n PathEl::QuadTo(p1, p2) => {\r\n if let Some(p0) = last_pt {\r\n let q = QuadBez::new(p0, p1, p2);\r\n let params = q.estimate_subdiv(sqrt_tol);\r\n let n = ((0.5 * params.val / sqrt_tol).ceil() as usize).max(1);\r\n let step = 1.0 / (n as f64);\r\n for i in 1..n {\r\n let u = (i as f64) * step;\r\n let t = q.determine_subdiv_t(¶ms, u);\r\n let p = q.eval(t);\r\n callback(PathEl::LineTo(p));\r\n }\r\n callback(PathEl::LineTo(p2));\r\n }\r\n last_pt = Some(p2);\r\n }\r\n PathEl::CurveTo(p1, p2, p3) => {\r\n if let Some(p0) = last_pt {\r\n let c = CubicBez::new(p0, p1, p2, p3);\r\n\r\n // Subdivide into quadratics, and estimate the number of\r\n // subdivisions required for each, summing to arrive at an\r\n // estimate for the number of subdivisions for the cubic.\r\n // Also retain these parameters for later.\r\n let iter = c.to_quads(tolerance * TO_QUAD_TOL);\r\n quad_buf.clear();\r\n quad_buf.reserve(iter.size_hint().0);\r\n let sqrt_remain_tol = sqrt_tol * (1.0 - TO_QUAD_TOL).sqrt();\r\n let mut sum = 0.0;\r\n for (_, _, q) in iter {\r\n let params = q.estimate_subdiv(sqrt_remain_tol);\r\n sum += params.val;\r\n quad_buf.push((q, params));\r\n }\r\n let n = ((0.5 * sum / sqrt_remain_tol).ceil() as usize).max(1);\r\n\r\n // Iterate through the quadratics, outputting the points of\r\n // subdivisions that fall within that quadratic.\r\n let step = sum / (n as f64);\r\n let mut i = 1;\r\n let mut val_sum = 0.0;\r\n for (q, params) in &quad_buf {\r\n let mut target = (i as f64) * step;\r\n let recip_val = params.val.recip();\r\n while target < val_sum + params.val {\r\n let u = (target - val_sum) * recip_val;\r\n let t = q.determine_subdiv_t(params, u);\r\n let p = q.eval(t);\r\n callback(PathEl::LineTo(p));\r\n i += 1;\r\n if i == n + 1 {\r\n break;\r\n }\r\n target = (i as f64) * step;\r\n }\r\n val_sum += params.val;\r\n }\r\n callback(PathEl::LineTo(p3));\r\n }\r\n last_pt = Some(p3);\r\n }\r\n PathEl::ClosePath => {\r\n last_pt = None;\r\n callback(PathEl::ClosePath);\r\n }\r\n }\r\n }\r\n}\r\n\r\nimpl Mul for Affine {\r\n type Output = PathEl;\r\n\r\n fn mul(self, other: PathEl) -> PathEl {\r\n match other {\r\n PathEl::MoveTo(p) => PathEl::MoveTo(self * p),\r\n PathEl::LineTo(p) => PathEl::LineTo(self * p),\r\n PathEl::QuadTo(p1, p2) => PathEl::QuadTo(self * p1, self * p2),\r\n PathEl::CurveTo(p1, p2, p3) => PathEl::CurveTo(self * p1, self * p2, self * p3),\r\n PathEl::ClosePath => PathEl::ClosePath,\r\n }\r\n }\r\n}\r\n\r\nimpl Mul for Affine {\r\n type Output = PathSeg;\r\n\r\n fn mul(self, other: PathSeg) -> PathSeg {\r\n match other {\r\n PathSeg::Line(line) => PathSeg::Line(self * line),\r\n PathSeg::Quad(quad) => PathSeg::Quad(self * quad),\r\n PathSeg::Cubic(cubic) => PathSeg::Cubic(self * cubic),\r\n }\r\n }\r\n}\r\n\r\nimpl Mul for Affine {\r\n type Output = BezPath;\r\n\r\n fn mul(self, other: BezPath) -> BezPath {\r\n BezPath(other.0.iter().map(|&el| self * el).collect())\r\n }\r\n}\r\n\r\nimpl<'a> Mul<&'a BezPath> for Affine {\r\n type Output = BezPath;\r\n\r\n fn mul(self, other: &BezPath) -> BezPath {\r\n BezPath(other.0.iter().map(|&el| self * el).collect())\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = PathEl;\r\n\r\n fn mul(self, other: PathEl) -> PathEl {\r\n match other {\r\n PathEl::MoveTo(p) => PathEl::MoveTo(self * p),\r\n PathEl::LineTo(p) => PathEl::LineTo(self * p),\r\n PathEl::QuadTo(p1, p2) => PathEl::QuadTo(self * p1, self * p2),\r\n PathEl::CurveTo(p1, p2, p3) => PathEl::CurveTo(self * p1, self * p2, self * p3),\r\n PathEl::ClosePath => PathEl::ClosePath,\r\n }\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = PathSeg;\r\n\r\n fn mul(self, other: PathSeg) -> PathSeg {\r\n match other {\r\n PathSeg::Line(line) => PathSeg::Line(self * line),\r\n PathSeg::Quad(quad) => PathSeg::Quad(self * quad),\r\n PathSeg::Cubic(cubic) => PathSeg::Cubic(self * cubic),\r\n }\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = BezPath;\r\n\r\n fn mul(self, other: BezPath) -> BezPath {\r\n BezPath(other.0.iter().map(|&el| self * el).collect())\r\n }\r\n}\r\n\r\nimpl<'a> Mul<&'a BezPath> for TranslateScale {\r\n type Output = BezPath;\r\n\r\n fn mul(self, other: &BezPath) -> BezPath {\r\n BezPath(other.0.iter().map(|&el| self * el).collect())\r\n }\r\n}\r\n\r\n/// Transform an iterator over path elements into one over path\r\n/// segments.\r\n///\r\n/// See also [`BezPath::segments`].\r\n/// This signature is a bit more general, allowing `&[PathEl]` slices\r\n/// and other iterators yielding `PathEl`.\r\npub fn segments(elements: I) -> Segments\r\nwhere\r\n I: IntoIterator,\r\n{\r\n Segments {\r\n elements: elements.into_iter(),\r\n start_last: None,\r\n }\r\n}\r\n\r\n/// An iterator that transforms path elements to path segments.\r\n///\r\n/// This struct is created by the [`segments`] function.\r\n#[derive(Clone)]\r\npub struct Segments> {\r\n elements: I,\r\n start_last: Option<(Point, Point)>,\r\n}\r\n\r\nimpl> Iterator for Segments {\r\n type Item = PathSeg;\r\n\r\n fn next(&mut self) -> Option {\r\n for el in &mut self.elements {\r\n // We first need to check whether this is the first\r\n // path element we see to fill in the start position.\r\n let (start, last) = self.start_last.get_or_insert_with(|| {\r\n let point = match el {\r\n PathEl::MoveTo(p) => p,\r\n PathEl::LineTo(p) => p,\r\n PathEl::QuadTo(_, p2) => p2,\r\n PathEl::CurveTo(_, _, p3) => p3,\r\n PathEl::ClosePath => panic!(\"Can't start a segment on a ClosePath\"),\r\n };\r\n (point, point)\r\n });\r\n\r\n return Some(match el {\r\n PathEl::MoveTo(p) => {\r\n *start = p;\r\n *last = p;\r\n continue;\r\n }\r\n PathEl::LineTo(p) => PathSeg::Line(Line::new(mem::replace(last, p), p)),\r\n PathEl::QuadTo(p1, p2) => {\r\n PathSeg::Quad(QuadBez::new(mem::replace(last, p2), p1, p2))\r\n }\r\n PathEl::CurveTo(p1, p2, p3) => {\r\n PathSeg::Cubic(CubicBez::new(mem::replace(last, p3), p1, p2, p3))\r\n }\r\n PathEl::ClosePath => {\r\n if *last != *start {\r\n PathSeg::Line(Line::new(mem::replace(last, *start), *start))\r\n } else {\r\n continue;\r\n }\r\n }\r\n });\r\n }\r\n\r\n None\r\n }\r\n}\r\n\r\nimpl> Segments {\r\n /// Here, `accuracy` specifies the accuracy for each Bézier segment. At worst,\r\n /// the total error is `accuracy` times the number of Bézier segments.\r\n\r\n // TODO: pub? Or is this subsumed by method of &[PathEl]?\r\n pub(crate) fn perimeter(self, accuracy: f64) -> f64 {\r\n self.map(|seg| seg.arclen(accuracy)).sum()\r\n }\r\n\r\n // Same\r\n pub(crate) fn area(self) -> f64 {\r\n self.map(|seg| seg.signed_area()).sum()\r\n }\r\n\r\n // Same\r\n pub(crate) fn winding(self, p: Point) -> i32 {\r\n self.map(|seg| seg.winding(p)).sum()\r\n }\r\n\r\n // Same\r\n pub(crate) fn bounding_box(self) -> Rect {\r\n let mut bbox: Option = None;\r\n for seg in self {\r\n let seg_bb = ParamCurveExtrema::bounding_box(&seg);\r\n if let Some(bb) = bbox {\r\n bbox = Some(bb.union(seg_bb));\r\n } else {\r\n bbox = Some(seg_bb)\r\n }\r\n }\r\n bbox.unwrap_or_default()\r\n }\r\n}\r\n\r\nimpl ParamCurve for PathSeg {\r\n fn eval(&self, t: f64) -> Point {\r\n match *self {\r\n PathSeg::Line(line) => line.eval(t),\r\n PathSeg::Quad(quad) => quad.eval(t),\r\n PathSeg::Cubic(cubic) => cubic.eval(t),\r\n }\r\n }\r\n\r\n fn subsegment(&self, range: Range) -> PathSeg {\r\n match *self {\r\n PathSeg::Line(line) => PathSeg::Line(line.subsegment(range)),\r\n PathSeg::Quad(quad) => PathSeg::Quad(quad.subsegment(range)),\r\n PathSeg::Cubic(cubic) => PathSeg::Cubic(cubic.subsegment(range)),\r\n }\r\n }\r\n}\r\n\r\nimpl ParamCurveArclen for PathSeg {\r\n fn arclen(&self, accuracy: f64) -> f64 {\r\n match *self {\r\n PathSeg::Line(line) => line.arclen(accuracy),\r\n PathSeg::Quad(quad) => quad.arclen(accuracy),\r\n PathSeg::Cubic(cubic) => cubic.arclen(accuracy),\r\n }\r\n }\r\n\r\n fn inv_arclen(&self, arclen: f64, accuracy: f64) -> f64 {\r\n match *self {\r\n PathSeg::Line(line) => line.inv_arclen(arclen, accuracy),\r\n PathSeg::Quad(quad) => quad.inv_arclen(arclen, accuracy),\r\n PathSeg::Cubic(cubic) => cubic.inv_arclen(arclen, accuracy),\r\n }\r\n }\r\n}\r\n\r\nimpl ParamCurveArea for PathSeg {\r\n fn signed_area(&self) -> f64 {\r\n match *self {\r\n PathSeg::Line(line) => line.signed_area(),\r\n PathSeg::Quad(quad) => quad.signed_area(),\r\n PathSeg::Cubic(cubic) => cubic.signed_area(),\r\n }\r\n }\r\n}\r\n\r\nimpl ParamCurveNearest for PathSeg {\r\n fn nearest(&self, p: Point, accuracy: f64) -> Nearest {\r\n match *self {\r\n PathSeg::Line(line) => line.nearest(p, accuracy),\r\n PathSeg::Quad(quad) => quad.nearest(p, accuracy),\r\n PathSeg::Cubic(cubic) => cubic.nearest(p, accuracy),\r\n }\r\n }\r\n}\r\n\r\nimpl ParamCurveExtrema for PathSeg {\r\n fn extrema(&self) -> ArrayVec {\r\n match *self {\r\n PathSeg::Line(line) => line.extrema(),\r\n PathSeg::Quad(quad) => quad.extrema(),\r\n PathSeg::Cubic(cubic) => cubic.extrema(),\r\n }\r\n }\r\n}\r\n\r\nimpl PathSeg {\r\n /// Get the [`PathEl`] that is equivalent to discarding the segment start point.\r\n pub fn as_path_el(&self) -> PathEl {\r\n match self {\r\n PathSeg::Line(line) => PathEl::LineTo(line.p1),\r\n PathSeg::Quad(q) => PathEl::QuadTo(q.p1, q.p2),\r\n PathSeg::Cubic(c) => PathEl::CurveTo(c.p1, c.p2, c.p3),\r\n }\r\n }\r\n\r\n /// Returns a new `PathSeg` describing the same path as `self`, but with\r\n /// the points reversed.\r\n pub fn reverse(&self) -> PathSeg {\r\n match self {\r\n PathSeg::Line(Line { p0, p1 }) => PathSeg::Line(Line::new(*p1, *p0)),\r\n PathSeg::Quad(q) => PathSeg::Quad(QuadBez::new(q.p2, q.p1, q.p0)),\r\n PathSeg::Cubic(c) => PathSeg::Cubic(CubicBez::new(c.p3, c.p2, c.p1, c.p0)),\r\n }\r\n }\r\n\r\n /// Convert this segment to a cubic bezier.\r\n pub fn to_cubic(&self) -> CubicBez {\r\n match *self {\r\n PathSeg::Line(Line { p0, p1 }) => CubicBez::new(p0, p0, p1, p1),\r\n PathSeg::Cubic(c) => c,\r\n PathSeg::Quad(q) => q.raise(),\r\n }\r\n }\r\n\r\n // Assumes split at extrema.\r\n fn winding_inner(&self, p: Point) -> i32 {\r\n let start = self.start();\r\n let end = self.end();\r\n let sign = if end.y > start.y {\r\n if p.y < start.y || p.y >= end.y {\r\n return 0;\r\n }\r\n -1\r\n } else if end.y < start.y {\r\n if p.y < end.y || p.y >= start.y {\r\n return 0;\r\n }\r\n 1\r\n } else {\r\n return 0;\r\n };\r\n match *self {\r\n PathSeg::Line(_line) => {\r\n if p.x < start.x.min(end.x) {\r\n return 0;\r\n }\r\n if p.x >= start.x.max(end.x) {\r\n return sign;\r\n }\r\n // line equation ax + by = c\r\n let a = end.y - start.y;\r\n let b = start.x - end.x;\r\n let c = a * start.x + b * start.y;\r\n if (a * p.x + b * p.y - c) * (sign as f64) <= 0.0 {\r\n sign\r\n } else {\r\n 0\r\n }\r\n }\r\n PathSeg::Quad(quad) => {\r\n let p1 = quad.p1;\r\n if p.x < start.x.min(end.x).min(p1.x) {\r\n return 0;\r\n }\r\n if p.x >= start.x.max(end.x).max(p1.x) {\r\n return sign;\r\n }\r\n let a = end.y - 2.0 * p1.y + start.y;\r\n let b = 2.0 * (p1.y - start.y);\r\n let c = start.y - p.y;\r\n for t in solve_quadratic(c, b, a) {\r\n if (0.0..=1.0).contains(&t) {\r\n let x = quad.eval(t).x;\r\n if p.x >= x {\r\n return sign;\r\n } else {\r\n return 0;\r\n }\r\n }\r\n }\r\n 0\r\n }\r\n PathSeg::Cubic(cubic) => {\r\n let p1 = cubic.p1;\r\n let p2 = cubic.p2;\r\n if p.x < start.x.min(end.x).min(p1.x).min(p2.x) {\r\n return 0;\r\n }\r\n if p.x >= start.x.max(end.x).max(p1.x).max(p2.x) {\r\n return sign;\r\n }\r\n let a = end.y - 3.0 * p2.y + 3.0 * p1.y - start.y;\r\n let b = 3.0 * (p2.y - 2.0 * p1.y + start.y);\r\n let c = 3.0 * (p1.y - start.y);\r\n let d = start.y - p.y;\r\n for t in solve_cubic(d, c, b, a) {\r\n if (0.0..=1.0).contains(&t) {\r\n let x = cubic.eval(t).x;\r\n if p.x >= x {\r\n return sign;\r\n } else {\r\n return 0;\r\n }\r\n }\r\n }\r\n 0\r\n }\r\n }\r\n }\r\n\r\n /// Compute the winding number contribution of a single segment.\r\n ///\r\n /// Cast a ray to the left and count intersections.\r\n fn winding(&self, p: Point) -> i32 {\r\n self.extrema_ranges()\r\n .into_iter()\r\n .map(|range| self.subsegment(range).winding_inner(p))\r\n .sum()\r\n }\r\n\r\n /// Compute intersections against a line.\r\n ///\r\n /// Returns a vector of the intersections. For each intersection,\r\n /// the `t` value of the segment and line are given.\r\n ///\r\n /// Note: This test is designed to be inclusive of points near the endpoints\r\n /// of the segment. This is so that testing a line against multiple\r\n /// contiguous segments of a path will be guaranteed to catch at least one\r\n /// of them. In such cases, use higher level logic to coalesce the hits\r\n /// (the `t` value may be slightly outside the range of 0..1).\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// # use kurbo::*;\r\n /// let seg = PathSeg::Line(Line::new((0.0, 0.0), (2.0, 0.0)));\r\n /// let line = Line::new((1.0, 2.0), (1.0, -2.0));\r\n /// let intersection = seg.intersect_line(line);\r\n /// assert_eq!(intersection.len(), 1);\r\n /// let intersection = intersection[0];\r\n /// assert_eq!(intersection.segment_t, 0.5);\r\n /// assert_eq!(intersection.line_t, 0.5);\r\n ///\r\n /// let point = seg.eval(intersection.segment_t);\r\n /// assert_eq!(point, Point::new(1.0, 0.0));\r\n /// ```\r\n pub fn intersect_line(&self, line: Line) -> ArrayVec {\r\n const EPSILON: f64 = 1e-9;\r\n let p0 = line.p0;\r\n let p1 = line.p1;\r\n let dx = p1.x - p0.x;\r\n let dy = p1.y - p0.y;\r\n let mut result = ArrayVec::new();\r\n match self {\r\n PathSeg::Line(l) => {\r\n let det = dx * (l.p1.y - l.p0.y) - dy * (l.p1.x - l.p0.x);\r\n if det.abs() < EPSILON {\r\n // Lines are coincident (or nearly so).\r\n return result;\r\n }\r\n let t = dx * (p0.y - l.p0.y) - dy * (p0.x - l.p0.x);\r\n // t = position on self\r\n let t = t / det;\r\n if (-EPSILON..=(1.0 + EPSILON)).contains(&t) {\r\n // u = position on probe line\r\n let u =\r\n (l.p0.x - p0.x) * (l.p1.y - l.p0.y) - (l.p0.y - p0.y) * (l.p1.x - l.p0.x);\r\n let u = u / det;\r\n if (0.0..=1.0).contains(&u) {\r\n result.push(LineIntersection::new(u, t));\r\n }\r\n }\r\n }\r\n PathSeg::Quad(q) => {\r\n // The basic technique here is to determine x and y as a quadratic polynomial\r\n // as a function of t. Then plug those values into the line equation for the\r\n // probe line (giving a sort of signed distance from the probe line) and solve\r\n // that for t.\r\n let (px0, px1, px2) = quadratic_bez_coefs(q.p0.x, q.p1.x, q.p2.x);\r\n let (py0, py1, py2) = quadratic_bez_coefs(q.p0.y, q.p1.y, q.p2.y);\r\n let c0 = dy * (px0 - p0.x) - dx * (py0 - p0.y);\r\n let c1 = dy * px1 - dx * py1;\r\n let c2 = dy * px2 - dx * py2;\r\n let invlen2 = (dx * dx + dy * dy).recip();\r\n for t in crate::common::solve_quadratic(c0, c1, c2) {\r\n if (-EPSILON..=(1.0 + EPSILON)).contains(&t) {\r\n let x = px0 + t * px1 + t * t * px2;\r\n let y = py0 + t * py1 + t * t * py2;\r\n let u = ((x - p0.x) * dx + (y - p0.y) * dy) * invlen2;\r\n if (0.0..=1.0).contains(&u) {\r\n result.push(LineIntersection::new(u, t));\r\n }\r\n }\r\n }\r\n }\r\n PathSeg::Cubic(c) => {\r\n // Same technique as above, but cubic polynomial.\r\n let (px0, px1, px2, px3) = cubic_bez_coefs(c.p0.x, c.p1.x, c.p2.x, c.p3.x);\r\n let (py0, py1, py2, py3) = cubic_bez_coefs(c.p0.y, c.p1.y, c.p2.y, c.p3.y);\r\n let c0 = dy * (px0 - p0.x) - dx * (py0 - p0.y);\r\n let c1 = dy * px1 - dx * py1;\r\n let c2 = dy * px2 - dx * py2;\r\n let c3 = dy * px3 - dx * py3;\r\n let invlen2 = (dx * dx + dy * dy).recip();\r\n for t in crate::common::solve_cubic(c0, c1, c2, c3) {\r\n if (-EPSILON..=(1.0 + EPSILON)).contains(&t) {\r\n let x = px0 + t * px1 + t * t * px2 + t * t * t * px3;\r\n let y = py0 + t * py1 + t * t * py2 + t * t * t * py3;\r\n let u = ((x - p0.x) * dx + (y - p0.y) * dy) * invlen2;\r\n if (0.0..=1.0).contains(&u) {\r\n result.push(LineIntersection::new(u, t));\r\n }\r\n }\r\n }\r\n }\r\n }\r\n result\r\n }\r\n\r\n /// Is this Bezier path finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n match self {\r\n PathSeg::Line(line) => line.is_finite(),\r\n PathSeg::Quad(quad_bez) => quad_bez.is_finite(),\r\n PathSeg::Cubic(cubic_bez) => cubic_bez.is_finite(),\r\n }\r\n }\r\n\r\n /// Is this Bezier path NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n match self {\r\n PathSeg::Line(line) => line.is_nan(),\r\n PathSeg::Quad(quad_bez) => quad_bez.is_nan(),\r\n PathSeg::Cubic(cubic_bez) => cubic_bez.is_nan(),\r\n }\r\n }\r\n\r\n #[inline]\r\n fn as_vec2_vec(&self) -> ArrayVec {\r\n let mut a = ArrayVec::new();\r\n match self {\r\n PathSeg::Line(l) => {\r\n a.push(l.p0.to_vec2());\r\n a.push(l.p1.to_vec2());\r\n }\r\n PathSeg::Quad(q) => {\r\n a.push(q.p0.to_vec2());\r\n a.push(q.p1.to_vec2());\r\n a.push(q.p2.to_vec2());\r\n }\r\n PathSeg::Cubic(c) => {\r\n a.push(c.p0.to_vec2());\r\n a.push(c.p1.to_vec2());\r\n a.push(c.p2.to_vec2());\r\n a.push(c.p3.to_vec2());\r\n }\r\n };\r\n a\r\n }\r\n\r\n /// Minimum distance between two PathSegs\r\n ///\r\n /// Returns a tuple of the distance, the path time `t1` of the closest point\r\n /// on the first PathSeg, and the path time `t2` of the closest point on the\r\n /// second PathSeg.\r\n pub fn min_dist(&self, other: PathSeg, accuracy: f64) -> MinDistance {\r\n let (distance, t1, t2) = crate::mindist::min_dist_param(\r\n &self.as_vec2_vec(),\r\n &other.as_vec2_vec(),\r\n (0.0, 1.0),\r\n (0.0, 1.0),\r\n accuracy,\r\n None,\r\n );\r\n MinDistance {\r\n distance: distance.sqrt(),\r\n t1,\r\n t2,\r\n }\r\n }\r\n\r\n /// Compute endpoint tangents of a path segment.\r\n ///\r\n /// This version is robust to the path segment not being a regular curve.\r\n pub(crate) fn tangents(&self) -> (Vec2, Vec2) {\r\n const EPS: f64 = 1e-12;\r\n match self {\r\n PathSeg::Line(l) => {\r\n let d = l.p1 - l.p0;\r\n (d, d)\r\n }\r\n PathSeg::Quad(q) => {\r\n let d01 = q.p1 - q.p0;\r\n let d0 = if d01.hypot2() > EPS { d01 } else { q.p2 - q.p0 };\r\n let d12 = q.p2 - q.p1;\r\n let d1 = if d12.hypot2() > EPS { d12 } else { q.p2 - q.p0 };\r\n (d0, d1)\r\n }\r\n PathSeg::Cubic(c) => {\r\n let d01 = c.p1 - c.p0;\r\n let d0 = if d01.hypot2() > EPS {\r\n d01\r\n } else {\r\n let d02 = c.p2 - c.p0;\r\n if d02.hypot2() > EPS {\r\n d02\r\n } else {\r\n c.p3 - c.p0\r\n }\r\n };\r\n let d23 = c.p3 - c.p2;\r\n let d1 = if d23.hypot2() > EPS {\r\n d23\r\n } else {\r\n let d13 = c.p3 - c.p1;\r\n if d13.hypot2() > EPS {\r\n d13\r\n } else {\r\n c.p3 - c.p0\r\n }\r\n };\r\n (d0, d1)\r\n }\r\n }\r\n }\r\n}\r\n\r\nimpl LineIntersection {\r\n fn new(line_t: f64, segment_t: f64) -> Self {\r\n LineIntersection { line_t, segment_t }\r\n }\r\n\r\n /// Is this line intersection finite?\r\n #[inline]\r\n pub fn is_finite(self) -> bool {\r\n self.line_t.is_finite() && self.segment_t.is_finite()\r\n }\r\n\r\n /// Is this line intersection NaN?\r\n #[inline]\r\n pub fn is_nan(self) -> bool {\r\n self.line_t.is_nan() || self.segment_t.is_nan()\r\n }\r\n}\r\n\r\n// Return polynomial coefficients given cubic bezier coordinates.\r\nfn quadratic_bez_coefs(x0: f64, x1: f64, x2: f64) -> (f64, f64, f64) {\r\n let p0 = x0;\r\n let p1 = 2.0 * x1 - 2.0 * x0;\r\n let p2 = x2 - 2.0 * x1 + x0;\r\n (p0, p1, p2)\r\n}\r\n\r\n// Return polynomial coefficients given cubic bezier coordinates.\r\nfn cubic_bez_coefs(x0: f64, x1: f64, x2: f64, x3: f64) -> (f64, f64, f64, f64) {\r\n let p0 = x0;\r\n let p1 = 3.0 * x1 - 3.0 * x0;\r\n let p2 = 3.0 * x2 - 6.0 * x1 + 3.0 * x0;\r\n let p3 = x3 - 3.0 * x2 + 3.0 * x1 - x0;\r\n (p0, p1, p2, p3)\r\n}\r\n\r\nimpl From for PathSeg {\r\n fn from(cubic_bez: CubicBez) -> PathSeg {\r\n PathSeg::Cubic(cubic_bez)\r\n }\r\n}\r\n\r\nimpl From for PathSeg {\r\n fn from(line: Line) -> PathSeg {\r\n PathSeg::Line(line)\r\n }\r\n}\r\n\r\nimpl From for PathSeg {\r\n fn from(quad_bez: QuadBez) -> PathSeg {\r\n PathSeg::Quad(quad_bez)\r\n }\r\n}\r\n\r\nimpl Shape for BezPath {\r\n type PathElementsIter<'iter> = core::iter::Copied>;\r\n\r\n fn path_elements(&self, _tolerance: f64) -> Self::PathElementsIter<'_> {\r\n self.0.iter().copied()\r\n }\r\n\r\n fn to_path(&self, _tolerance: f64) -> BezPath {\r\n self.clone()\r\n }\r\n\r\n fn into_path(self, _tolerance: f64) -> BezPath {\r\n self\r\n }\r\n\r\n /// Signed area.\r\n fn area(&self) -> f64 {\r\n self.elements().area()\r\n }\r\n\r\n fn perimeter(&self, accuracy: f64) -> f64 {\r\n self.elements().perimeter(accuracy)\r\n }\r\n\r\n /// Winding number of point.\r\n fn winding(&self, pt: Point) -> i32 {\r\n self.elements().winding(pt)\r\n }\r\n\r\n fn bounding_box(&self) -> Rect {\r\n self.elements().bounding_box()\r\n }\r\n\r\n fn as_path_slice(&self) -> Option<&[PathEl]> {\r\n Some(&self.0)\r\n }\r\n}\r\n\r\nimpl PathEl {\r\n /// Is this path element finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n match self {\r\n PathEl::MoveTo(p) => p.is_finite(),\r\n PathEl::LineTo(p) => p.is_finite(),\r\n PathEl::QuadTo(p, p2) => p.is_finite() && p2.is_finite(),\r\n PathEl::CurveTo(p, p2, p3) => p.is_finite() && p2.is_finite() && p3.is_finite(),\r\n PathEl::ClosePath => true,\r\n }\r\n }\r\n\r\n /// Is this path element NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n match self {\r\n PathEl::MoveTo(p) => p.is_nan(),\r\n PathEl::LineTo(p) => p.is_nan(),\r\n PathEl::QuadTo(p, p2) => p.is_nan() || p2.is_nan(),\r\n PathEl::CurveTo(p, p2, p3) => p.is_nan() || p2.is_nan() || p3.is_nan(),\r\n PathEl::ClosePath => false,\r\n }\r\n }\r\n}\r\n\r\n/// Implements [`Shape`] for a slice of [`PathEl`], provided that the first element of the slice is\r\n/// not a `PathEl::ClosePath`. If it is, several of these functions will panic.\r\n///\r\n/// If the slice starts with `LineTo`, `QuadTo`, or `CurveTo`, it will be treated as a `MoveTo`.\r\nimpl<'a> Shape for &'a [PathEl] {\r\n type PathElementsIter<'iter>\r\n\r\n = core::iter::Copied> where 'a: 'iter;\r\n\r\n #[inline]\r\n fn path_elements(&self, _tolerance: f64) -> Self::PathElementsIter<'_> {\r\n self.iter().copied()\r\n }\r\n\r\n fn to_path(&self, _tolerance: f64) -> BezPath {\r\n BezPath::from_vec(self.to_vec())\r\n }\r\n\r\n /// Signed area.\r\n fn area(&self) -> f64 {\r\n segments(self.iter().copied()).area()\r\n }\r\n\r\n fn perimeter(&self, accuracy: f64) -> f64 {\r\n segments(self.iter().copied()).perimeter(accuracy)\r\n }\r\n\r\n /// Winding number of point.\r\n fn winding(&self, pt: Point) -> i32 {\r\n segments(self.iter().copied()).winding(pt)\r\n }\r\n\r\n fn bounding_box(&self) -> Rect {\r\n segments(self.iter().copied()).bounding_box()\r\n }\r\n\r\n #[inline]\r\n fn as_path_slice(&self) -> Option<&[PathEl]> {\r\n Some(self)\r\n }\r\n}\r\n\r\n/// Implements [`Shape`] for an array of [`PathEl`], provided that the first element of the array is\r\n/// not a `PathEl::ClosePath`. If it is, several of these functions will panic.\r\n///\r\n/// If the array starts with `LineTo`, `QuadTo`, or `CurveTo`, it will be treated as a `MoveTo`.\r\nimpl Shape for [PathEl; N] {\r\n type PathElementsIter<'iter> = core::iter::Copied>;\r\n\r\n #[inline]\r\n fn path_elements(&self, _tolerance: f64) -> Self::PathElementsIter<'_> {\r\n self.iter().copied()\r\n }\r\n\r\n fn to_path(&self, _tolerance: f64) -> BezPath {\r\n BezPath::from_vec(self.to_vec())\r\n }\r\n\r\n /// Signed area.\r\n fn area(&self) -> f64 {\r\n segments(self.iter().copied()).area()\r\n }\r\n\r\n fn perimeter(&self, accuracy: f64) -> f64 {\r\n segments(self.iter().copied()).perimeter(accuracy)\r\n }\r\n\r\n /// Winding number of point.\r\n fn winding(&self, pt: Point) -> i32 {\r\n segments(self.iter().copied()).winding(pt)\r\n }\r\n\r\n fn bounding_box(&self) -> Rect {\r\n segments(self.iter().copied()).bounding_box()\r\n }\r\n\r\n #[inline]\r\n fn as_path_slice(&self) -> Option<&[PathEl]> {\r\n Some(self)\r\n }\r\n}\r\n\r\n/// An iterator for path segments.\r\npub struct PathSegIter {\r\n seg: PathSeg,\r\n ix: usize,\r\n}\r\n\r\nimpl Shape for PathSeg {\r\n type PathElementsIter<'iter> = PathSegIter;\r\n\r\n #[inline]\r\n fn path_elements(&self, _tolerance: f64) -> PathSegIter {\r\n PathSegIter { seg: *self, ix: 0 }\r\n }\r\n\r\n /// The area under the curve.\r\n ///\r\n /// We could just return 0, but this seems more useful.\r\n fn area(&self) -> f64 {\r\n self.signed_area()\r\n }\r\n\r\n #[inline]\r\n fn perimeter(&self, accuracy: f64) -> f64 {\r\n self.arclen(accuracy)\r\n }\r\n\r\n fn winding(&self, _pt: Point) -> i32 {\r\n 0\r\n }\r\n\r\n #[inline]\r\n fn bounding_box(&self) -> Rect {\r\n ParamCurveExtrema::bounding_box(self)\r\n }\r\n\r\n fn as_line(&self) -> Option {\r\n if let PathSeg::Line(line) = self {\r\n Some(*line)\r\n } else {\r\n None\r\n }\r\n }\r\n}\r\n\r\nimpl Iterator for PathSegIter {\r\n type Item = PathEl;\r\n\r\n fn next(&mut self) -> Option {\r\n self.ix += 1;\r\n match (self.ix, self.seg) {\r\n // yes I could do some fancy bindings thing here but... :shrug:\r\n (1, PathSeg::Line(seg)) => Some(PathEl::MoveTo(seg.p0)),\r\n (1, PathSeg::Quad(seg)) => Some(PathEl::MoveTo(seg.p0)),\r\n (1, PathSeg::Cubic(seg)) => Some(PathEl::MoveTo(seg.p0)),\r\n (2, PathSeg::Line(seg)) => Some(PathEl::LineTo(seg.p1)),\r\n (2, PathSeg::Quad(seg)) => Some(PathEl::QuadTo(seg.p1, seg.p2)),\r\n (2, PathSeg::Cubic(seg)) => Some(PathEl::CurveTo(seg.p1, seg.p2, seg.p3)),\r\n _ => None,\r\n }\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{Circle, DEFAULT_ACCURACY};\r\n\r\n use super::*;\r\n\r\n fn assert_approx_eq(x: f64, y: f64) {\r\n assert!((x - y).abs() < 1e-8, \"{x} != {y}\");\r\n }\r\n\r\n #[test]\r\n #[should_panic(expected = \"no open subpath\")]\r\n fn test_elements_to_segments_starts_on_closepath() {\r\n let mut path = BezPath::new();\r\n path.close_path();\r\n path.segments().next();\r\n }\r\n\r\n #[test]\r\n fn test_elements_to_segments_closepath_refers_to_last_moveto() {\r\n let mut path = BezPath::new();\r\n path.move_to((5.0, 5.0));\r\n path.line_to((15.0, 15.0));\r\n path.move_to((10.0, 10.0));\r\n path.line_to((15.0, 15.0));\r\n path.close_path();\r\n assert_eq!(\r\n path.segments().collect::>().last(),\r\n Some(&Line::new((15.0, 15.0), (10.0, 10.0)).into()),\r\n );\r\n }\r\n\r\n #[test]\r\n #[should_panic(expected = \"no open subpath\")]\r\n fn test_must_not_start_on_quad() {\r\n let mut path = BezPath::new();\r\n path.quad_to((5.0, 5.0), (10.0, 10.0));\r\n path.line_to((15.0, 15.0));\r\n path.close_path();\r\n }\r\n\r\n #[test]\r\n fn test_intersect_line() {\r\n let h_line = Line::new((0.0, 0.0), (100.0, 0.0));\r\n let v_line = Line::new((10.0, -10.0), (10.0, 10.0));\r\n let intersection = PathSeg::Line(h_line).intersect_line(v_line)[0];\r\n assert_approx_eq(intersection.segment_t, 0.1);\r\n assert_approx_eq(intersection.line_t, 0.5);\r\n\r\n let v_line = Line::new((-10.0, -10.0), (-10.0, 10.0));\r\n assert!(PathSeg::Line(h_line).intersect_line(v_line).is_empty());\r\n\r\n let v_line = Line::new((10.0, 10.0), (10.0, 20.0));\r\n assert!(PathSeg::Line(h_line).intersect_line(v_line).is_empty());\r\n }\r\n\r\n #[test]\r\n fn test_intersect_qad() {\r\n let q = QuadBez::new((0.0, -10.0), (10.0, 20.0), (20.0, -10.0));\r\n let v_line = Line::new((10.0, -10.0), (10.0, 10.0));\r\n assert_eq!(PathSeg::Quad(q).intersect_line(v_line).len(), 1);\r\n let intersection = PathSeg::Quad(q).intersect_line(v_line)[0];\r\n assert_approx_eq(intersection.segment_t, 0.5);\r\n assert_approx_eq(intersection.line_t, 0.75);\r\n\r\n let h_line = Line::new((0.0, 0.0), (100.0, 0.0));\r\n assert_eq!(PathSeg::Quad(q).intersect_line(h_line).len(), 2);\r\n }\r\n\r\n #[test]\r\n fn test_intersect_cubic() {\r\n let c = CubicBez::new((0.0, -10.0), (10.0, 20.0), (20.0, -20.0), (30.0, 10.0));\r\n let v_line = Line::new((10.0, -10.0), (10.0, 10.0));\r\n assert_eq!(PathSeg::Cubic(c).intersect_line(v_line).len(), 1);\r\n let intersection = PathSeg::Cubic(c).intersect_line(v_line)[0];\r\n assert_approx_eq(intersection.segment_t, 0.333333333);\r\n assert_approx_eq(intersection.line_t, 0.592592592);\r\n\r\n let h_line = Line::new((0.0, 0.0), (100.0, 0.0));\r\n assert_eq!(PathSeg::Cubic(c).intersect_line(h_line).len(), 3);\r\n }\r\n\r\n #[test]\r\n fn test_contains() {\r\n let mut path = BezPath::new();\r\n path.move_to((0.0, 0.0));\r\n path.line_to((1.0, 1.0));\r\n path.line_to((2.0, 0.0));\r\n path.close_path();\r\n assert_eq!(path.winding(Point::new(1.0, 0.5)), -1);\r\n assert!(path.contains(Point::new(1.0, 0.5)));\r\n }\r\n\r\n // get_seg(i) should produce the same results as path_segments().nth(i - 1).\r\n #[test]\r\n fn test_get_seg() {\r\n let circle = Circle::new((10.0, 10.0), 2.0).to_path(DEFAULT_ACCURACY);\r\n let segments = circle.path_segments(DEFAULT_ACCURACY).collect::>();\r\n let get_segs = (1..usize::MAX)\r\n .map_while(|i| circle.get_seg(i))\r\n .collect::>();\r\n assert_eq!(segments, get_segs);\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/circle.rs", "sha256": "9903550ae098c18380e1e4c34f3b3782ac1ba671a69e59b9331287ad1ab06d92", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Implementation of circle shape.\r\n\r\nuse core::{\r\n f64::consts::{FRAC_PI_2, PI},\r\n iter,\r\n ops::{Add, Mul, Sub},\r\n};\r\n\r\nuse crate::{Affine, Arc, ArcAppendIter, Ellipse, PathEl, Point, Rect, Shape, Vec2};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// A circle.\r\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct Circle {\r\n /// The center.\r\n pub center: Point,\r\n /// The radius.\r\n pub radius: f64,\r\n}\r\n\r\nimpl Circle {\r\n /// A new circle from center and radius.\r\n #[inline]\r\n pub fn new(center: impl Into, radius: f64) -> Circle {\r\n Circle {\r\n center: center.into(),\r\n radius,\r\n }\r\n }\r\n\r\n /// Create a [`CircleSegment`] by cutting out parts of this circle.\r\n pub fn segment(self, inner_radius: f64, start_angle: f64, sweep_angle: f64) -> CircleSegment {\r\n CircleSegment {\r\n center: self.center,\r\n outer_radius: self.radius,\r\n inner_radius,\r\n start_angle,\r\n sweep_angle,\r\n }\r\n }\r\n\r\n /// Is this circle finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n self.center.is_finite() && self.radius.is_finite()\r\n }\r\n\r\n /// Is this circle NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n self.center.is_nan() || self.radius.is_nan()\r\n }\r\n}\r\n\r\nimpl Add for Circle {\r\n type Output = Circle;\r\n\r\n #[inline]\r\n fn add(self, v: Vec2) -> Circle {\r\n Circle {\r\n center: self.center + v,\r\n radius: self.radius,\r\n }\r\n }\r\n}\r\n\r\nimpl Sub for Circle {\r\n type Output = Circle;\r\n\r\n #[inline]\r\n fn sub(self, v: Vec2) -> Circle {\r\n Circle {\r\n center: self.center - v,\r\n radius: self.radius,\r\n }\r\n }\r\n}\r\n\r\nimpl Mul for Affine {\r\n type Output = Ellipse;\r\n fn mul(self, other: Circle) -> Self::Output {\r\n self * Ellipse::from(other)\r\n }\r\n}\r\n\r\n#[doc(hidden)]\r\npub struct CirclePathIter {\r\n circle: Circle,\r\n delta_th: f64,\r\n arm_len: f64,\r\n ix: usize,\r\n n: usize,\r\n}\r\n\r\nimpl Shape for Circle {\r\n type PathElementsIter<'iter> = CirclePathIter;\r\n\r\n fn path_elements(&self, tolerance: f64) -> CirclePathIter {\r\n let scaled_err = self.radius.abs() / tolerance;\r\n let (n, arm_len) = if scaled_err < 1.0 / 1.9608e-4 {\r\n // Solution from http://spencermortensen.com/articles/bezier-circle/\r\n (4, 0.551915024494)\r\n } else {\r\n // This is empirically determined to fall within error tolerance.\r\n let n = (1.1163 * scaled_err).powf(1.0 / 6.0).ceil() as usize;\r\n // Note: this isn't minimum error, but it is simple and we can easily\r\n // estimate the error.\r\n let arm_len = (4.0 / 3.0) * (FRAC_PI_2 / (n as f64)).tan();\r\n (n, arm_len)\r\n };\r\n CirclePathIter {\r\n circle: *self,\r\n delta_th: 2.0 * PI / (n as f64),\r\n arm_len,\r\n ix: 0,\r\n n,\r\n }\r\n }\r\n\r\n #[inline]\r\n fn area(&self) -> f64 {\r\n PI * self.radius.powi(2)\r\n }\r\n\r\n #[inline]\r\n fn perimeter(&self, _accuracy: f64) -> f64 {\r\n (2.0 * PI * self.radius).abs()\r\n }\r\n\r\n fn winding(&self, pt: Point) -> i32 {\r\n if (pt - self.center).hypot2() < self.radius.powi(2) {\r\n 1\r\n } else {\r\n 0\r\n }\r\n }\r\n\r\n #[inline]\r\n fn bounding_box(&self) -> Rect {\r\n let r = self.radius.abs();\r\n let (x, y) = self.center.into();\r\n Rect::new(x - r, y - r, x + r, y + r)\r\n }\r\n\r\n fn as_circle(&self) -> Option {\r\n Some(*self)\r\n }\r\n}\r\n\r\nimpl Iterator for CirclePathIter {\r\n type Item = PathEl;\r\n\r\n fn next(&mut self) -> Option {\r\n let a = self.arm_len;\r\n let r = self.circle.radius;\r\n let (x, y) = self.circle.center.into();\r\n let ix = self.ix;\r\n self.ix += 1;\r\n if ix == 0 {\r\n Some(PathEl::MoveTo(Point::new(x + r, y)))\r\n } else if ix <= self.n {\r\n let th1 = self.delta_th * (ix as f64);\r\n let th0 = th1 - self.delta_th;\r\n let (s0, c0) = th0.sin_cos();\r\n let (s1, c1) = if ix == self.n {\r\n (0.0, 1.0)\r\n } else {\r\n th1.sin_cos()\r\n };\r\n Some(PathEl::CurveTo(\r\n Point::new(x + r * (c0 - a * s0), y + r * (s0 + a * c0)),\r\n Point::new(x + r * (c1 + a * s1), y + r * (s1 - a * c1)),\r\n Point::new(x + r * c1, y + r * s1),\r\n ))\r\n } else if ix == self.n + 1 {\r\n Some(PathEl::ClosePath)\r\n } else {\r\n None\r\n }\r\n }\r\n}\r\n\r\n/// A segment of a circle.\r\n///\r\n/// If `inner_radius > 0`, then the shape will be a doughnut segment.\r\npub struct CircleSegment {\r\n /// The center.\r\n pub center: Point,\r\n /// The outer radius.\r\n pub outer_radius: f64,\r\n /// The inner radius.\r\n pub inner_radius: f64,\r\n /// The angle to start drawing the segment (in radians).\r\n pub start_angle: f64,\r\n /// The arc length of the segment (in radians).\r\n pub sweep_angle: f64,\r\n}\r\n\r\nimpl CircleSegment {\r\n /// Create a `CircleSegment` out of its constituent parts.\r\n pub fn new(\r\n center: impl Into,\r\n outer_radius: f64,\r\n inner_radius: f64,\r\n start_angle: f64,\r\n sweep_angle: f64,\r\n ) -> Self {\r\n CircleSegment {\r\n center: center.into(),\r\n outer_radius,\r\n inner_radius,\r\n start_angle,\r\n sweep_angle,\r\n }\r\n }\r\n\r\n /// Is this circle segment finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n self.center.is_finite()\r\n && self.outer_radius.is_finite()\r\n && self.inner_radius.is_finite()\r\n && self.start_angle.is_finite()\r\n && self.sweep_angle.is_finite()\r\n }\r\n\r\n /// Is this circle segment NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n self.center.is_nan()\r\n || self.outer_radius.is_nan()\r\n || self.inner_radius.is_nan()\r\n || self.start_angle.is_nan()\r\n || self.sweep_angle.is_nan()\r\n }\r\n}\r\n\r\nimpl Add for CircleSegment {\r\n type Output = CircleSegment;\r\n\r\n #[inline]\r\n fn add(self, v: Vec2) -> Self {\r\n Self {\r\n center: self.center + v,\r\n ..self\r\n }\r\n }\r\n}\r\n\r\nimpl Sub for CircleSegment {\r\n type Output = CircleSegment;\r\n\r\n #[inline]\r\n fn sub(self, v: Vec2) -> Self {\r\n Self {\r\n center: self.center - v,\r\n ..self\r\n }\r\n }\r\n}\r\n\r\ntype CircleSegmentPathIter = iter::Chain<\r\n iter::Chain<\r\n iter::Chain, iter::Once>, ArcAppendIter>,\r\n iter::Once,\r\n >,\r\n ArcAppendIter,\r\n>;\r\n\r\nimpl Shape for CircleSegment {\r\n type PathElementsIter<'iter> = CircleSegmentPathIter;\r\n\r\n fn path_elements(&self, tolerance: f64) -> CircleSegmentPathIter {\r\n iter::once(PathEl::MoveTo(point_on_circle(\r\n self.center,\r\n self.inner_radius,\r\n self.start_angle,\r\n )))\r\n // First radius\r\n .chain(iter::once(PathEl::LineTo(point_on_circle(\r\n self.center,\r\n self.outer_radius,\r\n self.start_angle,\r\n ))))\r\n // outer arc\r\n .chain(\r\n Arc {\r\n center: self.center,\r\n radii: Vec2::new(self.outer_radius, self.outer_radius),\r\n start_angle: self.start_angle,\r\n sweep_angle: self.sweep_angle,\r\n x_rotation: 0.0,\r\n }\r\n .append_iter(tolerance),\r\n )\r\n // second radius\r\n .chain(iter::once(PathEl::LineTo(point_on_circle(\r\n self.center,\r\n self.inner_radius,\r\n self.start_angle + self.sweep_angle,\r\n ))))\r\n // inner arc\r\n .chain(\r\n Arc {\r\n center: self.center,\r\n radii: Vec2::new(self.inner_radius, self.inner_radius),\r\n start_angle: self.start_angle + self.sweep_angle,\r\n sweep_angle: -self.sweep_angle,\r\n x_rotation: 0.0,\r\n }\r\n .append_iter(tolerance),\r\n )\r\n }\r\n\r\n #[inline]\r\n fn area(&self) -> f64 {\r\n 0.5 * (self.outer_radius.powi(2) - self.inner_radius.powi(2)).abs() * self.sweep_angle\r\n }\r\n\r\n #[inline]\r\n fn perimeter(&self, _accuracy: f64) -> f64 {\r\n 2.0 * (self.outer_radius - self.inner_radius).abs()\r\n + self.sweep_angle * (self.inner_radius + self.outer_radius)\r\n }\r\n\r\n fn winding(&self, pt: Point) -> i32 {\r\n let angle = (pt - self.center).atan2();\r\n if angle < self.start_angle || angle > self.start_angle + self.sweep_angle {\r\n return 0;\r\n }\r\n let dist2 = (pt - self.center).hypot2();\r\n if (dist2 < self.outer_radius.powi(2) && dist2 > self.inner_radius.powi(2)) ||\r\n // case where outer_radius < inner_radius\r\n (dist2 < self.inner_radius.powi(2) && dist2 > self.outer_radius.powi(2))\r\n {\r\n 1\r\n } else {\r\n 0\r\n }\r\n }\r\n\r\n #[inline]\r\n fn bounding_box(&self) -> Rect {\r\n // todo this is currently not tight\r\n let r = self.inner_radius.max(self.outer_radius);\r\n let (x, y) = self.center.into();\r\n Rect::new(x - r, y - r, x + r, y + r)\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{Circle, Point, Shape};\r\n use std::f64::consts::PI;\r\n\r\n fn assert_approx_eq(x: f64, y: f64) {\r\n // Note: we might want to be more rigorous in testing the accuracy\r\n // of the conversion into Béziers. But this seems good enough.\r\n assert!((x - y).abs() < 1e-7, \"{x} != {y}\");\r\n }\r\n\r\n #[test]\r\n fn area_sign() {\r\n let center = Point::new(5.0, 5.0);\r\n let c = Circle::new(center, 5.0);\r\n assert_approx_eq(c.area(), 25.0 * PI);\r\n\r\n assert_eq!(c.winding(center), 1);\r\n\r\n let p = c.to_path(1e-9);\r\n assert_approx_eq(c.area(), p.area());\r\n assert_eq!(c.winding(center), p.winding(center));\r\n\r\n let c_neg_radius = Circle::new(center, -5.0);\r\n assert_approx_eq(c_neg_radius.area(), 25.0 * PI);\r\n\r\n assert_eq!(c_neg_radius.winding(center), 1);\r\n\r\n let p_neg_radius = c_neg_radius.to_path(1e-9);\r\n assert_approx_eq(c_neg_radius.area(), p_neg_radius.area());\r\n assert_eq!(c_neg_radius.winding(center), p_neg_radius.winding(center));\r\n }\r\n}\r\n\r\n#[inline]\r\nfn point_on_circle(center: Point, radius: f64, angle: f64) -> Point {\r\n let (angle_sin, angle_cos) = angle.sin_cos();\r\n center\r\n + Vec2 {\r\n x: angle_cos * radius,\r\n y: angle_sin * radius,\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/common.rs", "sha256": "5ebc1a293896953d44bc3c242c8fe7157b45d0a1434af9d1f67def00d8b865ca", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Common mathematical operations\r\n\r\n#![allow(missing_docs)]\r\n\r\nuse arrayvec::ArrayVec;\r\n\r\n/// Defines a trait that chooses between libstd or libm implementations of float methods.\r\nmacro_rules! define_float_funcs {\r\n ($(\r\n fn $name:ident(self $(,$arg:ident: $arg_ty:ty)*) -> $ret:ty\r\n => $lname:ident/$lfname:ident;\r\n )+) => {\r\n #[cfg(not(feature = \"std\"))]\r\n pub(crate) trait FloatFuncs : Sized {\r\n /// Special implementation for signum, because libm doesn't have it.\r\n fn signum(self) -> Self;\r\n\r\n $(fn $name(self $(,$arg: $arg_ty)*) -> $ret;)+\r\n }\r\n\r\n #[cfg(not(feature = \"std\"))]\r\n impl FloatFuncs for f32 {\r\n #[inline]\r\n fn signum(self) -> f32 {\r\n if self.is_nan() {\r\n f32::NAN\r\n } else {\r\n 1.0_f32.copysign(self)\r\n }\r\n }\r\n\r\n $(fn $name(self $(,$arg: $arg_ty)*) -> $ret {\r\n #[cfg(feature = \"libm\")]\r\n return libm::$lfname(self $(,$arg as _)*);\r\n\r\n #[cfg(not(feature = \"libm\"))]\r\n compile_error!(\"kurbo requires either the `std` or `libm` feature\")\r\n })+\r\n }\r\n\r\n #[cfg(not(feature = \"std\"))]\r\n impl FloatFuncs for f64 {\r\n #[inline]\r\n fn signum(self) -> f64 {\r\n if self.is_nan() {\r\n f64::NAN\r\n } else {\r\n 1.0_f64.copysign(self)\r\n }\r\n }\r\n\r\n $(fn $name(self $(,$arg: $arg_ty)*) -> $ret {\r\n #[cfg(feature = \"libm\")]\r\n return libm::$lname(self $(,$arg as _)*);\r\n\r\n #[cfg(not(feature = \"libm\"))]\r\n compile_error!(\"kurbo requires either the `std` or `libm` feature\")\r\n })+\r\n }\r\n }\r\n}\r\n\r\ndefine_float_funcs! {\r\n fn abs(self) -> Self => fabs/fabsf;\r\n fn acos(self) -> Self => acos/acosf;\r\n fn atan2(self, other: Self) -> Self => atan2/atan2f;\r\n fn cbrt(self) -> Self => cbrt/cbrtf;\r\n fn ceil(self) -> Self => ceil/ceilf;\r\n fn cos(self) -> Self => cos/cosf;\r\n fn copysign(self, sign: Self) -> Self => copysign/copysignf;\r\n fn floor(self) -> Self => floor/floorf;\r\n fn hypot(self, other: Self) -> Self => hypot/hypotf;\r\n fn ln(self) -> Self => log/logf;\r\n fn log2(self) -> Self => log2/log2f;\r\n fn mul_add(self, a: Self, b: Self) -> Self => fma/fmaf;\r\n fn powi(self, n: i32) -> Self => pow/powf;\r\n fn powf(self, n: Self) -> Self => pow/powf;\r\n fn round(self) -> Self => round/roundf;\r\n fn sin_cos(self) -> (Self, Self) => sincos/sincosf;\r\n fn sqrt(self) -> Self => sqrt/sqrtf;\r\n fn tan(self) -> Self => tan/tanf;\r\n fn trunc(self) -> Self => trunc/truncf;\r\n}\r\n\r\n/// Adds convenience methods to `f32` and `f64`.\r\npub trait FloatExt {\r\n /// Rounds to the nearest integer away from zero,\r\n /// unless the provided value is already an integer.\r\n ///\r\n /// It is to `ceil` what `trunc` is to `floor`.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::common::FloatExt;\r\n ///\r\n /// let f = 3.7_f64;\r\n /// let g = 3.0_f64;\r\n /// let h = -3.7_f64;\r\n /// let i = -5.1_f32;\r\n ///\r\n /// assert_eq!(f.expand(), 4.0);\r\n /// assert_eq!(g.expand(), 3.0);\r\n /// assert_eq!(h.expand(), -4.0);\r\n /// assert_eq!(i.expand(), -6.0);\r\n /// ```\r\n fn expand(&self) -> T;\r\n}\r\n\r\nimpl FloatExt for f64 {\r\n #[inline]\r\n fn expand(&self) -> f64 {\r\n self.abs().ceil().copysign(*self)\r\n }\r\n}\r\n\r\nimpl FloatExt for f32 {\r\n #[inline]\r\n fn expand(&self) -> f32 {\r\n self.abs().ceil().copysign(*self)\r\n }\r\n}\r\n\r\n/// Find real roots of cubic equation.\r\n///\r\n/// The implementation is not (yet) fully robust, but it does handle the case\r\n/// where `c3` is zero (in that case, solving the quadratic equation).\r\n///\r\n/// See: \r\n///\r\n/// That implementation is in turn based on Jim Blinn's \"How to Solve a Cubic\r\n/// Equation\", which is masterful.\r\n///\r\n/// Return values of x for which c0 + c1 x + c2 x² + c3 x³ = 0.\r\npub fn solve_cubic(c0: f64, c1: f64, c2: f64, c3: f64) -> ArrayVec {\r\n let mut result = ArrayVec::new();\r\n let c3_recip = c3.recip();\r\n const ONETHIRD: f64 = 1. / 3.;\r\n let scaled_c2 = c2 * (ONETHIRD * c3_recip);\r\n let scaled_c1 = c1 * (ONETHIRD * c3_recip);\r\n let scaled_c0 = c0 * c3_recip;\r\n if !(scaled_c0.is_finite() && scaled_c1.is_finite() && scaled_c2.is_finite()) {\r\n // cubic coefficient is zero or nearly so.\r\n return solve_quadratic(c0, c1, c2).iter().copied().collect();\r\n }\r\n let (c0, c1, c2) = (scaled_c0, scaled_c1, scaled_c2);\r\n // (d0, d1, d2) is called \"Delta\" in article\r\n let d0 = (-c2).mul_add(c2, c1);\r\n let d1 = (-c1).mul_add(c2, c0);\r\n let d2 = c2 * c0 - c1 * c1;\r\n // d is called \"Discriminant\"\r\n let d = 4.0 * d0 * d2 - d1 * d1;\r\n // de is called \"Depressed.x\", Depressed.y = d0\r\n let de = (-2.0 * c2).mul_add(d0, d1);\r\n // TODO: handle the cases where these intermediate results overflow.\r\n if d < 0.0 {\r\n let sq = (-0.25 * d).sqrt();\r\n let r = -0.5 * de;\r\n let t1 = (r + sq).cbrt() + (r - sq).cbrt();\r\n result.push(t1 - c2);\r\n } else if d == 0.0 {\r\n let t1 = (-d0).sqrt().copysign(de);\r\n result.push(t1 - c2);\r\n result.push(-2.0 * t1 - c2);\r\n } else {\r\n let th = d.sqrt().atan2(-de) * ONETHIRD;\r\n // (th_cos, th_sin) is called \"CubicRoot\"\r\n let (th_sin, th_cos) = th.sin_cos();\r\n // (r0, r1, r2) is called \"Root\"\r\n let r0 = th_cos;\r\n let ss3 = th_sin * 3.0f64.sqrt();\r\n let r1 = 0.5 * (-th_cos + ss3);\r\n let r2 = 0.5 * (-th_cos - ss3);\r\n let t = 2.0 * (-d0).sqrt();\r\n result.push(t.mul_add(r0, -c2));\r\n result.push(t.mul_add(r1, -c2));\r\n result.push(t.mul_add(r2, -c2));\r\n }\r\n result\r\n}\r\n\r\n/// Find real roots of quadratic equation.\r\n///\r\n/// Return values of x for which c0 + c1 x + c2 x² = 0.\r\n///\r\n/// This function tries to be quite numerically robust. If the equation\r\n/// is nearly linear, it will return the root ignoring the quadratic term;\r\n/// the other root might be out of representable range. In the degenerate\r\n/// case where all coefficients are zero, so that all values of x satisfy\r\n/// the equation, a single `0.0` is returned.\r\npub fn solve_quadratic(c0: f64, c1: f64, c2: f64) -> ArrayVec {\r\n let mut result = ArrayVec::new();\r\n let sc0 = c0 * c2.recip();\r\n let sc1 = c1 * c2.recip();\r\n if !sc0.is_finite() || !sc1.is_finite() {\r\n // c2 is zero or very small, treat as linear eqn\r\n let root = -c0 / c1;\r\n if root.is_finite() {\r\n result.push(root);\r\n } else if c0 == 0.0 && c1 == 0.0 {\r\n // Degenerate case\r\n result.push(0.0);\r\n }\r\n return result;\r\n }\r\n let arg = sc1 * sc1 - 4. * sc0;\r\n let root1 = if !arg.is_finite() {\r\n // Likely, calculation of sc1 * sc1 overflowed. Find one root\r\n // using sc1 x + x² = 0, other root as sc0 / root1.\r\n -sc1\r\n } else {\r\n if arg < 0.0 {\r\n return result;\r\n } else if arg == 0.0 {\r\n result.push(-0.5 * sc1);\r\n return result;\r\n }\r\n // See https://math.stackexchange.com/questions/866331\r\n -0.5 * (sc1 + arg.sqrt().copysign(sc1))\r\n };\r\n let root2 = sc0 / root1;\r\n if root2.is_finite() {\r\n // Sort just to be friendly and make results deterministic.\r\n if root2 > root1 {\r\n result.push(root1);\r\n result.push(root2);\r\n } else {\r\n result.push(root2);\r\n result.push(root1);\r\n }\r\n } else {\r\n result.push(root1);\r\n }\r\n result\r\n}\r\n\r\n/// Compute epsilon relative to coefficient.\r\n///\r\n/// A helper function from the Orellana and De Michele paper.\r\nfn eps_rel(raw: f64, a: f64) -> f64 {\r\n if a == 0.0 {\r\n raw.abs()\r\n } else {\r\n ((raw - a) / a).abs()\r\n }\r\n}\r\n\r\n/// Find real roots of a quartic equation.\r\n///\r\n/// This is a fairly literal implementation of the method described in:\r\n/// Algorithm 1010: Boosting Efficiency in Solving Quartic Equations with\r\n/// No Compromise in Accuracy, Orellana and De Michele, ACM\r\n/// Transactions on Mathematical Software, Vol. 46, No. 2, May 2020.\r\npub fn solve_quartic(c0: f64, c1: f64, c2: f64, c3: f64, c4: f64) -> ArrayVec {\r\n if c4 == 0.0 {\r\n return solve_cubic(c0, c1, c2, c3).iter().copied().collect();\r\n }\r\n if c0 == 0.0 {\r\n // Note: appends 0 root at end, doesn't sort. We might want to do that.\r\n return solve_cubic(c1, c2, c3, c4)\r\n .iter()\r\n .copied()\r\n .chain(Some(0.0))\r\n .collect();\r\n }\r\n let a = c3 / c4;\r\n let b = c2 / c4;\r\n let c = c1 / c4;\r\n let d = c0 / c4;\r\n if let Some(result) = solve_quartic_inner(a, b, c, d, false) {\r\n return result;\r\n }\r\n // Do polynomial rescaling\r\n const K_Q: f64 = 7.16e76;\r\n for rescale in [false, true] {\r\n if let Some(result) = solve_quartic_inner(\r\n a / K_Q,\r\n b / K_Q.powi(2),\r\n c / K_Q.powi(3),\r\n d / K_Q.powi(4),\r\n rescale,\r\n ) {\r\n return result.iter().map(|x| x * K_Q).collect();\r\n }\r\n }\r\n // Overflow happened, just return no roots.\r\n //println!(\"overflow, no roots returned\");\r\n Default::default()\r\n}\r\n\r\nfn solve_quartic_inner(a: f64, b: f64, c: f64, d: f64, rescale: bool) -> Option> {\r\n factor_quartic_inner(a, b, c, d, rescale).map(|quadratics| {\r\n quadratics\r\n .iter()\r\n .flat_map(|(a, b)| solve_quadratic(*b, *a, 1.0))\r\n .collect()\r\n })\r\n}\r\n\r\n/// Factor a quartic into two quadratics.\r\n///\r\n/// Attempt to factor a quartic equation into two quadratic equations. Returns None either if there\r\n/// is overflow (in which case rescaling might succeed) or the factorization would result in\r\n/// complex coefficients.\r\n///\r\n/// Discussion question: distinguish the two cases in return value?\r\npub fn factor_quartic_inner(\r\n a: f64,\r\n b: f64,\r\n c: f64,\r\n d: f64,\r\n rescale: bool,\r\n) -> Option> {\r\n let calc_eps_q = |a1, b1, a2, b2| {\r\n let eps_a = eps_rel(a1 + a2, a);\r\n let eps_b = eps_rel(b1 + a1 * a2 + b2, b);\r\n let eps_c = eps_rel(b1 * a2 + a1 * b2, c);\r\n eps_a + eps_b + eps_c\r\n };\r\n let calc_eps_t = |a1, b1, a2, b2| calc_eps_q(a1, b1, a2, b2) + eps_rel(b1 * b2, d);\r\n let disc = 9. * a * a - 24. * b;\r\n let s = if disc >= 0.0 {\r\n -2. * b / (3. * a + disc.sqrt().copysign(a))\r\n } else {\r\n -0.25 * a\r\n };\r\n let a_prime = a + 4. * s;\r\n let b_prime = b + 3. * s * (a + 2. * s);\r\n let c_prime = c + s * (2. * b + s * (3. * a + 4. * s));\r\n let d_prime = d + s * (c + s * (b + s * (a + s)));\r\n let g_prime;\r\n let h_prime;\r\n const K_C: f64 = 3.49e102;\r\n if rescale {\r\n let a_prime_s = a_prime / K_C;\r\n let b_prime_s = b_prime / K_C;\r\n let c_prime_s = c_prime / K_C;\r\n let d_prime_s = d_prime / K_C;\r\n g_prime = a_prime_s * c_prime_s - (4. / K_C) * d_prime_s - (1. / 3.) * b_prime_s.powi(2);\r\n h_prime = (a_prime_s * c_prime_s + (8. / K_C) * d_prime_s - (2. / 9.) * b_prime_s.powi(2))\r\n * (1. / 3.)\r\n * b_prime_s\r\n - c_prime_s * (c_prime_s / K_C)\r\n - a_prime_s.powi(2) * d_prime_s;\r\n } else {\r\n g_prime = a_prime * c_prime - 4. * d_prime - (1. / 3.) * b_prime.powi(2);\r\n h_prime =\r\n (a_prime * c_prime + 8. * d_prime - (2. / 9.) * b_prime.powi(2)) * (1. / 3.) * b_prime\r\n - c_prime.powi(2)\r\n - a_prime.powi(2) * d_prime;\r\n }\r\n if !(g_prime.is_finite() && h_prime.is_finite()) {\r\n return None;\r\n }\r\n let phi = depressed_cubic_dominant(g_prime, h_prime);\r\n let phi = if rescale { phi * K_C } else { phi };\r\n let l_1 = a * 0.5;\r\n let l_3 = (1. / 6.) * b + 0.5 * phi;\r\n let delt_2 = c - a * l_3;\r\n let d_2_cand_1 = (2. / 3.) * b - phi - l_1 * l_1;\r\n let l_2_cand_1 = 0.5 * delt_2 / d_2_cand_1;\r\n let l_2_cand_2 = 2. * (d - l_3 * l_3) / delt_2;\r\n let d_2_cand_2 = 0.5 * delt_2 / l_2_cand_2;\r\n let d_2_cand_3 = d_2_cand_1;\r\n let l_2_cand_3 = l_2_cand_2;\r\n let mut d_2_best = 0.0;\r\n let mut l_2_best = 0.0;\r\n let mut eps_l_best = 0.0;\r\n for (i, (d_2, l_2)) in [\r\n (d_2_cand_1, l_2_cand_1),\r\n (d_2_cand_2, l_2_cand_2),\r\n (d_2_cand_3, l_2_cand_3),\r\n ]\r\n .iter()\r\n .enumerate()\r\n {\r\n let eps_0 = eps_rel(d_2 + l_1 * l_1 + 2. * l_3, b);\r\n let eps_1 = eps_rel(2. * (d_2 * l_2 + l_1 * l_3), c);\r\n let eps_2 = eps_rel(d_2 * l_2 * l_2 + l_3 * l_3, d);\r\n let eps_l = eps_0 + eps_1 + eps_2;\r\n if i == 0 || eps_l < eps_l_best {\r\n d_2_best = *d_2;\r\n l_2_best = *l_2;\r\n eps_l_best = eps_l;\r\n }\r\n }\r\n let d_2 = d_2_best;\r\n let l_2 = l_2_best;\r\n let mut alpha_1;\r\n let mut beta_1;\r\n let mut alpha_2;\r\n let mut beta_2;\r\n //println!(\"phi = {}, d_2 = {}\", phi, d_2);\r\n if d_2 < 0.0 {\r\n let sq = (-d_2).sqrt();\r\n alpha_1 = l_1 + sq;\r\n beta_1 = l_3 + sq * l_2;\r\n alpha_2 = l_1 - sq;\r\n beta_2 = l_3 - sq * l_2;\r\n if beta_2.abs() < beta_1.abs() {\r\n beta_2 = d / beta_1;\r\n } else if beta_2.abs() > beta_1.abs() {\r\n beta_1 = d / beta_2;\r\n }\r\n let cands;\r\n if alpha_1.abs() != alpha_2.abs() {\r\n if alpha_1.abs() < alpha_2.abs() {\r\n let a1_cand_1 = (c - beta_1 * alpha_2) / beta_2;\r\n let a1_cand_2 = (b - beta_2 - beta_1) / alpha_2;\r\n let a1_cand_3 = a - alpha_2;\r\n // Note: cand 3 is first because it is infallible, simplifying logic\r\n cands = [\r\n (a1_cand_3, alpha_2),\r\n (a1_cand_1, alpha_2),\r\n (a1_cand_2, alpha_2),\r\n ];\r\n } else {\r\n let a2_cand_1 = (c - alpha_1 * beta_2) / beta_1;\r\n let a2_cand_2 = (b - beta_2 - beta_1) / alpha_1;\r\n let a2_cand_3 = a - alpha_1;\r\n cands = [\r\n (alpha_1, a2_cand_3),\r\n (alpha_1, a2_cand_1),\r\n (alpha_1, a2_cand_2),\r\n ];\r\n }\r\n let mut eps_q_best = 0.0;\r\n for (i, (a1, a2)) in cands.iter().enumerate() {\r\n if a1.is_finite() && a2.is_finite() {\r\n let eps_q = calc_eps_q(*a1, beta_1, *a2, beta_2);\r\n if i == 0 || eps_q < eps_q_best {\r\n alpha_1 = *a1;\r\n alpha_2 = *a2;\r\n eps_q_best = eps_q;\r\n }\r\n }\r\n }\r\n }\r\n } else if d_2 == 0.0 {\r\n let d_3 = d - l_3 * l_3;\r\n alpha_1 = l_1;\r\n beta_1 = l_3 + (-d_3).sqrt();\r\n alpha_2 = l_1;\r\n beta_2 = l_3 - (-d_3).sqrt();\r\n if beta_1.abs() > beta_2.abs() {\r\n beta_2 = d / beta_1;\r\n } else if beta_2.abs() > beta_1.abs() {\r\n beta_1 = d / beta_2;\r\n }\r\n // TODO: handle case d_2 is very small?\r\n } else {\r\n // This case means no real roots; in the most general case we might want\r\n // to factor into quadratic equations with complex coefficients.\r\n return None;\r\n }\r\n // Newton-Raphson iteration on alpha/beta coeff's.\r\n let mut eps_t = calc_eps_t(alpha_1, beta_1, alpha_2, beta_2);\r\n for _ in 0..8 {\r\n //println!(\"a1 {} b1 {} a2 {} b2 {}\", alpha_1, beta_1, alpha_2, beta_2);\r\n //println!(\"eps_t = {:e}\", eps_t);\r\n if eps_t == 0.0 {\r\n break;\r\n }\r\n let f_0 = beta_1 * beta_2 - d;\r\n let f_1 = beta_1 * alpha_2 + alpha_1 * beta_2 - c;\r\n let f_2 = beta_1 + alpha_1 * alpha_2 + beta_2 - b;\r\n let f_3 = alpha_1 + alpha_2 - a;\r\n let c_1 = alpha_1 - alpha_2;\r\n let det_j = beta_1 * beta_1 - beta_1 * (alpha_2 * c_1 + 2. * beta_2)\r\n + beta_2 * (alpha_1 * c_1 + beta_2);\r\n if det_j == 0.0 {\r\n break;\r\n }\r\n let inv = det_j.recip();\r\n let c_2 = beta_2 - beta_1;\r\n let c_3 = beta_1 * alpha_2 - alpha_1 * beta_2;\r\n let dz_0 = c_1 * f_0 + c_2 * f_1 + c_3 * f_2 - (beta_1 * c_2 + alpha_1 * c_3) * f_3;\r\n let dz_1 = (alpha_1 * c_1 + c_2) * f_0\r\n - beta_1 * c_1 * f_1\r\n - beta_1 * c_2 * f_2\r\n - beta_1 * c_3 * f_3;\r\n let dz_2 = -c_1 * f_0 - c_2 * f_1 - c_3 * f_2 + (alpha_2 * c_3 + beta_2 * c_2) * f_3;\r\n let dz_3 = -(alpha_2 * c_1 + c_2) * f_0\r\n + beta_2 * c_1 * f_1\r\n + beta_2 * c_2 * f_2\r\n + beta_2 * c_3 * f_3;\r\n let a1 = alpha_1 - inv * dz_0;\r\n let b1 = beta_1 - inv * dz_1;\r\n let a2 = alpha_2 - inv * dz_2;\r\n let b2 = beta_2 - inv * dz_3;\r\n let new_eps_t = calc_eps_t(a1, b1, a2, b2);\r\n // We break if the new eps is equal, paper keeps going\r\n if new_eps_t < eps_t {\r\n alpha_1 = a1;\r\n beta_1 = b1;\r\n alpha_2 = a2;\r\n beta_2 = b2;\r\n eps_t = new_eps_t;\r\n } else {\r\n //println!(\"new_eps_t got worse: {:e}\", new_eps_t);\r\n break;\r\n }\r\n }\r\n Some([(alpha_1, beta_1), (alpha_2, beta_2)].into())\r\n}\r\n\r\n/// Dominant root of depressed cubic x^3 + gx + h = 0.\r\n///\r\n/// Section 2.2 of Orellana and De Michele.\r\n// Note: some of the techniques in here might be useful to improve the\r\n// cubic solver, and vice versa.\r\nfn depressed_cubic_dominant(g: f64, h: f64) -> f64 {\r\n let q = (-1. / 3.) * g;\r\n let r = 0.5 * h;\r\n let phi_0;\r\n let k = if q.abs() < 1e102 && r.abs() < 1e154 {\r\n None\r\n } else if q.abs() < r.abs() {\r\n Some(1. - q * (q / r).powi(2))\r\n } else {\r\n Some(q.signum() * ((r / q).powi(2) / q - 1.0))\r\n };\r\n if k.is_some() && r == 0.0 {\r\n if g > 0.0 {\r\n phi_0 = 0.0;\r\n } else {\r\n phi_0 = (-g).sqrt();\r\n }\r\n } else if k.map(|k| k < 0.0).unwrap_or_else(|| r * r < q.powi(3)) {\r\n let t = if k.is_some() {\r\n r / q / q.sqrt()\r\n } else {\r\n r / q.powi(3).sqrt()\r\n };\r\n phi_0 = -2. * q.sqrt() * (t.abs().acos() * (1. / 3.)).cos().copysign(t);\r\n } else {\r\n let a = if let Some(k) = k {\r\n if q.abs() < r.abs() {\r\n -r * (1. + k.sqrt())\r\n } else {\r\n -r - (q.abs().sqrt() * q * k.sqrt()).copysign(r)\r\n }\r\n } else {\r\n -r - (r * r - q.powi(3)).sqrt().copysign(r)\r\n }\r\n .cbrt();\r\n let b = if a == 0.0 { 0.0 } else { q / a };\r\n phi_0 = a + b;\r\n }\r\n // Refine with Newton-Raphson iteration\r\n let mut x = phi_0;\r\n let mut f = (x * x + g) * x + h;\r\n //println!(\"g = {:e}, h = {:e}, x = {:e}, f = {:e}\", g, h, x, f);\r\n const EPS_M: f64 = 2.22045e-16;\r\n if f.abs() < EPS_M * x.powi(3).max(g * x).max(h) {\r\n return x;\r\n }\r\n for _ in 0..8 {\r\n let delt_f = 3. * x * x + g;\r\n if delt_f == 0.0 {\r\n break;\r\n }\r\n let new_x = x - f / delt_f;\r\n let new_f = (new_x * new_x + g) * new_x + h;\r\n //println!(\"delt_f = {:e}, new_f = {:e}\", delt_f, new_f);\r\n if new_f == 0.0 {\r\n return new_x;\r\n }\r\n if new_f.abs() >= f.abs() {\r\n break;\r\n }\r\n x = new_x;\r\n f = new_f;\r\n }\r\n x\r\n}\r\n\r\n/// Solve an arbitrary function for a zero-crossing.\r\n///\r\n/// This uses the [ITP method], as described in the paper\r\n/// [An Enhancement of the Bisection Method Average Performance Preserving Minmax Optimality].\r\n///\r\n/// The values of `ya` and `yb` are given as arguments rather than\r\n/// computed from `f`, as the values may already be known, or they may\r\n/// be less expensive to compute as special cases.\r\n///\r\n/// It is assumed that `ya < 0.0` and `yb > 0.0`, otherwise unexpected\r\n/// results may occur.\r\n///\r\n/// The value of `epsilon` must be larger than 2^-63 times `b - a`,\r\n/// otherwise integer overflow may occur. The `a` and `b` parameters\r\n/// represent the lower and upper bounds of the bracket searched for a\r\n/// solution.\r\n///\r\n/// The ITP method has tuning parameters. This implementation hardwires\r\n/// k2 to 2, both because it avoids an expensive floating point\r\n/// exponentiation, and because this value has been tested to work well\r\n/// with curve fitting problems.\r\n///\r\n/// The `n0` parameter controls the relative impact of the bisection and\r\n/// secant components. When it is 0, the number of iterations is\r\n/// guaranteed to be no more than the number required by bisection (thus,\r\n/// this method is strictly superior to bisection). However, when the\r\n/// function is smooth, a value of 1 gives the secant method more of a\r\n/// chance to engage, so the average number of iterations is likely\r\n/// lower, though there can be one more iteration than bisection in the\r\n/// worst case.\r\n///\r\n/// The `k1` parameter is harder to characterize, and interested users\r\n/// are referred to the paper, as well as encouraged to do empirical\r\n/// testing. To match the paper, a value of `0.2 / (b - a)` is\r\n/// suggested, and this is confirmed to give good results.\r\n///\r\n/// When the function is monotonic, the returned result is guaranteed to\r\n/// be within `epsilon` of the zero crossing. For more detailed analysis,\r\n/// again see the paper.\r\n///\r\n/// [ITP method]: https://en.wikipedia.org/wiki/ITP_Method\r\n/// [An Enhancement of the Bisection Method Average Performance Preserving Minmax Optimality]: https://dl.acm.org/doi/10.1145/3423597\r\n#[allow(clippy::too_many_arguments)]\r\npub fn solve_itp(\r\n mut f: impl FnMut(f64) -> f64,\r\n mut a: f64,\r\n mut b: f64,\r\n epsilon: f64,\r\n n0: usize,\r\n k1: f64,\r\n mut ya: f64,\r\n mut yb: f64,\r\n) -> f64 {\r\n let n1_2 = (((b - a) / epsilon).log2().ceil() - 1.0).max(0.0) as usize;\r\n let nmax = n0 + n1_2;\r\n let mut scaled_epsilon = epsilon * (1u64 << nmax) as f64;\r\n while b - a > 2.0 * epsilon {\r\n let x1_2 = 0.5 * (a + b);\r\n let r = scaled_epsilon - 0.5 * (b - a);\r\n let xf = (yb * a - ya * b) / (yb - ya);\r\n let sigma = x1_2 - xf;\r\n // This has k2 = 2 hardwired for efficiency.\r\n let delta = k1 * (b - a).powi(2);\r\n let xt = if delta <= (x1_2 - xf).abs() {\r\n xf + delta.copysign(sigma)\r\n } else {\r\n x1_2\r\n };\r\n let xitp = if (xt - x1_2).abs() <= r {\r\n xt\r\n } else {\r\n x1_2 - r.copysign(sigma)\r\n };\r\n let yitp = f(xitp);\r\n if yitp > 0.0 {\r\n b = xitp;\r\n yb = yitp;\r\n } else if yitp < 0.0 {\r\n a = xitp;\r\n ya = yitp;\r\n } else {\r\n return xitp;\r\n }\r\n scaled_epsilon *= 0.5;\r\n }\r\n 0.5 * (a + b)\r\n}\r\n\r\n/// A variant ITP solver that allows fallible functions.\r\n///\r\n/// Another difference: it returns the bracket that contains the root,\r\n/// which may be important if the function has a discontinuity.\r\n#[allow(clippy::too_many_arguments)]\r\npub(crate) fn solve_itp_fallible(\r\n mut f: impl FnMut(f64) -> Result,\r\n mut a: f64,\r\n mut b: f64,\r\n epsilon: f64,\r\n n0: usize,\r\n k1: f64,\r\n mut ya: f64,\r\n mut yb: f64,\r\n) -> Result<(f64, f64), E> {\r\n let n1_2 = (((b - a) / epsilon).log2().ceil() - 1.0).max(0.0) as usize;\r\n let nmax = n0 + n1_2;\r\n let mut scaled_epsilon = epsilon * (1u64 << nmax) as f64;\r\n while b - a > 2.0 * epsilon {\r\n let x1_2 = 0.5 * (a + b);\r\n let r = scaled_epsilon - 0.5 * (b - a);\r\n let xf = (yb * a - ya * b) / (yb - ya);\r\n let sigma = x1_2 - xf;\r\n // This has k2 = 2 hardwired for efficiency.\r\n let delta = k1 * (b - a).powi(2);\r\n let xt = if delta <= (x1_2 - xf).abs() {\r\n xf + delta.copysign(sigma)\r\n } else {\r\n x1_2\r\n };\r\n let xitp = if (xt - x1_2).abs() <= r {\r\n xt\r\n } else {\r\n x1_2 - r.copysign(sigma)\r\n };\r\n let yitp = f(xitp)?;\r\n if yitp > 0.0 {\r\n b = xitp;\r\n yb = yitp;\r\n } else if yitp < 0.0 {\r\n a = xitp;\r\n ya = yitp;\r\n } else {\r\n return Ok((xitp, xitp));\r\n }\r\n scaled_epsilon *= 0.5;\r\n }\r\n Ok((a, b))\r\n}\r\n\r\n// Tables of Legendre-Gauss quadrature coefficients, adapted from:\r\n// \r\n\r\npub const GAUSS_LEGENDRE_COEFFS_3: &[(f64, f64)] = &[\r\n (0.8888888888888888, 0.0000000000000000),\r\n (0.5555555555555556, -0.7745966692414834),\r\n (0.5555555555555556, 0.7745966692414834),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_4: &[(f64, f64)] = &[\r\n (0.6521451548625461, -0.3399810435848563),\r\n (0.6521451548625461, 0.3399810435848563),\r\n (0.3478548451374538, -0.8611363115940526),\r\n (0.3478548451374538, 0.8611363115940526),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_5: &[(f64, f64)] = &[\r\n (0.5688888888888889, 0.0000000000000000),\r\n (0.4786286704993665, -0.5384693101056831),\r\n (0.4786286704993665, 0.5384693101056831),\r\n (0.2369268850561891, -0.9061798459386640),\r\n (0.2369268850561891, 0.9061798459386640),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_6: &[(f64, f64)] = &[\r\n (0.3607615730481386, 0.6612093864662645),\r\n (0.3607615730481386, -0.6612093864662645),\r\n (0.4679139345726910, -0.2386191860831969),\r\n (0.4679139345726910, 0.2386191860831969),\r\n (0.1713244923791704, -0.9324695142031521),\r\n (0.1713244923791704, 0.9324695142031521),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_7: &[(f64, f64)] = &[\r\n (0.4179591836734694, 0.0000000000000000),\r\n (0.3818300505051189, 0.4058451513773972),\r\n (0.3818300505051189, -0.4058451513773972),\r\n (0.2797053914892766, -0.7415311855993945),\r\n (0.2797053914892766, 0.7415311855993945),\r\n (0.1294849661688697, -0.9491079123427585),\r\n (0.1294849661688697, 0.9491079123427585),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_8: &[(f64, f64)] = &[\r\n (0.3626837833783620, -0.1834346424956498),\r\n (0.3626837833783620, 0.1834346424956498),\r\n (0.3137066458778873, -0.5255324099163290),\r\n (0.3137066458778873, 0.5255324099163290),\r\n (0.2223810344533745, -0.7966664774136267),\r\n (0.2223810344533745, 0.7966664774136267),\r\n (0.1012285362903763, -0.9602898564975363),\r\n (0.1012285362903763, 0.9602898564975363),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_8_HALF: &[(f64, f64)] = &[\r\n (0.3626837833783620, 0.1834346424956498),\r\n (0.3137066458778873, 0.5255324099163290),\r\n (0.2223810344533745, 0.7966664774136267),\r\n (0.1012285362903763, 0.9602898564975363),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_9: &[(f64, f64)] = &[\r\n (0.3302393550012598, 0.0000000000000000),\r\n (0.1806481606948574, -0.8360311073266358),\r\n (0.1806481606948574, 0.8360311073266358),\r\n (0.0812743883615744, -0.9681602395076261),\r\n (0.0812743883615744, 0.9681602395076261),\r\n (0.3123470770400029, -0.3242534234038089),\r\n (0.3123470770400029, 0.3242534234038089),\r\n (0.2606106964029354, -0.6133714327005904),\r\n (0.2606106964029354, 0.6133714327005904),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_11: &[(f64, f64)] = &[\r\n (0.2729250867779006, 0.0000000000000000),\r\n (0.2628045445102467, -0.2695431559523450),\r\n (0.2628045445102467, 0.2695431559523450),\r\n (0.2331937645919905, -0.5190961292068118),\r\n (0.2331937645919905, 0.5190961292068118),\r\n (0.1862902109277343, -0.7301520055740494),\r\n (0.1862902109277343, 0.7301520055740494),\r\n (0.1255803694649046, -0.8870625997680953),\r\n (0.1255803694649046, 0.8870625997680953),\r\n (0.0556685671161737, -0.9782286581460570),\r\n (0.0556685671161737, 0.9782286581460570),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_16: &[(f64, f64)] = &[\r\n (0.1894506104550685, -0.0950125098376374),\r\n (0.1894506104550685, 0.0950125098376374),\r\n (0.1826034150449236, -0.2816035507792589),\r\n (0.1826034150449236, 0.2816035507792589),\r\n (0.1691565193950025, -0.4580167776572274),\r\n (0.1691565193950025, 0.4580167776572274),\r\n (0.1495959888165767, -0.6178762444026438),\r\n (0.1495959888165767, 0.6178762444026438),\r\n (0.1246289712555339, -0.7554044083550030),\r\n (0.1246289712555339, 0.7554044083550030),\r\n (0.0951585116824928, -0.8656312023878318),\r\n (0.0951585116824928, 0.8656312023878318),\r\n (0.0622535239386479, -0.9445750230732326),\r\n (0.0622535239386479, 0.9445750230732326),\r\n (0.0271524594117541, -0.9894009349916499),\r\n (0.0271524594117541, 0.9894009349916499),\r\n];\r\n\r\n// Just the positive x_i values.\r\npub const GAUSS_LEGENDRE_COEFFS_16_HALF: &[(f64, f64)] = &[\r\n (0.1894506104550685, 0.0950125098376374),\r\n (0.1826034150449236, 0.2816035507792589),\r\n (0.1691565193950025, 0.4580167776572274),\r\n (0.1495959888165767, 0.6178762444026438),\r\n (0.1246289712555339, 0.7554044083550030),\r\n (0.0951585116824928, 0.8656312023878318),\r\n (0.0622535239386479, 0.9445750230732326),\r\n (0.0271524594117541, 0.9894009349916499),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_24: &[(f64, f64)] = &[\r\n (0.1279381953467522, -0.0640568928626056),\r\n (0.1279381953467522, 0.0640568928626056),\r\n (0.1258374563468283, -0.1911188674736163),\r\n (0.1258374563468283, 0.1911188674736163),\r\n (0.1216704729278034, -0.3150426796961634),\r\n (0.1216704729278034, 0.3150426796961634),\r\n (0.1155056680537256, -0.4337935076260451),\r\n (0.1155056680537256, 0.4337935076260451),\r\n (0.1074442701159656, -0.5454214713888396),\r\n (0.1074442701159656, 0.5454214713888396),\r\n (0.0976186521041139, -0.6480936519369755),\r\n (0.0976186521041139, 0.6480936519369755),\r\n (0.0861901615319533, -0.7401241915785544),\r\n (0.0861901615319533, 0.7401241915785544),\r\n (0.0733464814110803, -0.8200019859739029),\r\n (0.0733464814110803, 0.8200019859739029),\r\n (0.0592985849154368, -0.8864155270044011),\r\n (0.0592985849154368, 0.8864155270044011),\r\n (0.0442774388174198, -0.9382745520027328),\r\n (0.0442774388174198, 0.9382745520027328),\r\n (0.0285313886289337, -0.9747285559713095),\r\n (0.0285313886289337, 0.9747285559713095),\r\n (0.0123412297999872, -0.9951872199970213),\r\n (0.0123412297999872, 0.9951872199970213),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_24_HALF: &[(f64, f64)] = &[\r\n (0.1279381953467522, 0.0640568928626056),\r\n (0.1258374563468283, 0.1911188674736163),\r\n (0.1216704729278034, 0.3150426796961634),\r\n (0.1155056680537256, 0.4337935076260451),\r\n (0.1074442701159656, 0.5454214713888396),\r\n (0.0976186521041139, 0.6480936519369755),\r\n (0.0861901615319533, 0.7401241915785544),\r\n (0.0733464814110803, 0.8200019859739029),\r\n (0.0592985849154368, 0.8864155270044011),\r\n (0.0442774388174198, 0.9382745520027328),\r\n (0.0285313886289337, 0.9747285559713095),\r\n (0.0123412297999872, 0.9951872199970213),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_32: &[(f64, f64)] = &[\r\n (0.0965400885147278, -0.0483076656877383),\r\n (0.0965400885147278, 0.0483076656877383),\r\n (0.0956387200792749, -0.1444719615827965),\r\n (0.0956387200792749, 0.1444719615827965),\r\n (0.0938443990808046, -0.2392873622521371),\r\n (0.0938443990808046, 0.2392873622521371),\r\n (0.0911738786957639, -0.3318686022821277),\r\n (0.0911738786957639, 0.3318686022821277),\r\n (0.0876520930044038, -0.4213512761306353),\r\n (0.0876520930044038, 0.4213512761306353),\r\n (0.0833119242269467, -0.5068999089322294),\r\n (0.0833119242269467, 0.5068999089322294),\r\n (0.0781938957870703, -0.5877157572407623),\r\n (0.0781938957870703, 0.5877157572407623),\r\n (0.0723457941088485, -0.6630442669302152),\r\n (0.0723457941088485, 0.6630442669302152),\r\n (0.0658222227763618, -0.7321821187402897),\r\n (0.0658222227763618, 0.7321821187402897),\r\n (0.0586840934785355, -0.7944837959679424),\r\n (0.0586840934785355, 0.7944837959679424),\r\n (0.0509980592623762, -0.8493676137325700),\r\n (0.0509980592623762, 0.8493676137325700),\r\n (0.0428358980222267, -0.8963211557660521),\r\n (0.0428358980222267, 0.8963211557660521),\r\n (0.0342738629130214, -0.9349060759377397),\r\n (0.0342738629130214, 0.9349060759377397),\r\n (0.0253920653092621, -0.9647622555875064),\r\n (0.0253920653092621, 0.9647622555875064),\r\n (0.0162743947309057, -0.9856115115452684),\r\n (0.0162743947309057, 0.9856115115452684),\r\n (0.0070186100094701, -0.9972638618494816),\r\n (0.0070186100094701, 0.9972638618494816),\r\n];\r\n\r\npub const GAUSS_LEGENDRE_COEFFS_32_HALF: &[(f64, f64)] = &[\r\n (0.0965400885147278, 0.0483076656877383),\r\n (0.0956387200792749, 0.1444719615827965),\r\n (0.0938443990808046, 0.2392873622521371),\r\n (0.0911738786957639, 0.3318686022821277),\r\n (0.0876520930044038, 0.4213512761306353),\r\n (0.0833119242269467, 0.5068999089322294),\r\n (0.0781938957870703, 0.5877157572407623),\r\n (0.0723457941088485, 0.6630442669302152),\r\n (0.0658222227763618, 0.7321821187402897),\r\n (0.0586840934785355, 0.7944837959679424),\r\n (0.0509980592623762, 0.8493676137325700),\r\n (0.0428358980222267, 0.8963211557660521),\r\n (0.0342738629130214, 0.9349060759377397),\r\n (0.0253920653092621, 0.9647622555875064),\r\n (0.0162743947309057, 0.9856115115452684),\r\n (0.0070186100094701, 0.9972638618494816),\r\n];\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::common::*;\r\n use arrayvec::ArrayVec;\r\n\r\n fn verify(mut roots: ArrayVec, expected: &[f64]) {\r\n assert_eq!(expected.len(), roots.len());\r\n let epsilon = 1e-12;\r\n roots.sort_by(|a, b| a.partial_cmp(b).unwrap());\r\n for i in 0..expected.len() {\r\n assert!((roots[i] - expected[i]).abs() < epsilon);\r\n }\r\n }\r\n\r\n #[test]\r\n fn test_solve_cubic() {\r\n verify(solve_cubic(-5.0, 0.0, 0.0, 1.0), &[5.0f64.cbrt()]);\r\n verify(solve_cubic(-5.0, -1.0, 0.0, 1.0), &[1.90416085913492]);\r\n verify(solve_cubic(0.0, -1.0, 0.0, 1.0), &[-1.0, 0.0, 1.0]);\r\n verify(solve_cubic(-2.0, -3.0, 0.0, 1.0), &[-1.0, 2.0]);\r\n verify(solve_cubic(2.0, -3.0, 0.0, 1.0), &[-2.0, 1.0]);\r\n verify(\r\n solve_cubic(2.0 - 1e-12, 5.0, 4.0, 1.0),\r\n &[\r\n -1.9999999999989995,\r\n -1.0000010000848456,\r\n -0.9999989999161546,\r\n ],\r\n );\r\n verify(solve_cubic(2.0 + 1e-12, 5.0, 4.0, 1.0), &[-2.0]);\r\n }\r\n\r\n #[test]\r\n fn test_solve_quadratic() {\r\n verify(\r\n solve_quadratic(-5.0, 0.0, 1.0),\r\n &[-(5.0f64.sqrt()), 5.0f64.sqrt()],\r\n );\r\n verify(solve_quadratic(5.0, 0.0, 1.0), &[]);\r\n verify(solve_quadratic(5.0, 1.0, 0.0), &[-5.0]);\r\n verify(solve_quadratic(1.0, 2.0, 1.0), &[-1.0]);\r\n }\r\n\r\n #[test]\r\n fn test_solve_quartic() {\r\n // These test cases are taken from Orellana and De Michele paper (Table 1).\r\n fn test_with_roots(coeffs: [f64; 4], roots: &[f64], rel_err: f64) {\r\n // Note: in paper, coefficients are in decreasing order.\r\n let mut actual = solve_quartic(coeffs[3], coeffs[2], coeffs[1], coeffs[0], 1.0);\r\n actual.sort_by(f64::total_cmp);\r\n assert_eq!(actual.len(), roots.len());\r\n for (actual, expected) in actual.iter().zip(roots) {\r\n assert!(\r\n (actual - expected).abs() < rel_err * expected.abs(),\r\n \"actual {:e}, expected {:e}, err {:e}\",\r\n actual,\r\n expected,\r\n actual - expected\r\n );\r\n }\r\n }\r\n\r\n fn test_vieta_roots(x1: f64, x2: f64, x3: f64, x4: f64, roots: &[f64], rel_err: f64) {\r\n let a = -(x1 + x2 + x3 + x4);\r\n let b = x1 * (x2 + x3) + x2 * (x3 + x4) + x4 * (x1 + x3);\r\n let c = -x1 * x2 * (x3 + x4) - x3 * x4 * (x1 + x2);\r\n let d = x1 * x2 * x3 * x4;\r\n test_with_roots([a, b, c, d], roots, rel_err);\r\n }\r\n\r\n fn test_vieta(x1: f64, x2: f64, x3: f64, x4: f64, rel_err: f64) {\r\n test_vieta_roots(x1, x2, x3, x4, &[x1, x2, x3, x4], rel_err);\r\n }\r\n\r\n // case 1\r\n test_vieta(1., 1e3, 1e6, 1e9, 1e-16);\r\n // case 2\r\n test_vieta(2., 2.001, 2.002, 2.003, 1e-6);\r\n // case 3\r\n test_vieta(1e47, 1e49, 1e50, 1e53, 2e-16);\r\n // case 4\r\n test_vieta(-1., 1., 2., 1e14, 1e-16);\r\n // case 5\r\n test_vieta(-2e7, -1., 1., 1e7, 1e-16);\r\n // case 6\r\n test_with_roots(\r\n [-9000002.0, -9999981999998.0, 19999982e6, -2e13],\r\n &[-1e6, 1e7],\r\n 1e-16,\r\n );\r\n // case 7\r\n test_with_roots(\r\n [2000011.0, 1010022000028.0, 11110056e6, 2828e10],\r\n &[-7., -4.],\r\n 1e-16,\r\n );\r\n // case 8\r\n test_with_roots(\r\n [-100002011.0, 201101022001.0, -102200111000011.0, 11000011e8],\r\n &[11., 1e8],\r\n 1e-16,\r\n );\r\n // cases 9-13 have no real roots\r\n // case 14\r\n test_vieta_roots(1000., 1000., 1000., 1000., &[1000., 1000.], 1e-16);\r\n // case 15\r\n test_vieta_roots(1e-15, 1000., 1000., 1000., &[1e-15, 1000., 1000.], 1e-15);\r\n // case 16 no real roots\r\n // case 17\r\n test_vieta(10000., 10001., 10010., 10100., 1e-6);\r\n // case 19\r\n test_vieta_roots(1., 1e30, 1e30, 1e44, &[1., 1e30, 1e44], 1e-16);\r\n // case 20\r\n // FAILS, error too big\r\n test_vieta(1., 1e7, 1e7, 1e14, 1e-7);\r\n // case 21 doesn't pick up double root\r\n // case 22\r\n test_vieta(1., 10., 1e152, 1e154, 3e-16);\r\n // case 23\r\n test_with_roots(\r\n [1., 1., 3. / 8., 1e-3],\r\n &[-0.497314148060048, -0.00268585193995149],\r\n 2e-15,\r\n );\r\n // case 24\r\n const S: f64 = 1e30;\r\n test_with_roots(\r\n [-(1. + 1. / S), 1. / S - S * S, S * S + S, -S],\r\n &[-S, 1e-30, 1., S],\r\n 2e-16,\r\n );\r\n }\r\n\r\n #[test]\r\n fn test_solve_itp() {\r\n let f = |x: f64| x.powi(3) - x - 2.0;\r\n let x = solve_itp(f, 1., 2., 1e-12, 0, 0.2, f(1.), f(2.));\r\n assert!(f(x).abs() < 6e-12);\r\n }\r\n\r\n #[test]\r\n fn test_inv_arclen() {\r\n use crate::{ParamCurve, ParamCurveArclen};\r\n let c = crate::CubicBez::new(\r\n (0.0, 0.0),\r\n (100.0 / 3.0, 0.0),\r\n (200.0 / 3.0, 100.0 / 3.0),\r\n (100.0, 100.0),\r\n );\r\n let target = 100.0;\r\n let _ = solve_itp(\r\n |t| c.subsegment(0.0..t).arclen(1e-9) - target,\r\n 0.,\r\n 1.,\r\n 1e-6,\r\n 1,\r\n 0.2,\r\n -target,\r\n c.arclen(1e-9) - target,\r\n );\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/cubicbez.rs", "sha256": "c56c0d65e352dc3174bbbcf10f11f7560e037016db181703bbee84384b33a3d2", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Cubic Bézier segments.\r\n\r\nuse alloc::vec;\r\nuse alloc::vec::Vec;\r\nuse core::ops::{Mul, Range};\r\n\r\nuse crate::MAX_EXTREMA;\r\nuse crate::{Line, QuadSpline, Vec2};\r\nuse arrayvec::ArrayVec;\r\n\r\nuse crate::common::{\r\n solve_quadratic, GAUSS_LEGENDRE_COEFFS_16_HALF, GAUSS_LEGENDRE_COEFFS_24_HALF,\r\n GAUSS_LEGENDRE_COEFFS_8, GAUSS_LEGENDRE_COEFFS_8_HALF,\r\n};\r\nuse crate::{\r\n Affine, Nearest, ParamCurve, ParamCurveArclen, ParamCurveArea, ParamCurveCurvature,\r\n ParamCurveDeriv, ParamCurveExtrema, ParamCurveNearest, PathEl, Point, QuadBez, Rect, Shape,\r\n};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\nconst MAX_SPLINE_SPLIT: usize = 100;\r\n\r\n/// A single cubic Bézier segment.\r\n#[derive(Clone, Copy, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\n#[allow(missing_docs)]\r\npub struct CubicBez {\r\n pub p0: Point,\r\n pub p1: Point,\r\n pub p2: Point,\r\n pub p3: Point,\r\n}\r\n\r\n/// An iterator which produces quadratic Bézier segments.\r\nstruct ToQuads {\r\n c: CubicBez,\r\n i: usize,\r\n n: usize,\r\n}\r\n\r\nimpl CubicBez {\r\n /// Create a new cubic Bézier segment.\r\n #[inline]\r\n pub fn new>(p0: P, p1: P, p2: P, p3: P) -> CubicBez {\r\n CubicBez {\r\n p0: p0.into(),\r\n p1: p1.into(),\r\n p2: p2.into(),\r\n p3: p3.into(),\r\n }\r\n }\r\n\r\n /// Convert to quadratic Béziers.\r\n ///\r\n /// The iterator returns the start and end parameter in the cubic of each quadratic\r\n /// segment, along with the quadratic.\r\n ///\r\n /// Note that the resulting quadratic Béziers are not in general G1 continuous;\r\n /// they are optimized for minimizing distance error.\r\n ///\r\n /// This iterator will always produce at least one `QuadBez`.\r\n #[inline]\r\n pub fn to_quads(&self, accuracy: f64) -> impl Iterator {\r\n // The maximum error, as a vector from the cubic to the best approximating\r\n // quadratic, is proportional to the third derivative, which is constant\r\n // across the segment. Thus, the error scales down as the third power of\r\n // the number of subdivisions. Our strategy then is to subdivide `t` evenly.\r\n //\r\n // This is an overestimate of the error because only the component\r\n // perpendicular to the first derivative is important. But the simplicity is\r\n // appealing.\r\n\r\n // This magic number is the square of 36 / sqrt(3).\r\n // See: http://caffeineowl.com/graphics/2d/vectorial/cubic2quad01.html\r\n let max_hypot2 = 432.0 * accuracy * accuracy;\r\n let p1x2 = 3.0 * self.p1.to_vec2() - self.p0.to_vec2();\r\n let p2x2 = 3.0 * self.p2.to_vec2() - self.p3.to_vec2();\r\n let err = (p2x2 - p1x2).hypot2();\r\n let n = ((err / max_hypot2).powf(1. / 6.0).ceil() as usize).max(1);\r\n\r\n ToQuads { c: *self, n, i: 0 }\r\n }\r\n\r\n /// Return a [`QuadSpline`] approximating this cubic Bézier.\r\n ///\r\n /// Returns `None` if no suitable approximation is found within the given\r\n /// tolerance.\r\n pub fn approx_spline(&self, accuracy: f64) -> Option {\r\n (1..=MAX_SPLINE_SPLIT).find_map(|n| self.approx_spline_n(n, accuracy))\r\n }\r\n\r\n // Approximate a cubic curve with a quadratic spline of `n` curves\r\n fn approx_spline_n(&self, n: usize, accuracy: f64) -> Option {\r\n if n == 1 {\r\n return self\r\n .try_approx_quadratic(accuracy)\r\n .map(|quad| QuadSpline::new(vec![quad.p0, quad.p1, quad.p2]));\r\n }\r\n let mut cubics = self.split_into_n(n);\r\n\r\n // The above function guarantees that the iterator returns n items,\r\n // which is why we're unwrapping things with wild abandon.\r\n let mut next_cubic = cubics.next().unwrap();\r\n let mut next_q1: Point = next_cubic.approx_quad_control(0.0);\r\n let mut q2 = self.p0;\r\n let mut d1 = Vec2::ZERO;\r\n let mut spline = vec![self.p0, next_q1];\r\n for i in 1..=n {\r\n let current_cubic: CubicBez = next_cubic;\r\n let q0 = q2;\r\n let q1 = next_q1;\r\n q2 = if i < n {\r\n next_cubic = cubics.next().unwrap();\r\n next_q1 = next_cubic.approx_quad_control(i as f64 / (n - 1) as f64);\r\n\r\n spline.push(next_q1);\r\n q1.midpoint(next_q1)\r\n } else {\r\n current_cubic.p3\r\n };\r\n let d0 = d1;\r\n d1 = q2.to_vec2() - current_cubic.p3.to_vec2();\r\n\r\n if d1.hypot() > accuracy\r\n || !CubicBez::new(\r\n d0.to_point(),\r\n q0.lerp(q1, 2.0 / 3.0) - current_cubic.p1.to_vec2(),\r\n q2.lerp(q1, 2.0 / 3.0) - current_cubic.p2.to_vec2(),\r\n d1.to_point(),\r\n )\r\n .fit_inside(accuracy)\r\n {\r\n return None;\r\n }\r\n }\r\n spline.push(self.p3);\r\n Some(QuadSpline::new(spline))\r\n }\r\n\r\n fn approx_quad_control(&self, t: f64) -> Point {\r\n let p1 = self.p0 + (self.p1 - self.p0) * 1.5;\r\n let p2 = self.p3 + (self.p2 - self.p3) * 1.5;\r\n p1.lerp(p2, t)\r\n }\r\n\r\n /// Approximate a cubic with a single quadratic\r\n ///\r\n /// Returns a quadratic approximating the given cubic that maintains\r\n /// endpoint tangents if that is within tolerance, or None otherwise.\r\n fn try_approx_quadratic(&self, accuracy: f64) -> Option {\r\n if let Some(q1) = Line::new(self.p0, self.p1).crossing_point(Line::new(self.p2, self.p3)) {\r\n let c1 = self.p0.lerp(q1, 2.0 / 3.0);\r\n let c2 = self.p3.lerp(q1, 2.0 / 3.0);\r\n if !CubicBez::new(\r\n Point::ZERO,\r\n c1 - self.p1.to_vec2(),\r\n c2 - self.p2.to_vec2(),\r\n Point::ZERO,\r\n )\r\n .fit_inside(accuracy)\r\n {\r\n return None;\r\n }\r\n return Some(QuadBez::new(self.p0, q1, self.p3));\r\n }\r\n None\r\n }\r\n\r\n fn split_into_n(&self, n: usize) -> impl Iterator {\r\n // for certain small values of `n` we precompute all our values.\r\n // if we have six or fewer items we precompute them.\r\n let mut storage = ArrayVec::<_, 6>::new();\r\n\r\n match n {\r\n 1 => storage.push(*self),\r\n 2 => {\r\n let (l, r) = self.subdivide();\r\n storage.try_extend_from_slice(&[r, l]).unwrap();\r\n }\r\n 3 => {\r\n let (left, mid, right) = self.subdivide_3();\r\n storage.try_extend_from_slice(&[right, mid, left]).unwrap();\r\n }\r\n 4 => {\r\n let (l, r) = self.subdivide();\r\n let (ll, lr) = l.subdivide();\r\n let (rl, rr) = r.subdivide();\r\n storage.try_extend_from_slice(&[rr, rl, lr, ll]).unwrap();\r\n }\r\n 6 => {\r\n let (l, r) = self.subdivide();\r\n let (l1, l2, l3) = l.subdivide_3();\r\n let (r1, r2, r3) = r.subdivide_3();\r\n storage\r\n .try_extend_from_slice(&[r3, r2, r1, l3, l2, l1])\r\n .unwrap();\r\n }\r\n _ => (),\r\n }\r\n\r\n // a limitation of returning 'impl Trait' is that the implementation\r\n // can only return a single concrete type; that is you cannot return\r\n // Vec::into_iter() from one branch, and then HashSet::into_iter from\r\n // another branch.\r\n //\r\n // This means we have to get a bit fancy, and have a single concrete\r\n // type that represents both of our possible cases.\r\n\r\n let mut storage = if storage.is_empty() {\r\n None\r\n } else {\r\n Some(storage)\r\n };\r\n\r\n // used in the fallback case\r\n let mut i = 0;\r\n let (a, b, c, d) = self.parameters();\r\n let dt = 1.0 / n as f64;\r\n let delta_2 = dt * dt;\r\n let delta_3 = dt * delta_2;\r\n\r\n core::iter::from_fn(move || {\r\n // if storage exists, we use it exclusively\r\n if let Some(storage) = storage.as_mut() {\r\n return storage.pop();\r\n }\r\n\r\n // if storage does not exist, we are exclusively working down here.\r\n if i >= n {\r\n return None;\r\n }\r\n\r\n let t1 = i as f64 * dt;\r\n let t1_2 = t1 * t1;\r\n let a1 = a * delta_3;\r\n let b1 = (3.0 * a * t1 + b) * delta_2;\r\n let c1 = (2.0 * b * t1 + c + 3.0 * a * t1_2) * dt;\r\n let d1 = a * t1 * t1_2 + b * t1_2 + c * t1 + d;\r\n let result = CubicBez::from_parameters(a1, b1, c1, d1);\r\n i += 1;\r\n Some(result)\r\n })\r\n }\r\n\r\n fn parameters(&self) -> (Vec2, Vec2, Vec2, Vec2) {\r\n let c = (self.p1 - self.p0) * 3.0;\r\n let b = (self.p2 - self.p1) * 3.0 - c;\r\n let d = self.p0.to_vec2();\r\n let a = self.p3.to_vec2() - d - c - b;\r\n (a, b, c, d)\r\n }\r\n\r\n /// Rust port of cu2qu [calc_cubic_points](https://github.com/fonttools/fonttools/blob/3b9a73ff8379ab49d3ce35aaaaf04b3a7d9d1655/Lib/fontTools/cu2qu/cu2qu.py#L63-L68)\r\n fn from_parameters(a: Vec2, b: Vec2, c: Vec2, d: Vec2) -> Self {\r\n let p0 = d.to_point();\r\n let p1 = c.div_exact(3.0).to_point() + d;\r\n let p2 = (b + c).div_exact(3.0).to_point() + p1.to_vec2();\r\n let p3 = (a + d + c + b).to_point();\r\n CubicBez::new(p0, p1, p2, p3)\r\n }\r\n\r\n fn subdivide_3(&self) -> (CubicBez, CubicBez, CubicBez) {\r\n let (p0, p1, p2, p3) = (\r\n self.p0.to_vec2(),\r\n self.p1.to_vec2(),\r\n self.p2.to_vec2(),\r\n self.p3.to_vec2(),\r\n );\r\n // The original Python cu2qu code here does not use division operator to divide by 27 but\r\n // instead uses multiplication by the reciprocal 1 / 27. We want to match it exactly\r\n // to avoid any floating point differences, hence in this particular case we do not use div_exact.\r\n // I could directly use the Vec2 Div trait (also implemented as multiplication by reciprocal)\r\n // but I prefer to be explicit here.\r\n // Source: https://github.com/fonttools/fonttools/blob/85c80be/Lib/fontTools/cu2qu/cu2qu.py#L215-L218\r\n // See also: https://github.com/linebender/kurbo/issues/272\r\n let one_27th = 27.0_f64.recip();\r\n let mid1 = ((8.0 * p0 + 12.0 * p1 + 6.0 * p2 + p3) * one_27th).to_point();\r\n let deriv1 = (p3 + 3.0 * p2 - 4.0 * p0) * one_27th;\r\n let mid2 = ((p0 + 6.0 * p1 + 12.0 * p2 + 8.0 * p3) * one_27th).to_point();\r\n let deriv2 = (4.0 * p3 - 3.0 * p1 - p0) * one_27th;\r\n\r\n let left = CubicBez::new(\r\n self.p0,\r\n (2.0 * p0 + p1).div_exact(3.0).to_point(),\r\n mid1 - deriv1,\r\n mid1,\r\n );\r\n let mid = CubicBez::new(mid1, mid1 + deriv1, mid2 - deriv2, mid2);\r\n let right = CubicBez::new(\r\n mid2,\r\n mid2 + deriv2,\r\n (p2 + 2.0 * p3).div_exact(3.0).to_point(),\r\n self.p3,\r\n );\r\n (left, mid, right)\r\n }\r\n\r\n /// Does this curve fit inside the given distance from the origin?\r\n ///\r\n /// Rust port of cu2qu [cubic_farthest_fit_inside](https://github.com/fonttools/fonttools/blob/3b9a73ff8379ab49d3ce35aaaaf04b3a7d9d1655/Lib/fontTools/cu2qu/cu2qu.py#L281)\r\n fn fit_inside(&self, distance: f64) -> bool {\r\n if self.p2.to_vec2().hypot() <= distance && self.p1.to_vec2().hypot() <= distance {\r\n return true;\r\n }\r\n let mid =\r\n (self.p0.to_vec2() + 3.0 * (self.p1.to_vec2() + self.p2.to_vec2()) + self.p3.to_vec2())\r\n * 0.125;\r\n if mid.hypot() > distance {\r\n return false;\r\n }\r\n // Split in two. Note that cu2qu here uses a 3/8 subdivision. I don't know why.\r\n let (left, right) = self.subdivide();\r\n left.fit_inside(distance) && right.fit_inside(distance)\r\n }\r\n\r\n /// Is this cubic Bezier curve finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n self.p0.is_finite() && self.p1.is_finite() && self.p2.is_finite() && self.p3.is_finite()\r\n }\r\n\r\n /// Is this cubic Bezier curve NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n self.p0.is_nan() || self.p1.is_nan() || self.p2.is_nan() || self.p3.is_nan()\r\n }\r\n\r\n /// Determine the inflection points.\r\n ///\r\n /// Return value is t parameter for the inflection points of the curve segment.\r\n /// There are a maximum of two for a cubic Bézier.\r\n ///\r\n /// See \r\n /// for the theory.\r\n pub fn inflections(&self) -> ArrayVec {\r\n let a = self.p1 - self.p0;\r\n let b = (self.p2 - self.p1) - a;\r\n let c = (self.p3 - self.p0) - 3. * (self.p2 - self.p1);\r\n solve_quadratic(a.cross(b), a.cross(c), b.cross(c))\r\n .iter()\r\n .copied()\r\n .filter(|t| *t >= 0.0 && *t <= 1.0)\r\n .collect()\r\n }\r\n}\r\n\r\n/// An iterator for cubic beziers.\r\npub struct CubicBezIter {\r\n cubic: CubicBez,\r\n ix: usize,\r\n}\r\n\r\nimpl Shape for CubicBez {\r\n type PathElementsIter<'iter> = CubicBezIter;\r\n\r\n #[inline]\r\n fn path_elements(&self, _tolerance: f64) -> CubicBezIter {\r\n CubicBezIter {\r\n cubic: *self,\r\n ix: 0,\r\n }\r\n }\r\n\r\n fn area(&self) -> f64 {\r\n 0.0\r\n }\r\n\r\n #[inline]\r\n fn perimeter(&self, accuracy: f64) -> f64 {\r\n self.arclen(accuracy)\r\n }\r\n\r\n fn winding(&self, _pt: Point) -> i32 {\r\n 0\r\n }\r\n\r\n #[inline]\r\n fn bounding_box(&self) -> Rect {\r\n ParamCurveExtrema::bounding_box(self)\r\n }\r\n}\r\n\r\nimpl Iterator for CubicBezIter {\r\n type Item = PathEl;\r\n\r\n fn next(&mut self) -> Option {\r\n self.ix += 1;\r\n match self.ix {\r\n 1 => Some(PathEl::MoveTo(self.cubic.p0)),\r\n 2 => Some(PathEl::CurveTo(self.cubic.p1, self.cubic.p2, self.cubic.p3)),\r\n _ => None,\r\n }\r\n }\r\n}\r\n\r\nimpl ParamCurve for CubicBez {\r\n #[inline]\r\n fn eval(&self, t: f64) -> Point {\r\n let mt = 1.0 - t;\r\n let v = self.p0.to_vec2() * (mt * mt * mt)\r\n + (self.p1.to_vec2() * (mt * mt * 3.0)\r\n + (self.p2.to_vec2() * (mt * 3.0) + self.p3.to_vec2() * t) * t)\r\n * t;\r\n v.to_point()\r\n }\r\n\r\n fn subsegment(&self, range: Range) -> CubicBez {\r\n let (t0, t1) = (range.start, range.end);\r\n let p0 = self.eval(t0);\r\n let p3 = self.eval(t1);\r\n let d = self.deriv();\r\n let scale = (t1 - t0) * (1.0 / 3.0);\r\n let p1 = p0 + scale * d.eval(t0).to_vec2();\r\n let p2 = p3 - scale * d.eval(t1).to_vec2();\r\n CubicBez { p0, p1, p2, p3 }\r\n }\r\n\r\n /// Subdivide into halves, using de Casteljau.\r\n #[inline]\r\n fn subdivide(&self) -> (CubicBez, CubicBez) {\r\n let pm = self.eval(0.5);\r\n (\r\n CubicBez::new(\r\n self.p0,\r\n self.p0.midpoint(self.p1),\r\n ((self.p0.to_vec2() + self.p1.to_vec2() * 2.0 + self.p2.to_vec2()) * 0.25)\r\n .to_point(),\r\n pm,\r\n ),\r\n CubicBez::new(\r\n pm,\r\n ((self.p1.to_vec2() + self.p2.to_vec2() * 2.0 + self.p3.to_vec2()) * 0.25)\r\n .to_point(),\r\n self.p2.midpoint(self.p3),\r\n self.p3,\r\n ),\r\n )\r\n }\r\n\r\n #[inline]\r\n fn start(&self) -> Point {\r\n self.p0\r\n }\r\n\r\n #[inline]\r\n fn end(&self) -> Point {\r\n self.p3\r\n }\r\n}\r\n\r\nimpl ParamCurveDeriv for CubicBez {\r\n type DerivResult = QuadBez;\r\n\r\n #[inline]\r\n fn deriv(&self) -> QuadBez {\r\n QuadBez::new(\r\n (3.0 * (self.p1 - self.p0)).to_point(),\r\n (3.0 * (self.p2 - self.p1)).to_point(),\r\n (3.0 * (self.p3 - self.p2)).to_point(),\r\n )\r\n }\r\n}\r\n\r\nfn arclen_quadrature_core(coeffs: &[(f64, f64)], dm: Vec2, dm1: Vec2, dm2: Vec2) -> f64 {\r\n coeffs\r\n .iter()\r\n .map(|&(wi, xi)| {\r\n let d = dm + dm2 * (xi * xi);\r\n let dpx = (d + dm1 * xi).hypot();\r\n let dmx = (d - dm1 * xi).hypot();\r\n (2.25f64.sqrt() * wi) * (dpx + dmx)\r\n })\r\n .sum::()\r\n}\r\n\r\nfn arclen_rec(c: &CubicBez, accuracy: f64, depth: usize) -> f64 {\r\n let d03 = c.p3 - c.p0;\r\n let d01 = c.p1 - c.p0;\r\n let d12 = c.p2 - c.p1;\r\n let d23 = c.p3 - c.p2;\r\n let lp_lc = d01.hypot() + d12.hypot() + d23.hypot() - d03.hypot();\r\n let dd1 = d12 - d01;\r\n let dd2 = d23 - d12;\r\n // It might be faster to do direct multiplies, the data dependencies would be shorter.\r\n // The following values don't have the factor of 3 for first deriv\r\n let dm = 0.25 * (d01 + d23) + 0.5 * d12; // first derivative at midpoint\r\n let dm1 = 0.5 * (dd2 + dd1); // second derivative at midpoint\r\n let dm2 = 0.25 * (dd2 - dd1); // 0.5 * (third derivative at midpoint)\r\n\r\n let est = GAUSS_LEGENDRE_COEFFS_8\r\n .iter()\r\n .map(|&(wi, xi)| {\r\n wi * {\r\n let d_norm2 = (dm + dm1 * xi + dm2 * (xi * xi)).hypot2();\r\n let dd_norm2 = (dm1 + dm2 * (2.0 * xi)).hypot2();\r\n dd_norm2 / d_norm2\r\n }\r\n })\r\n .sum::();\r\n let est_gauss8_error = (est.powi(3) * 2.5e-6).min(3e-2) * lp_lc;\r\n if est_gauss8_error < accuracy {\r\n return arclen_quadrature_core(GAUSS_LEGENDRE_COEFFS_8_HALF, dm, dm1, dm2);\r\n }\r\n let est_gauss16_error = (est.powi(6) * 1.5e-11).min(9e-3) * lp_lc;\r\n if est_gauss16_error < accuracy {\r\n return arclen_quadrature_core(GAUSS_LEGENDRE_COEFFS_16_HALF, dm, dm1, dm2);\r\n }\r\n let est_gauss24_error = (est.powi(9) * 3.5e-16).min(3.5e-3) * lp_lc;\r\n if est_gauss24_error < accuracy || depth >= 20 {\r\n return arclen_quadrature_core(GAUSS_LEGENDRE_COEFFS_24_HALF, dm, dm1, dm2);\r\n }\r\n let (c0, c1) = c.subdivide();\r\n arclen_rec(&c0, accuracy * 0.5, depth + 1) + arclen_rec(&c1, accuracy * 0.5, depth + 1)\r\n}\r\n\r\nimpl ParamCurveArclen for CubicBez {\r\n /// Arclength of a cubic Bézier segment.\r\n ///\r\n /// This is an adaptive subdivision approach using Legendre-Gauss quadrature\r\n /// in the base case, and an error estimate to decide when to subdivide.\r\n fn arclen(&self, accuracy: f64) -> f64 {\r\n arclen_rec(self, accuracy, 0)\r\n }\r\n}\r\n\r\nimpl ParamCurveArea for CubicBez {\r\n #[inline]\r\n fn signed_area(&self) -> f64 {\r\n (self.p0.x * (6.0 * self.p1.y + 3.0 * self.p2.y + self.p3.y)\r\n + 3.0\r\n * (self.p1.x * (-2.0 * self.p0.y + self.p2.y + self.p3.y)\r\n - self.p2.x * (self.p0.y + self.p1.y - 2.0 * self.p3.y))\r\n - self.p3.x * (self.p0.y + 3.0 * self.p1.y + 6.0 * self.p2.y))\r\n * (1.0 / 20.0)\r\n }\r\n}\r\n\r\nimpl ParamCurveNearest for CubicBez {\r\n /// Find the nearest point, using subdivision.\r\n fn nearest(&self, p: Point, accuracy: f64) -> Nearest {\r\n let mut best_r = None;\r\n let mut best_t = 0.0;\r\n for (t0, t1, q) in self.to_quads(accuracy) {\r\n let nearest = q.nearest(p, accuracy);\r\n if best_r\r\n .map(|best_r| nearest.distance_sq < best_r)\r\n .unwrap_or(true)\r\n {\r\n best_t = t0 + nearest.t * (t1 - t0);\r\n best_r = Some(nearest.distance_sq);\r\n }\r\n }\r\n Nearest {\r\n t: best_t,\r\n distance_sq: best_r.unwrap(),\r\n }\r\n }\r\n}\r\n\r\nimpl ParamCurveCurvature for CubicBez {}\r\n\r\nimpl ParamCurveExtrema for CubicBez {\r\n fn extrema(&self) -> ArrayVec {\r\n fn one_coord(result: &mut ArrayVec, d0: f64, d1: f64, d2: f64) {\r\n let a = d0 - 2.0 * d1 + d2;\r\n let b = 2.0 * (d1 - d0);\r\n let c = d0;\r\n let roots = solve_quadratic(c, b, a);\r\n for &t in &roots {\r\n if t > 0.0 && t < 1.0 {\r\n result.push(t);\r\n }\r\n }\r\n }\r\n let mut result = ArrayVec::new();\r\n let d0 = self.p1 - self.p0;\r\n let d1 = self.p2 - self.p1;\r\n let d2 = self.p3 - self.p2;\r\n one_coord(&mut result, d0.x, d1.x, d2.x);\r\n one_coord(&mut result, d0.y, d1.y, d2.y);\r\n result.sort_by(|a, b| a.partial_cmp(b).unwrap());\r\n result\r\n }\r\n}\r\n\r\nimpl Mul for Affine {\r\n type Output = CubicBez;\r\n\r\n #[inline]\r\n fn mul(self, c: CubicBez) -> CubicBez {\r\n CubicBez {\r\n p0: self * c.p0,\r\n p1: self * c.p1,\r\n p2: self * c.p2,\r\n p3: self * c.p3,\r\n }\r\n }\r\n}\r\n\r\nimpl Iterator for ToQuads {\r\n type Item = (f64, f64, QuadBez);\r\n\r\n fn next(&mut self) -> Option<(f64, f64, QuadBez)> {\r\n if self.i == self.n {\r\n return None;\r\n }\r\n let t0 = self.i as f64 / self.n as f64;\r\n let t1 = (self.i + 1) as f64 / self.n as f64;\r\n let seg = self.c.subsegment(t0..t1);\r\n let p1x2 = 3.0 * seg.p1.to_vec2() - seg.p0.to_vec2();\r\n let p2x2 = 3.0 * seg.p2.to_vec2() - seg.p3.to_vec2();\r\n let result = QuadBez::new(seg.p0, ((p1x2 + p2x2) / 4.0).to_point(), seg.p3);\r\n self.i += 1;\r\n Some((t0, t1, result))\r\n }\r\n\r\n fn size_hint(&self) -> (usize, Option) {\r\n let remaining = self.n - self.i;\r\n (remaining, Some(remaining))\r\n }\r\n}\r\n\r\n/// Convert multiple cubic Bézier curves to quadratic splines.\r\n///\r\n/// Ensures that the resulting splines have the same number of control points.\r\n///\r\n/// Rust port of cu2qu [cubic_approx_quadratic](https://github.com/fonttools/fonttools/blob/3b9a73ff8379ab49d3ce35aaaaf04b3a7d9d1655/Lib/fontTools/cu2qu/cu2qu.py#L322)\r\npub fn cubics_to_quadratic_splines(curves: &[CubicBez], accuracy: f64) -> Option> {\r\n let mut result = Vec::new();\r\n let mut split_order = 0;\r\n\r\n while split_order <= MAX_SPLINE_SPLIT {\r\n split_order += 1;\r\n result.clear();\r\n\r\n for curve in curves {\r\n match curve.approx_spline_n(split_order, accuracy) {\r\n Some(spline) => result.push(spline),\r\n None => break,\r\n }\r\n }\r\n\r\n if result.len() == curves.len() {\r\n return Some(result);\r\n }\r\n }\r\n None\r\n}\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{\r\n cubics_to_quadratic_splines, Affine, CubicBez, Nearest, ParamCurve, ParamCurveArclen,\r\n ParamCurveArea, ParamCurveDeriv, ParamCurveExtrema, ParamCurveNearest, Point, QuadBez,\r\n QuadSpline,\r\n };\r\n\r\n #[test]\r\n fn cubicbez_deriv() {\r\n // y = x^2\r\n let c = CubicBez::new(\r\n (0.0, 0.0),\r\n (1.0 / 3.0, 0.0),\r\n (2.0 / 3.0, 1.0 / 3.0),\r\n (1.0, 1.0),\r\n );\r\n let deriv = c.deriv();\r\n\r\n let n = 10;\r\n for i in 0..=n {\r\n let t = (i as f64) * (n as f64).recip();\r\n let delta = 1e-6;\r\n let p = c.eval(t);\r\n let p1 = c.eval(t + delta);\r\n let d_approx = (p1 - p) * delta.recip();\r\n let d = deriv.eval(t).to_vec2();\r\n assert!((d - d_approx).hypot() < delta * 2.0);\r\n }\r\n }\r\n\r\n #[test]\r\n fn cubicbez_arclen() {\r\n // y = x^2\r\n let c = CubicBez::new(\r\n (0.0, 0.0),\r\n (1.0 / 3.0, 0.0),\r\n (2.0 / 3.0, 1.0 / 3.0),\r\n (1.0, 1.0),\r\n );\r\n let true_arclen = 0.5 * 5.0f64.sqrt() + 0.25 * (2.0 + 5.0f64.sqrt()).ln();\r\n for i in 0..12 {\r\n let accuracy = 0.1f64.powi(i);\r\n let error = c.arclen(accuracy) - true_arclen;\r\n assert!(error.abs() < accuracy);\r\n }\r\n }\r\n\r\n #[test]\r\n fn cubicbez_inv_arclen() {\r\n // y = x^2 / 100\r\n let c = CubicBez::new(\r\n (0.0, 0.0),\r\n (100.0 / 3.0, 0.0),\r\n (200.0 / 3.0, 100.0 / 3.0),\r\n (100.0, 100.0),\r\n );\r\n let true_arclen = 100.0 * (0.5 * 5.0f64.sqrt() + 0.25 * (2.0 + 5.0f64.sqrt()).ln());\r\n for i in 0..12 {\r\n let accuracy = 0.1f64.powi(i);\r\n let n = 10;\r\n for j in 0..=n {\r\n let arc = (j as f64) * ((n as f64).recip() * true_arclen);\r\n let t = c.inv_arclen(arc, accuracy * 0.5);\r\n let actual_arc = c.subsegment(0.0..t).arclen(accuracy * 0.5);\r\n assert!(\r\n (arc - actual_arc).abs() < accuracy,\r\n \"at accuracy {accuracy:e}, wanted {actual_arc} got {arc}\"\r\n );\r\n }\r\n }\r\n // corner case: user passes accuracy larger than total arc length\r\n let accuracy = true_arclen * 1.1;\r\n let arc = true_arclen * 0.5;\r\n let t = c.inv_arclen(arc, accuracy);\r\n let actual_arc = c.subsegment(0.0..t).arclen(accuracy);\r\n assert!(\r\n (arc - actual_arc).abs() < 2.0 * accuracy,\r\n \"at accuracy {accuracy:e}, want {actual_arc} got {arc}\"\r\n );\r\n }\r\n\r\n #[test]\r\n fn cubicbez_inv_arclen_accuracy() {\r\n let c = CubicBez::new((0.2, 0.73), (0.35, 1.08), (0.85, 1.08), (1.0, 0.73));\r\n let true_t = c.inv_arclen(0.5, 1e-12);\r\n for i in 1..12 {\r\n let accuracy = (0.1f64).powi(i);\r\n let approx_t = c.inv_arclen(0.5, accuracy);\r\n assert!((approx_t - true_t).abs() <= accuracy);\r\n }\r\n }\r\n\r\n #[test]\r\n #[allow(clippy::float_cmp)]\r\n fn cubicbez_signed_area_linear() {\r\n // y = 1 - x\r\n let c = CubicBez::new(\r\n (1.0, 0.0),\r\n (2.0 / 3.0, 1.0 / 3.0),\r\n (1.0 / 3.0, 2.0 / 3.0),\r\n (0.0, 1.0),\r\n );\r\n let epsilon = 1e-12;\r\n assert_eq!((Affine::rotate(0.5) * c).signed_area(), 0.5);\r\n assert!(((Affine::rotate(0.5) * c).signed_area() - 0.5).abs() < epsilon);\r\n assert!(((Affine::translate((0.0, 1.0)) * c).signed_area() - 1.0).abs() < epsilon);\r\n assert!(((Affine::translate((1.0, 0.0)) * c).signed_area() - 1.0).abs() < epsilon);\r\n }\r\n\r\n #[test]\r\n fn cubicbez_signed_area() {\r\n // y = 1 - x^3\r\n let c = CubicBez::new((1.0, 0.0), (2.0 / 3.0, 1.0), (1.0 / 3.0, 1.0), (0.0, 1.0));\r\n let epsilon = 1e-12;\r\n assert!((c.signed_area() - 0.75).abs() < epsilon);\r\n assert!(((Affine::rotate(0.5) * c).signed_area() - 0.75).abs() < epsilon);\r\n assert!(((Affine::translate((0.0, 1.0)) * c).signed_area() - 1.25).abs() < epsilon);\r\n assert!(((Affine::translate((1.0, 0.0)) * c).signed_area() - 1.25).abs() < epsilon);\r\n }\r\n\r\n #[test]\r\n fn cubicbez_nearest() {\r\n fn verify(result: Nearest, expected: f64) {\r\n assert!(\r\n (result.t - expected).abs() < 1e-6,\r\n \"got {result:?} expected {expected}\"\r\n );\r\n }\r\n // y = x^3\r\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 0.0), (1.0, 1.0));\r\n verify(c.nearest((0.1, 0.001).into(), 1e-6), 0.1);\r\n verify(c.nearest((0.2, 0.008).into(), 1e-6), 0.2);\r\n verify(c.nearest((0.3, 0.027).into(), 1e-6), 0.3);\r\n verify(c.nearest((0.4, 0.064).into(), 1e-6), 0.4);\r\n verify(c.nearest((0.5, 0.125).into(), 1e-6), 0.5);\r\n verify(c.nearest((0.6, 0.216).into(), 1e-6), 0.6);\r\n verify(c.nearest((0.7, 0.343).into(), 1e-6), 0.7);\r\n verify(c.nearest((0.8, 0.512).into(), 1e-6), 0.8);\r\n verify(c.nearest((0.9, 0.729).into(), 1e-6), 0.9);\r\n verify(c.nearest((1.0, 1.0).into(), 1e-6), 1.0);\r\n verify(c.nearest((1.1, 1.1).into(), 1e-6), 1.0);\r\n verify(c.nearest((-0.1, 0.0).into(), 1e-6), 0.0);\r\n let a = Affine::rotate(0.5);\r\n verify((a * c).nearest(a * Point::new(0.1, 0.001), 1e-6), 0.1);\r\n }\r\n\r\n // ensure to_quads returns something given colinear points\r\n #[test]\r\n fn degenerate_to_quads() {\r\n let c = CubicBez::new((0., 9.), (6., 6.), (12., 3.0), (18., 0.0));\r\n let quads = c.to_quads(1e-6).collect::>();\r\n assert_eq!(quads.len(), 1, \"{:?}\", &quads);\r\n }\r\n\r\n #[test]\r\n fn cubicbez_extrema() {\r\n // y = x^2\r\n let q = CubicBez::new((0.0, 0.0), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0));\r\n let extrema = q.extrema();\r\n assert_eq!(extrema.len(), 1);\r\n assert!((extrema[0] - 0.5).abs() < 1e-6);\r\n\r\n let q = CubicBez::new((0.4, 0.5), (0.0, 1.0), (1.0, 0.0), (0.5, 0.4));\r\n let extrema = q.extrema();\r\n assert_eq!(extrema.len(), 4);\r\n }\r\n\r\n #[test]\r\n fn cubicbez_toquads() {\r\n // y = x^3\r\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 0.0), (1.0, 1.0));\r\n for i in 0..10 {\r\n let accuracy = 0.1f64.powi(i);\r\n let mut worst: f64 = 0.0;\r\n for (_count, (t0, t1, q)) in c.to_quads(accuracy).enumerate() {\r\n let epsilon = 1e-12;\r\n assert!((q.start() - c.eval(t0)).hypot() < epsilon);\r\n assert!((q.end() - c.eval(t1)).hypot() < epsilon);\r\n let n = 4;\r\n for j in 0..=n {\r\n let t = (j as f64) * (n as f64).recip();\r\n let p = q.eval(t);\r\n let err = (p.y - p.x.powi(3)).abs();\r\n worst = worst.max(err);\r\n assert!(err < accuracy, \"got {err} wanted {accuracy}\");\r\n }\r\n }\r\n }\r\n }\r\n\r\n #[test]\r\n fn cubicbez_approx_spline() {\r\n let c1 = CubicBez::new(\r\n (550.0, 258.0),\r\n (1044.0, 482.0),\r\n (2029.0, 1841.0),\r\n (1934.0, 1554.0),\r\n );\r\n\r\n let quad = c1.try_approx_quadratic(344.0);\r\n let expected = QuadBez::new(\r\n Point::new(550.0, 258.0),\r\n Point::new(1673.665720592873, 767.5164401068898),\r\n Point::new(1934.0, 1554.0),\r\n );\r\n assert!(quad.is_some());\r\n assert_eq!(quad.unwrap(), expected);\r\n\r\n let quad = c1.try_approx_quadratic(343.0);\r\n assert!(quad.is_none());\r\n\r\n let spline = c1.approx_spline_n(2, 343.0);\r\n assert!(spline.is_some());\r\n let spline = spline.unwrap();\r\n let expected = vec![\r\n Point::new(550.0, 258.0),\r\n Point::new(920.5, 426.0),\r\n Point::new(2005.25, 1769.25),\r\n Point::new(1934.0, 1554.0),\r\n ];\r\n assert_eq!(spline.points().len(), expected.len());\r\n for (got, &wanted) in spline.points().iter().zip(expected.iter()) {\r\n assert!(got.distance(wanted) < 5.0)\r\n }\r\n\r\n let spline = c1.approx_spline(5.0);\r\n let expected = vec![\r\n Point::new(550.0, 258.0),\r\n Point::new(673.5, 314.0),\r\n Point::new(984.8777777777776, 584.2666666666667),\r\n Point::new(1312.6305555555557, 927.825),\r\n Point::new(1613.1194444444443, 1267.425),\r\n Point::new(1842.7055555555555, 1525.8166666666666),\r\n Point::new(1957.75, 1625.75),\r\n Point::new(1934.0, 1554.0),\r\n ];\r\n assert!(spline.is_some());\r\n let spline = spline.unwrap();\r\n assert_eq!(spline.points().len(), expected.len());\r\n for (got, &wanted) in spline.points().iter().zip(expected.iter()) {\r\n assert!(got.distance(wanted) < 5.0)\r\n }\r\n }\r\n\r\n #[test]\r\n fn cubicbez_cubics_to_quadratic_splines() {\r\n let curves = vec![\r\n CubicBez::new(\r\n (550.0, 258.0),\r\n (1044.0, 482.0),\r\n (2029.0, 1841.0),\r\n (1934.0, 1554.0),\r\n ),\r\n CubicBez::new(\r\n (859.0, 384.0),\r\n (1998.0, 116.0),\r\n (1596.0, 1772.0),\r\n (8.0, 1824.0),\r\n ),\r\n CubicBez::new(\r\n (1090.0, 937.0),\r\n (418.0, 1300.0),\r\n (125.0, 91.0),\r\n (104.0, 37.0),\r\n ),\r\n ];\r\n let converted = cubics_to_quadratic_splines(&curves, 5.0);\r\n assert!(converted.is_some());\r\n let converted = converted.unwrap();\r\n assert_eq!(converted[0].points().len(), 8);\r\n assert_eq!(converted[1].points().len(), 8);\r\n assert_eq!(converted[2].points().len(), 8);\r\n assert!(converted[0].points()[1].distance(Point::new(673.5, 314.0)) < 0.0001);\r\n assert!(\r\n converted[0].points()[2].distance(Point::new(88639.0 / 90.0, 52584.0 / 90.0)) < 0.0001\r\n );\r\n }\r\n\r\n #[test]\r\n fn cubicbez_approx_spline_div_exact() {\r\n // Ensure rounding behavior for division matches fonttools\r\n // cu2qu.\r\n // See \r\n let cubic = CubicBez::new(\r\n Point::new(408.0, 321.0),\r\n Point::new(408.0, 452.0),\r\n Point::new(342.0, 560.0),\r\n Point::new(260.0, 560.0),\r\n );\r\n let spline = cubic.approx_spline(1.0).unwrap();\r\n assert_eq!(\r\n spline.points(),\r\n &[\r\n Point::new(408.0, 321.0),\r\n // Previous behavior produced 386.49999999999994 for the\r\n // y coordinate leading to inconsistent rounding.\r\n Point::new(408.0, 386.5),\r\n Point::new(368.16666666666663, 495.0833333333333),\r\n Point::new(301.0, 560.0),\r\n Point::new(260.0, 560.0)\r\n ]\r\n )\r\n }\r\n\r\n #[test]\r\n fn cubicbez_inflections() {\r\n let c = CubicBez::new((0., 0.), (0.8, 1.), (0.2, 1.), (1., 0.));\r\n let inflections = c.inflections();\r\n assert_eq!(inflections.len(), 2);\r\n assert!((inflections[0] - 0.311018).abs() < 1e-6);\r\n assert!((inflections[1] - 0.688982).abs() < 1e-6);\r\n let c = CubicBez::new((0., 0.), (1., 1.), (2., -1.), (3., 0.));\r\n let inflections = c.inflections();\r\n assert_eq!(inflections.len(), 1);\r\n assert!((inflections[0] - 0.5).abs() < 1e-6);\r\n let c = CubicBez::new((0., 0.), (1., 1.), (2., 1.), (3., 0.));\r\n let inflections = c.inflections();\r\n assert_eq!(inflections.len(), 0);\r\n }\r\n\r\n #[test]\r\n fn cubic_to_quadratic_matches_python() {\r\n // from https://github.com/googlefonts/fontmake-rs/issues/217\r\n let cubic = CubicBez {\r\n p0: (796.0, 319.0).into(),\r\n p1: (727.0, 314.0).into(),\r\n p2: (242.0, 303.0).into(),\r\n p3: (106.0, 303.0).into(),\r\n };\r\n\r\n // FontTools can approximate this curve successfully in 7 splits, we can too\r\n assert!(cubic.approx_spline_n(7, 1.0).is_some());\r\n\r\n // FontTools can solve this with accuracy 0.001, we can too\r\n assert!(cubics_to_quadratic_splines(&[cubic], 0.001).is_some());\r\n }\r\n\r\n #[test]\r\n fn cubics_to_quadratic_splines_matches_python() {\r\n // https://github.com/linebender/kurbo/pull/273\r\n let light = CubicBez::new((378., 608.), (378., 524.), (355., 455.), (266., 455.));\r\n let regular = CubicBez::new((367., 607.), (367., 511.), (338., 472.), (243., 472.));\r\n let bold = CubicBez::new(\r\n (372.425, 593.05),\r\n (372.425, 524.95),\r\n (355.05, 485.95),\r\n (274., 485.95),\r\n );\r\n let qsplines = cubics_to_quadratic_splines(&[light, regular, bold], 1.0).unwrap();\r\n assert_eq!(\r\n qsplines,\r\n [\r\n QuadSpline::new(vec![\r\n (378.0, 608.0).into(),\r\n (378.0, 566.0).into(),\r\n (359.0833333333333, 496.5).into(),\r\n (310.5, 455.0).into(),\r\n (266.0, 455.0).into(),\r\n ]),\r\n QuadSpline::new(vec![\r\n (367.0, 607.0).into(),\r\n (367.0, 559.0).into(),\r\n // Previous behavior produced 496.5 for the y coordinate\r\n (344.5833333333333, 499.49999999999994).into(),\r\n (290.5, 472.0).into(),\r\n (243.0, 472.0).into(),\r\n ]),\r\n QuadSpline::new(vec![\r\n (372.425, 593.05).into(),\r\n (372.425, 559.0).into(),\r\n (356.98333333333335, 511.125).into(),\r\n (314.525, 485.95).into(),\r\n (274.0, 485.95).into(),\r\n ]),\r\n ]\r\n )\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/ellipse.rs", "sha256": "908b1a43b6c9c5ed7e2a26fa10e4a1e97fb3fb6951c24f549eea71857f1b5aee", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Implementation of ellipse shape.\r\n\r\nuse core::f64::consts::PI;\r\nuse core::{\r\n iter,\r\n ops::{Add, Mul, Sub},\r\n};\r\n\r\nuse crate::{Affine, Arc, ArcAppendIter, Circle, PathEl, Point, Rect, Shape, Size, Vec2};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// An ellipse.\r\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct Ellipse {\r\n /// All ellipses can be represented as an affine map of the unit circle,\r\n /// centred at (0, 0). Therefore we can store the ellipse as an affine map,\r\n /// with the implication that it be applied to the unit circle to recover the\r\n /// actual shape.\r\n inner: Affine,\r\n}\r\n\r\nimpl Ellipse {\r\n /// Create A new ellipse with a given center, radii, and rotation.\r\n ///\r\n /// The returned ellipse will be the result of taking a circle, stretching\r\n /// it by the `radii` along the x and y axes, then rotating it from the\r\n /// x asix by `rotation` radians, before finally translating the center\r\n /// to `center`.\r\n ///\r\n /// Rotation is clockwise in a y-down coordinate system. For more on\r\n /// rotation, see [`Affine::rotate`].\r\n #[inline]\r\n pub fn new(center: impl Into, radii: impl Into, x_rotation: f64) -> Ellipse {\r\n let Point { x: cx, y: cy } = center.into();\r\n let Vec2 { x: rx, y: ry } = radii.into();\r\n Ellipse::private_new(Vec2 { x: cx, y: cy }, rx, ry, x_rotation)\r\n }\r\n\r\n /// Returns the largest ellipse that can be bounded by this [`Rect`].\r\n ///\r\n /// This uses the absolute width and height of the rectangle.\r\n ///\r\n /// This ellipse is always axis-aligned; to apply rotation you can call\r\n /// [`with_rotation`] with the result.\r\n ///\r\n /// [`with_rotation`]: Ellipse::with_rotation\r\n #[inline]\r\n pub fn from_rect(rect: Rect) -> Self {\r\n let center = rect.center().to_vec2();\r\n let Size { width, height } = rect.size() / 2.0;\r\n Ellipse::private_new(center, width, height, 0.0)\r\n }\r\n\r\n /// Create an ellipse from an affine transformation of the unit circle.\r\n #[inline]\r\n pub fn from_affine(affine: Affine) -> Self {\r\n Ellipse { inner: affine }\r\n }\r\n\r\n /// Create a new `Ellipse` centered on the provided point.\r\n #[inline]\r\n #[must_use]\r\n pub fn with_center(self, new_center: Point) -> Ellipse {\r\n let Point { x: cx, y: cy } = new_center;\r\n Ellipse {\r\n inner: self.inner.with_translation(Vec2 { x: cx, y: cy }),\r\n }\r\n }\r\n\r\n /// Create a new `Ellipse` with the provided radii.\r\n #[must_use]\r\n pub fn with_radii(self, new_radii: Vec2) -> Ellipse {\r\n let rotation = self.inner.svd().1;\r\n let translation = self.inner.translation();\r\n Ellipse::private_new(translation, new_radii.x, new_radii.y, rotation)\r\n }\r\n\r\n /// Create a new `Ellipse`, with the rotation replaced by `rotation`\r\n /// radians.\r\n ///\r\n /// The rotation is clockwise, for a y-down coordinate system. For more\r\n /// on rotation, See [`Affine::rotate`].\r\n #[must_use]\r\n pub fn with_rotation(self, rotation: f64) -> Ellipse {\r\n let scale = self.inner.svd().0;\r\n let translation = self.inner.translation();\r\n Ellipse::private_new(translation, scale.x, scale.y, rotation)\r\n }\r\n\r\n #[deprecated(since = \"0.7.0\", note = \"use with_rotation instead\")]\r\n #[must_use]\r\n #[doc(hidden)]\r\n pub fn with_x_rotation(self, rotation_radians: f64) -> Ellipse {\r\n self.with_rotation(rotation_radians)\r\n }\r\n\r\n /// This gives us an internal method without any type conversions.\r\n #[inline]\r\n fn private_new(center: Vec2, scale_x: f64, scale_y: f64, x_rotation: f64) -> Ellipse {\r\n // Since the circle is symmetric about the x and y axes, using absolute values for the\r\n // radii results in the same ellipse. For simplicity we make this change here.\r\n Ellipse {\r\n inner: Affine::translate(center)\r\n * Affine::rotate(x_rotation)\r\n * Affine::scale_non_uniform(scale_x.abs(), scale_y.abs()),\r\n }\r\n }\r\n\r\n // Getters and setters.\r\n\r\n /// Returns the center of this ellipse.\r\n #[inline]\r\n pub fn center(&self) -> Point {\r\n let Vec2 { x: cx, y: cy } = self.inner.translation();\r\n Point { x: cx, y: cy }\r\n }\r\n\r\n /// Returns the two radii of this ellipse.\r\n ///\r\n /// The first number is the horizontal radius and the second is the vertical\r\n /// radius, before rotation.\r\n pub fn radii(&self) -> Vec2 {\r\n self.inner.svd().0\r\n }\r\n\r\n /// The ellipse's rotation, in radians.\r\n ///\r\n /// This allows all possible ellipses to be drawn by always starting with\r\n /// an ellipse with the two radii on the x and y axes.\r\n pub fn rotation(&self) -> f64 {\r\n self.inner.svd().1\r\n }\r\n\r\n /// Returns the radii and the rotation of this ellipse.\r\n ///\r\n /// Equivalent to `(self.radii(), self.rotation())` but more efficient.\r\n pub fn radii_and_rotation(&self) -> (Vec2, f64) {\r\n self.inner.svd()\r\n }\r\n\r\n /// Is this ellipse finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n self.inner.is_finite()\r\n }\r\n\r\n /// Is this ellipse NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n self.inner.is_nan()\r\n }\r\n\r\n #[doc(hidden)]\r\n #[deprecated(since = \"0.7.0\", note = \"use rotation() instead\")]\r\n pub fn x_rotation(&self) -> f64 {\r\n self.rotation()\r\n }\r\n}\r\n\r\nimpl Add for Ellipse {\r\n type Output = Ellipse;\r\n\r\n /// In this context adding a `Vec2` applies the corresponding translation to the ellipse.\r\n #[inline]\r\n #[allow(clippy::suspicious_arithmetic_impl)]\r\n fn add(self, v: Vec2) -> Ellipse {\r\n Ellipse {\r\n inner: Affine::translate(v) * self.inner,\r\n }\r\n }\r\n}\r\n\r\nimpl Sub for Ellipse {\r\n type Output = Ellipse;\r\n\r\n /// In this context subtracting a `Vec2` applies the corresponding translation to the ellipse.\r\n #[inline]\r\n fn sub(self, v: Vec2) -> Ellipse {\r\n Ellipse {\r\n inner: Affine::translate(-v) * self.inner,\r\n }\r\n }\r\n}\r\n\r\nimpl Mul for Affine {\r\n type Output = Ellipse;\r\n fn mul(self, other: Ellipse) -> Self::Output {\r\n Ellipse {\r\n inner: self * other.inner,\r\n }\r\n }\r\n}\r\n\r\nimpl From for Ellipse {\r\n fn from(circle: Circle) -> Self {\r\n Ellipse::new(circle.center, Vec2::splat(circle.radius), 0.0)\r\n }\r\n}\r\n\r\nimpl Shape for Ellipse {\r\n type PathElementsIter<'iter> = iter::Chain, ArcAppendIter>;\r\n\r\n fn path_elements(&self, tolerance: f64) -> Self::PathElementsIter<'_> {\r\n let (radii, x_rotation) = self.inner.svd();\r\n Arc {\r\n center: self.center(),\r\n radii,\r\n start_angle: 0.0,\r\n sweep_angle: 2.0 * PI,\r\n x_rotation,\r\n }\r\n .path_elements(tolerance)\r\n }\r\n\r\n #[inline]\r\n fn area(&self) -> f64 {\r\n let Vec2 { x, y } = self.radii();\r\n PI * x * y\r\n }\r\n\r\n #[inline]\r\n fn perimeter(&self, accuracy: f64) -> f64 {\r\n // TODO rather than delegate to the bezier path, it is possible to use various series\r\n // expansions to compute the perimeter to any accuracy. I believe Ramanujan authored the\r\n // quickest to converge. See\r\n // https://www.mathematica-journal.com/2009/11/23/on-the-perimeter-of-an-ellipse/\r\n // and https://en.wikipedia.org/wiki/Ellipse#Circumference\r\n //\r\n self.path_segments(0.1).perimeter(accuracy)\r\n }\r\n\r\n fn winding(&self, pt: Point) -> i32 {\r\n // Strategy here is to apply the inverse map to the point and see if it is in the unit\r\n // circle.\r\n let inv = self.inner.inverse();\r\n if (inv * pt).to_vec2().hypot2() < 1.0 {\r\n 1\r\n } else {\r\n 0\r\n }\r\n }\r\n\r\n // Compute a tight bounding box of the ellipse.\r\n //\r\n // See https://www.iquilezles.org/www/articles/ellipses/ellipses.htm. We can get the two\r\n // radius vectors by applying the affine map to the two impulses (1, 0) and (0, 1) which gives\r\n // (a, b) and (c, d) if the affine map is\r\n //\r\n // a | c | e\r\n // -----------\r\n // b | d | f\r\n //\r\n // We can then use the method in the link with the translation to get the bounding box.\r\n #[inline]\r\n fn bounding_box(&self) -> Rect {\r\n let aff = self.inner.as_coeffs();\r\n let a2 = aff[0] * aff[0];\r\n let b2 = aff[1] * aff[1];\r\n let c2 = aff[2] * aff[2];\r\n let d2 = aff[3] * aff[3];\r\n let cx = aff[4];\r\n let cy = aff[5];\r\n let range_x = (a2 + c2).sqrt();\r\n let range_y = (b2 + d2).sqrt();\r\n Rect {\r\n x0: cx - range_x,\r\n y0: cy - range_y,\r\n x1: cx + range_x,\r\n y1: cy + range_y,\r\n }\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{Ellipse, Point, Shape};\r\n use std::f64::consts::PI;\r\n\r\n fn assert_approx_eq(x: f64, y: f64) {\r\n // Note: we might want to be more rigorous in testing the accuracy\r\n // of the conversion into Béziers. But this seems good enough.\r\n assert!((x - y).abs() < 1e-7, \"{x} != {y}\");\r\n }\r\n\r\n #[test]\r\n fn area_sign() {\r\n let center = Point::new(5.0, 5.0);\r\n let e = Ellipse::new(center, (5.0, 5.0), 1.0);\r\n assert_approx_eq(e.area(), 25.0 * PI);\r\n let e = Ellipse::new(center, (5.0, 10.0), 1.0);\r\n assert_approx_eq(e.area(), 50.0 * PI);\r\n\r\n assert_eq!(e.winding(center), 1);\r\n\r\n let p = e.to_path(1e-9);\r\n assert_approx_eq(e.area(), p.area());\r\n assert_eq!(e.winding(center), p.winding(center));\r\n\r\n let e_neg_radius = Ellipse::new(center, (-5.0, 10.0), 1.0);\r\n assert_approx_eq(e_neg_radius.area(), 50.0 * PI);\r\n\r\n assert_eq!(e_neg_radius.winding(center), 1);\r\n\r\n let p_neg_radius = e_neg_radius.to_path(1e-9);\r\n assert_approx_eq(e_neg_radius.area(), p_neg_radius.area());\r\n assert_eq!(e_neg_radius.winding(center), p_neg_radius.winding(center));\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/fit.rs", "sha256": "2eded2d6b2bc5e18c563b224fc805c49fb0915dc87369ea8ba47b0647444343e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! An implementation of cubic Bézier curve fitting based on a quartic\r\n//! solver making signed area and moment match the source curve.\r\n\r\nuse core::ops::Range;\r\n\r\nuse alloc::vec::Vec;\r\n\r\nuse arrayvec::ArrayVec;\r\n\r\nuse crate::{\r\n common::{\r\n factor_quartic_inner, solve_cubic, solve_itp_fallible, solve_quadratic,\r\n GAUSS_LEGENDRE_COEFFS_16,\r\n },\r\n Affine, BezPath, CubicBez, ParamCurve, ParamCurveArclen, Point, Vec2,\r\n};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// The source curve for curve fitting.\r\n///\r\n/// This trait is a general representation of curves to be used as input to a curve\r\n/// fitting process. It can represent source curves with cusps and corners, though\r\n/// if the corners are known in advance, it may be better to run curve fitting on\r\n/// subcurves bounded by the corners.\r\n///\r\n/// The trait primarily works by sampling the source curve and computing the position\r\n/// and derivative at each sample. Those derivatives are then used for multiple\r\n/// sub-tasks, including ensuring G1 continuity at subdivision points, computing the\r\n/// area and moment of the curve for curve fitting, and casting rays for evaluation\r\n/// of a distance metric to test accuracy.\r\n///\r\n/// A major motivation is computation of offset curves, which often have cusps, but\r\n/// the presence and location of those cusps is not generally known. It is also\r\n/// intended for conversion between curve types (for example, piecewise Euler spiral\r\n/// or NURBS), and distortion effects such as perspective transform.\r\n///\r\n/// Note general similarities to [`ParamCurve`] but also important differences.\r\n/// Instead of separate [`eval`] and evaluation of derivative, have a single\r\n/// [`sample_pt_deriv`] method which can be more efficient and also handles cusps more\r\n/// robustly. Also there is no method for subsegment, as that is not needed and\r\n/// would be annoying to implement.\r\n///\r\n/// [`ParamCurve`]: crate::ParamCurve\r\n/// [`eval`]: crate::ParamCurve::eval\r\n/// [`sample_pt_deriv`]: ParamCurveFit::sample_pt_deriv\r\npub trait ParamCurveFit {\r\n /// Evaluate the curve and its tangent at parameter `t`.\r\n ///\r\n /// For a regular curve (one not containing a cusp or corner), the\r\n /// derivative is a good choice for the tangent vector and the `sign`\r\n /// parameter can be ignored. Otherwise, the `sign` parameter selects which\r\n /// side of the discontinuity the tangent will be sampled from.\r\n ///\r\n /// Generally `t` is in the range [0..1].\r\n fn sample_pt_tangent(&self, t: f64, sign: f64) -> CurveFitSample;\r\n\r\n /// Evaluate the point and derivative at parameter `t`.\r\n ///\r\n /// In curves with cusps, the derivative can go to zero.\r\n fn sample_pt_deriv(&self, t: f64) -> (Point, Vec2);\r\n\r\n /// Compute moment integrals.\r\n ///\r\n /// This method computes the integrals of y dx, x y dx, and y^2 dx over the\r\n /// length of this curve. From these integrals it is fairly straightforward\r\n /// to derive the moments needed for curve fitting.\r\n ///\r\n /// A default implementation is proved which does quadrature integration\r\n /// with Green's theorem, in terms of samples evaulated with\r\n /// [`sample_pt_deriv`].\r\n ///\r\n /// [`sample_pt_deriv`]: ParamCurveFit::sample_pt_deriv\r\n fn moment_integrals(&self, range: Range) -> (f64, f64, f64) {\r\n let t0 = 0.5 * (range.start + range.end);\r\n let dt = 0.5 * (range.end - range.start);\r\n let (a, x, y) = GAUSS_LEGENDRE_COEFFS_16\r\n .iter()\r\n .map(|(wi, xi)| {\r\n let t = t0 + xi * dt;\r\n let (p, d) = self.sample_pt_deriv(t);\r\n let a = wi * d.x * p.y;\r\n let x = p.x * a;\r\n let y = p.y * a;\r\n (a, x, y)\r\n })\r\n .fold((0.0, 0.0, 0.0), |(a0, x0, y0), (a1, x1, y1)| {\r\n (a0 + a1, x0 + x1, y0 + y1)\r\n });\r\n (a * dt, x * dt, y * dt)\r\n }\r\n\r\n /// Find a cusp or corner within the given range.\r\n ///\r\n /// If the range contains a corner or cusp, return it. If there is more\r\n /// than one such discontinuity, any can be reported, as the function will\r\n /// be called repeatedly after subdivision of the range.\r\n ///\r\n /// Do not report cusps at the endpoints of the range, as this may cause\r\n /// potentially infinite subdivision. In particular, when a cusp is reported\r\n /// and this method is called on a subdivided range bounded by the reported\r\n /// cusp, then the subsequent call should not report a cusp there.\r\n ///\r\n /// The definition of what exactly constitutes a cusp is somewhat loose.\r\n /// If a cusp is missed, then the curve fitting algorithm will attempt to\r\n /// fit the curve with a smooth curve, which is generally not a disaster\r\n /// will usually result in more subdivision. Conversely, it might be useful\r\n /// to report near-cusps, specifically points of curvature maxima where the\r\n /// curvature is large but still mathematically finite.\r\n fn break_cusp(&self, range: Range) -> Option;\r\n}\r\n\r\n/// A sample point of a curve for fitting.\r\npub struct CurveFitSample {\r\n /// A point on the curve at the sample location.\r\n pub p: Point,\r\n /// A vector tangent to the curve at the sample location.\r\n pub tangent: Vec2,\r\n}\r\n\r\nimpl CurveFitSample {\r\n /// Intersect a ray orthogonal to the tangent with the given cubic.\r\n ///\r\n /// Returns a vector of `t` values on the cubic.\r\n fn intersect(&self, c: CubicBez) -> ArrayVec {\r\n let p1 = 3.0 * (c.p1 - c.p0);\r\n let p2 = 3.0 * c.p2.to_vec2() - 6.0 * c.p1.to_vec2() + 3.0 * c.p0.to_vec2();\r\n let p3 = (c.p3 - c.p0) - 3.0 * (c.p2 - c.p1);\r\n let c0 = (c.p0 - self.p).dot(self.tangent);\r\n let c1 = p1.dot(self.tangent);\r\n let c2 = p2.dot(self.tangent);\r\n let c3 = p3.dot(self.tangent);\r\n solve_cubic(c0, c1, c2, c3)\r\n .into_iter()\r\n .filter(|t| (0.0..=1.0).contains(t))\r\n .collect()\r\n }\r\n}\r\n\r\n/// Generate a Bézier path that fits the source curve.\r\n///\r\n/// This function is still experimental and the signature might change; it's possible\r\n/// it might become a method on the [`ParamCurveFit`] trait.\r\n///\r\n/// This function recursively subdivides the curve in half by the parameter when the\r\n/// accuracy is not met. That gives a reasonably optimized result but not necessarily\r\n/// the minimum number of segments.\r\n///\r\n/// In general, the resulting Bézier path should have a Fréchet distance less than\r\n/// the provided `accuracy` parameter. However, this is not a rigorous guarantee, as\r\n/// the error metric is computed approximately.\r\n///\r\n/// When a higher degree of optimization is desired (at considerably more runtime cost),\r\n/// consider [`fit_to_bezpath_opt`] instead.\r\npub fn fit_to_bezpath(source: &impl ParamCurveFit, accuracy: f64) -> BezPath {\r\n let mut path = BezPath::new();\r\n fit_to_bezpath_rec(source, 0.0..1.0, accuracy, &mut path);\r\n path\r\n}\r\n\r\n// Discussion question: possibly should take endpoint samples, to avoid\r\n// duplication of that work.\r\nfn fit_to_bezpath_rec(\r\n source: &impl ParamCurveFit,\r\n range: Range,\r\n accuracy: f64,\r\n path: &mut BezPath,\r\n) {\r\n let start = range.start;\r\n let end = range.end;\r\n if let Some(t) = source.break_cusp(start..end) {\r\n fit_to_bezpath_rec(source, start..t, accuracy, path);\r\n fit_to_bezpath_rec(source, t..end, accuracy, path);\r\n } else if let Some((c, _)) = fit_to_cubic(source, start..end, accuracy) {\r\n if path.is_empty() {\r\n path.move_to(c.p0);\r\n }\r\n path.curve_to(c.p1, c.p2, c.p3);\r\n } else {\r\n // A smarter approach is possible than midpoint subdivision, but would be\r\n // a significant increase in complexity.\r\n let t = 0.5 * (start + end);\r\n fit_to_bezpath_rec(source, start..t, accuracy, path);\r\n fit_to_bezpath_rec(source, t..end, accuracy, path);\r\n }\r\n}\r\n\r\nconst N_SAMPLE: usize = 20;\r\n\r\n/// An acceleration structure for estimating curve distance\r\nstruct CurveDist {\r\n samples: ArrayVec,\r\n arcparams: ArrayVec,\r\n range: Range,\r\n /// A \"spicy\" curve is one with potentially extreme curvature variation,\r\n /// so use arc length measurement for better accuracy.\r\n spicy: bool,\r\n}\r\n\r\nimpl CurveDist {\r\n fn from_curve(source: &impl ParamCurveFit, range: Range) -> Self {\r\n let step = (range.end - range.start) * (1.0 / (N_SAMPLE + 1) as f64);\r\n let mut last_tan = None;\r\n let mut spicy = false;\r\n const SPICY_THRESH: f64 = 0.2;\r\n let mut samples = ArrayVec::new();\r\n for i in 0..N_SAMPLE + 2 {\r\n let sample = source.sample_pt_tangent(range.start + i as f64 * step, 1.0);\r\n if let Some(last_tan) = last_tan {\r\n let cross = sample.tangent.cross(last_tan);\r\n let dot = sample.tangent.dot(last_tan);\r\n if cross.abs() > SPICY_THRESH * dot.abs() {\r\n spicy = true;\r\n }\r\n }\r\n last_tan = Some(sample.tangent);\r\n if i > 0 && i < N_SAMPLE + 1 {\r\n samples.push(sample);\r\n }\r\n }\r\n CurveDist {\r\n samples,\r\n arcparams: Default::default(),\r\n range,\r\n spicy,\r\n }\r\n }\r\n\r\n fn compute_arc_params(&mut self, source: &impl ParamCurveFit) {\r\n const N_SUBSAMPLE: usize = 10;\r\n let (start, end) = (self.range.start, self.range.end);\r\n let dt = (end - start) * (1.0 / ((N_SAMPLE + 1) * N_SUBSAMPLE) as f64);\r\n let mut arclen = 0.0;\r\n for i in 0..N_SAMPLE + 1 {\r\n for j in 0..N_SUBSAMPLE {\r\n let t = start + dt * ((i * N_SUBSAMPLE + j) as f64 + 0.5);\r\n let (_, deriv) = source.sample_pt_deriv(t);\r\n arclen += deriv.hypot();\r\n }\r\n if i < N_SAMPLE {\r\n self.arcparams.push(arclen);\r\n }\r\n }\r\n let arclen_inv = arclen.recip();\r\n for x in &mut self.arcparams {\r\n *x *= arclen_inv;\r\n }\r\n }\r\n\r\n /// Evaluate distance based on arc length parametrization\r\n fn eval_arc(&self, c: CubicBez, acc2: f64) -> Option {\r\n // TODO: this could perhaps be tuned.\r\n const EPS: f64 = 1e-9;\r\n let c_arclen = c.arclen(EPS);\r\n let mut max_err2 = 0.0;\r\n for (sample, s) in self.samples.iter().zip(&self.arcparams) {\r\n let t = c.inv_arclen(c_arclen * s, EPS);\r\n let err = sample.p.distance_squared(c.eval(t));\r\n max_err2 = err.max(max_err2);\r\n if max_err2 > acc2 {\r\n return None;\r\n }\r\n }\r\n Some(max_err2)\r\n }\r\n\r\n /// Evaluate distance to a cubic approximation.\r\n ///\r\n /// If distance exceeds stated accuracy, can return None. Note that\r\n /// `acc2` is the square of the target.\r\n ///\r\n /// Returns the squre of the error, which is intended to be a good\r\n /// approximation of the Fréchet distance.\r\n fn eval_ray(&self, c: CubicBez, acc2: f64) -> Option {\r\n let mut max_err2 = 0.0;\r\n for sample in &self.samples {\r\n let mut best = acc2 + 1.0;\r\n for t in sample.intersect(c) {\r\n let err = sample.p.distance_squared(c.eval(t));\r\n best = best.min(err);\r\n }\r\n max_err2 = best.max(max_err2);\r\n if max_err2 > acc2 {\r\n return None;\r\n }\r\n }\r\n Some(max_err2)\r\n }\r\n\r\n fn eval_dist(&mut self, source: &impl ParamCurveFit, c: CubicBez, acc2: f64) -> Option {\r\n // Always compute cheaper distance, hoping for early-out.\r\n let ray_dist = self.eval_ray(c, acc2)?;\r\n if !self.spicy {\r\n return Some(ray_dist);\r\n }\r\n if self.arcparams.is_empty() {\r\n self.compute_arc_params(source);\r\n }\r\n self.eval_arc(c, acc2)\r\n }\r\n}\r\n\r\n/// As described in [Simplifying Bézier paths], strictly optimizing for\r\n/// Fréchet distance can create bumps. The problem is curves with long\r\n/// control arms (distance from the control point to the corresponding\r\n/// endpoint). We mitigate that by applying a penalty as a multiplier to\r\n/// the measured error (approximate Fréchet distance). This is ReLU-like,\r\n/// with a value of 1.0 below the elbow, and a given slope above it. The\r\n/// values here have been determined empirically to give good results.\r\n///\r\n/// [Simplifying Bézier paths]:\r\n/// https://raphlinus.github.io/curves/2023/04/18/bezpath-simplify.html\r\nconst D_PENALTY_ELBOW: f64 = 0.65;\r\nconst D_PENALTY_SLOPE: f64 = 2.0;\r\n\r\n/// Fit a single cubic to a range of the source curve.\r\n///\r\n/// Returns the cubic segment and the square of the error.\r\n/// Discussion question: should this be a method on the trait instead?\r\npub fn fit_to_cubic(\r\n source: &impl ParamCurveFit,\r\n range: Range,\r\n accuracy: f64,\r\n) -> Option<(CubicBez, f64)> {\r\n // TODO: handle case where chord is short; return `None` if subdivision\r\n // will be useful (ie resulting chord is longer), otherwise simple short\r\n // cubic (maybe raised line).\r\n\r\n let start = source.sample_pt_tangent(range.start, 1.0);\r\n let end = source.sample_pt_tangent(range.end, -1.0);\r\n let d = end.p - start.p;\r\n let th = d.atan2();\r\n let chord2 = d.hypot2();\r\n fn mod_2pi(th: f64) -> f64 {\r\n let th_scaled = th * core::f64::consts::FRAC_1_PI * 0.5;\r\n core::f64::consts::PI * 2.0 * (th_scaled - th_scaled.round())\r\n }\r\n let th0 = mod_2pi(start.tangent.atan2() - th);\r\n let th1 = mod_2pi(th - end.tangent.atan2());\r\n\r\n let (mut area, mut x, mut y) = source.moment_integrals(range.clone());\r\n let (x0, y0) = (start.p.x, start.p.y);\r\n let (dx, dy) = (d.x, d.y);\r\n // Subtract off area of chord\r\n area -= dx * (y0 + 0.5 * dy);\r\n // `area` is signed area of closed curve segment.\r\n // This quantity is invariant to translation and rotation.\r\n\r\n // Subtract off moment of chord\r\n let dy_3 = dy * (1. / 3.);\r\n x -= dx * (x0 * y0 + 0.5 * (x0 * dy + y0 * dx) + dy_3 * dx);\r\n y -= dx * (y0 * y0 + y0 * dy + dy_3 * dy);\r\n // Translate start point to origin; convert raw integrals to moments.\r\n x -= x0 * area;\r\n y = 0.5 * y - y0 * area;\r\n // Rotate into place (this also scales up by chordlength for efficiency).\r\n let moment = d.x * x + d.y * y;\r\n // `moment` is the chordlength times the x moment of the curve translated\r\n // so its start point is on the origin, and rotated so its end point is on the\r\n // x axis.\r\n\r\n let chord2_inv = chord2.recip();\r\n let unit_area = area * chord2_inv;\r\n let mx = moment * chord2_inv.powi(2);\r\n // `unit_area` is signed area scaled to unit chord; `mx` is scaled x moment\r\n\r\n let chord = chord2.sqrt();\r\n let aff = Affine::translate(start.p.to_vec2()) * Affine::rotate(th) * Affine::scale(chord);\r\n let mut curve_dist = CurveDist::from_curve(source, range);\r\n let acc2 = accuracy * accuracy;\r\n let mut best_c = None;\r\n let mut best_err2 = None;\r\n for (cand, d0, d1) in cubic_fit(th0, th1, unit_area, mx) {\r\n let c = aff * cand;\r\n if let Some(err2) = curve_dist.eval_dist(source, c, acc2) {\r\n fn scale_f(d: f64) -> f64 {\r\n 1.0 + (d - D_PENALTY_ELBOW).max(0.0) * D_PENALTY_SLOPE\r\n }\r\n let scale = scale_f(d0).max(scale_f(d1)).powi(2);\r\n let err2 = err2 * scale;\r\n if err2 < acc2 && best_err2.map(|best| err2 < best).unwrap_or(true) {\r\n best_c = Some(c);\r\n best_err2 = Some(err2);\r\n }\r\n }\r\n }\r\n match (best_c, best_err2) {\r\n (Some(c), Some(err2)) => Some((c, err2)),\r\n _ => None,\r\n }\r\n}\r\n\r\n/// Returns curves matching area and moment, given unit chord.\r\nfn cubic_fit(th0: f64, th1: f64, area: f64, mx: f64) -> ArrayVec<(CubicBez, f64, f64), 4> {\r\n // Note: maybe we want to take unit vectors instead of angle? Shouldn't\r\n // matter much either way though.\r\n let (s0, c0) = th0.sin_cos();\r\n let (s1, c1) = th1.sin_cos();\r\n let a4 = -9.\r\n * c0\r\n * (((2. * s1 * c1 * c0 + s0 * (2. * c1 * c1 - 1.)) * c0 - 2. * s1 * c1) * c0\r\n - c1 * c1 * s0);\r\n let a3 = 12.\r\n * ((((c1 * (30. * area * c1 - s1) - 15. * area) * c0 + 2. * s0\r\n - c1 * s0 * (c1 + 30. * area * s1))\r\n * c0\r\n + c1 * (s1 - 15. * area * c1))\r\n * c0\r\n - s0 * c1 * c1);\r\n let a2 = 12.\r\n * ((((70. * mx + 15. * area) * s1 * s1 + c1 * (9. * s1 - 70. * c1 * mx - 5. * c1 * area))\r\n * c0\r\n - 5. * s0 * s1 * (3. * s1 - 4. * c1 * (7. * mx + area)))\r\n * c0\r\n - c1 * (9. * s1 - 70. * c1 * mx - 5. * c1 * area));\r\n let a1 = 16.\r\n * (((12. * s0 - 5. * c0 * (42. * mx - 17. * area)) * s1\r\n - 70. * c1 * (3. * mx - area) * s0\r\n - 75. * c0 * c1 * area * area)\r\n * s1\r\n - 75. * c1 * c1 * area * area * s0);\r\n let a0 = 80. * s1 * (42. * s1 * mx - 25. * area * (s1 - c1 * area));\r\n // TODO: \"roots\" is not a good name for this variable, as it also contains\r\n // the real part of complex conjugate pairs.\r\n let mut roots = ArrayVec::::new();\r\n const EPS: f64 = 1e-12;\r\n if a4.abs() > EPS {\r\n let a = a3 / a4;\r\n let b = a2 / a4;\r\n let c = a1 / a4;\r\n let d = a0 / a4;\r\n if let Some(quads) = factor_quartic_inner(a, b, c, d, false) {\r\n for (qc1, qc0) in quads {\r\n let qroots = solve_quadratic(qc0, qc1, 1.0);\r\n if qroots.is_empty() {\r\n // Real part of pair of complex roots\r\n roots.push(-0.5 * qc1);\r\n } else {\r\n roots.extend(qroots);\r\n }\r\n }\r\n }\r\n } else if a3.abs() > EPS {\r\n roots.extend(solve_cubic(a0, a1, a2, a3));\r\n } else {\r\n roots.extend(solve_quadratic(a0, a1, a2));\r\n }\r\n\r\n let s01 = s0 * c1 + s1 * c0;\r\n roots\r\n .iter()\r\n .filter_map(|&d0| {\r\n let (d0, d1) = if d0 > 0.0 {\r\n let d1 = (d0 * s0 - area * (10. / 3.)) / (0.5 * d0 * s01 - s1);\r\n if d1 > 0.0 {\r\n (d0, d1)\r\n } else {\r\n (s1 / s01, 0.0)\r\n }\r\n } else {\r\n (0.0, s0 / s01)\r\n };\r\n // We could implement a maximum d value here.\r\n if d0 >= 0.0 && d1 >= 0.0 {\r\n Some((\r\n CubicBez::new(\r\n (0.0, 0.0),\r\n (d0 * c0, d0 * s0),\r\n (1.0 - d1 * c1, d1 * s1),\r\n (1.0, 0.0),\r\n ),\r\n d0,\r\n d1,\r\n ))\r\n } else {\r\n None\r\n }\r\n })\r\n .collect()\r\n}\r\n\r\n/// Generate a highly optimized Bézier path that fits the source curve.\r\n///\r\n/// This function is still experimental and the signature might change; it's possible\r\n/// it might become a method on the [`ParamCurveFit`] trait.\r\n///\r\n/// This function is considerably slower than [`fit_to_bezpath`], as it computes\r\n/// optimal subdivision points. Its result is expected to be very close to the optimum\r\n/// possible Bézier path for the source curve, in that it has a minimal number of curve\r\n/// segments, and a minimal error over all paths with that number of segments.\r\n///\r\n/// See [`fit_to_bezpath`] for an explanation of the `accuracy` parameter.\r\npub fn fit_to_bezpath_opt(source: &impl ParamCurveFit, accuracy: f64) -> BezPath {\r\n let mut cusps = Vec::new();\r\n let mut path = BezPath::new();\r\n let mut t0 = 0.0;\r\n loop {\r\n let t1 = cusps.last().copied().unwrap_or(1.0);\r\n match fit_to_bezpath_opt_inner(source, accuracy, t0..t1, &mut path) {\r\n Some(t) => cusps.push(t),\r\n None => match cusps.pop() {\r\n Some(t) => t0 = t,\r\n None => break,\r\n },\r\n }\r\n }\r\n path\r\n}\r\n\r\n/// Fit a range without cusps.\r\n///\r\n/// On Ok return, range has been added to the path. On Err return, report a cusp (and don't\r\n/// mutate path).\r\nfn fit_to_bezpath_opt_inner(\r\n source: &impl ParamCurveFit,\r\n accuracy: f64,\r\n range: Range,\r\n path: &mut BezPath,\r\n) -> Option {\r\n if let Some(t) = source.break_cusp(range.clone()) {\r\n return Some(t);\r\n }\r\n let err;\r\n if let Some((c, err2)) = fit_to_cubic(source, range.clone(), accuracy) {\r\n err = err2.sqrt();\r\n if err < accuracy {\r\n if range.start == 0.0 {\r\n path.move_to(c.p0);\r\n }\r\n path.curve_to(c.p1, c.p2, c.p3);\r\n return None;\r\n }\r\n } else {\r\n err = 2.0 * accuracy;\r\n }\r\n let (mut t0, t1) = (range.start, range.end);\r\n let mut n = 0;\r\n let last_err;\r\n loop {\r\n n += 1;\r\n match fit_opt_segment(source, accuracy, t0..t1) {\r\n FitResult::ParamVal(t) => t0 = t,\r\n FitResult::SegmentError(err) => {\r\n last_err = err;\r\n break;\r\n }\r\n FitResult::CuspFound(t) => return Some(t),\r\n }\r\n }\r\n t0 = range.start;\r\n const EPS: f64 = 1e-9;\r\n let f = |x| fit_opt_err_delta(source, x, accuracy, t0..t1, n);\r\n let k1 = 0.2 / accuracy;\r\n let ya = -err;\r\n let yb = accuracy - last_err;\r\n let (_, x) = match solve_itp_fallible(f, 0.0, accuracy, EPS, 1, k1, ya, yb) {\r\n Ok(x) => x,\r\n Err(t) => return Some(t),\r\n };\r\n //println!(\"got fit with n={}, err={}\", n, x);\r\n let path_len = path.elements().len();\r\n for i in 0..n {\r\n let t1 = if i < n - 1 {\r\n match fit_opt_segment(source, x, t0..range.end) {\r\n FitResult::ParamVal(t1) => t1,\r\n FitResult::SegmentError(_) => range.end,\r\n FitResult::CuspFound(t) => {\r\n path.truncate(path_len);\r\n return Some(t);\r\n }\r\n }\r\n } else {\r\n range.end\r\n };\r\n let (c, _) = fit_to_cubic(source, t0..t1, accuracy).unwrap();\r\n if i == 0 && range.start == 0.0 {\r\n path.move_to(c.p0);\r\n }\r\n path.curve_to(c.p1, c.p2, c.p3);\r\n t0 = t1;\r\n if t0 == range.end {\r\n // This is unlikely but could happen when not monotonic.\r\n break;\r\n }\r\n }\r\n None\r\n}\r\n\r\nfn measure_one_seg(source: &impl ParamCurveFit, range: Range, limit: f64) -> Option {\r\n fit_to_cubic(source, range, limit).map(|(_, err2)| err2.sqrt())\r\n}\r\n\r\nenum FitResult {\r\n /// The parameter (`t`) value that meets the desired accuracy.\r\n ParamVal(f64),\r\n /// Error of the measured segment.\r\n SegmentError(f64),\r\n /// The parameter value where a cusp was found.\r\n CuspFound(f64),\r\n}\r\n\r\nfn fit_opt_segment(source: &impl ParamCurveFit, accuracy: f64, range: Range) -> FitResult {\r\n if let Some(t) = source.break_cusp(range.clone()) {\r\n return FitResult::CuspFound(t);\r\n }\r\n let missing_err = accuracy * 2.0;\r\n let err = measure_one_seg(source, range.clone(), accuracy).unwrap_or(missing_err);\r\n if err <= accuracy {\r\n return FitResult::SegmentError(err);\r\n }\r\n let (t0, t1) = (range.start, range.end);\r\n let f = |x| {\r\n if let Some(t) = source.break_cusp(range.clone()) {\r\n return Err(t);\r\n }\r\n let err = measure_one_seg(source, t0..x, accuracy).unwrap_or(missing_err);\r\n Ok(err - accuracy)\r\n };\r\n const EPS: f64 = 1e-9;\r\n let k1 = 2.0 / (t1 - t0);\r\n match solve_itp_fallible(f, t0, t1, EPS, 1, k1, -accuracy, err - accuracy) {\r\n Ok((t1, _)) => FitResult::ParamVal(t1),\r\n Err(t) => FitResult::CuspFound(t),\r\n }\r\n}\r\n\r\n// Ok result is delta error (accuracy - error of last seg).\r\n// Err result is a cusp.\r\nfn fit_opt_err_delta(\r\n source: &impl ParamCurveFit,\r\n accuracy: f64,\r\n limit: f64,\r\n range: Range,\r\n n: usize,\r\n) -> Result {\r\n let (mut t0, t1) = (range.start, range.end);\r\n for _ in 0..n - 1 {\r\n t0 = match fit_opt_segment(source, accuracy, t0..t1) {\r\n FitResult::ParamVal(t0) => t0,\r\n // In this case, n - 1 will work, which of course means the error is highly\r\n // non-monotonic. We should probably harvest that solution.\r\n FitResult::SegmentError(_) => return Ok(accuracy),\r\n FitResult::CuspFound(t) => return Err(t),\r\n }\r\n }\r\n let err = measure_one_seg(source, t0..t1, limit).unwrap_or(accuracy * 2.0);\r\n Ok(accuracy - err)\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/insets.rs", "sha256": "2a6f573577bf95f89964f0b85c2c2da271853caf8ccc1db7f14f3ec8378374ce", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A description of the distances between the edges of two rectangles.\r\n\r\nuse core::ops::{Add, Neg, Sub};\r\n\r\nuse crate::{Rect, Size};\r\n\r\n/// Insets from the edges of a rectangle.\r\n///\r\n///\r\n/// The inset value for each edge can be thought of as a delta computed from\r\n/// the center of the rect to that edge. For instance, with an inset of `2.0` on\r\n/// the x-axis, a rectangle with the origin `(0.0, 0.0)` with that inset added\r\n/// will have the new origin at `(-2.0, 0.0)`.\r\n///\r\n/// Put alternatively, a positive inset represents increased distance from center,\r\n/// and a negative inset represents decreased distance from center.\r\n///\r\n/// # Examples\r\n///\r\n/// Positive insets added to a [`Rect`] produce a larger [`Rect`]:\r\n/// ```\r\n/// # use kurbo::{Insets, Rect};\r\n/// let rect = Rect::from_origin_size((0., 0.,), (10., 10.,));\r\n/// let insets = Insets::uniform_xy(3., 0.,);\r\n///\r\n/// let inset_rect = rect + insets;\r\n/// assert_eq!(inset_rect.width(), 16.0, \"10.0 + 3.0 × 2\");\r\n/// assert_eq!(inset_rect.x0, -3.0);\r\n/// ```\r\n///\r\n/// Negative insets added to a [`Rect`] produce a smaller [`Rect`]:\r\n///\r\n/// ```\r\n/// # use kurbo::{Insets, Rect};\r\n/// let rect = Rect::from_origin_size((0., 0.,), (10., 10.,));\r\n/// let insets = Insets::uniform_xy(-3., 0.,);\r\n///\r\n/// let inset_rect = rect + insets;\r\n/// assert_eq!(inset_rect.width(), 4.0, \"10.0 - 3.0 × 2\");\r\n/// assert_eq!(inset_rect.x0, 3.0);\r\n/// ```\r\n///\r\n/// [`Insets`] operate on the absolute rectangle [`Rect::abs`], and so ignore\r\n/// existing negative widths and heights.\r\n///\r\n/// ```\r\n/// # use kurbo::{Insets, Rect};\r\n/// let rect = Rect::new(7., 11., 0., 0.,);\r\n/// let insets = Insets::uniform_xy(0., 1.,);\r\n///\r\n/// assert_eq!(rect.width(), -7.0);\r\n///\r\n/// let inset_rect = rect + insets;\r\n/// assert_eq!(inset_rect.width(), 7.0);\r\n/// assert_eq!(inset_rect.x0, 0.0);\r\n/// assert_eq!(inset_rect.height(), 13.0);\r\n/// ```\r\n///\r\n/// The width and height of an inset operation can still be negative if the\r\n/// [`Insets`]' dimensions are greater than the dimensions of the original [`Rect`].\r\n///\r\n/// ```\r\n/// # use kurbo::{Insets, Rect};\r\n/// let rect = Rect::new(0., 0., 3., 5.);\r\n/// let insets = Insets::uniform_xy(0., 7.,);\r\n///\r\n/// let inset_rect = rect - insets;\r\n/// assert_eq!(inset_rect.height(), -9., \"5 - 7 × 2\")\r\n/// ```\r\n///\r\n/// `Rect - Rect = Insets`:\r\n///\r\n///\r\n/// ```\r\n/// # use kurbo::{Insets, Rect};\r\n/// let rect = Rect::new(0., 0., 5., 11.);\r\n/// let insets = Insets::uniform_xy(1., 7.,);\r\n///\r\n/// let inset_rect = rect + insets;\r\n/// let insets2 = inset_rect - rect;\r\n///\r\n/// assert_eq!(insets2.x0, insets.x0);\r\n/// assert_eq!(insets2.y1, insets.y1);\r\n/// assert_eq!(insets2.x_value(), insets.x_value());\r\n/// assert_eq!(insets2.y_value(), insets.y_value());\r\n/// ```\r\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct Insets {\r\n /// The minimum x coordinate (left edge).\r\n pub x0: f64,\r\n /// The minimum y coordinate (top edge in y-down spaces).\r\n pub y0: f64,\r\n /// The maximum x coordinate (right edge).\r\n pub x1: f64,\r\n /// The maximum y coordinate (bottom edge in y-down spaces).\r\n pub y1: f64,\r\n}\r\n\r\nimpl Insets {\r\n /// Zeroed insets.\r\n pub const ZERO: Insets = Insets::uniform(0.);\r\n\r\n /// New uniform insets.\r\n #[inline]\r\n pub const fn uniform(d: f64) -> Insets {\r\n Insets {\r\n x0: d,\r\n y0: d,\r\n x1: d,\r\n y1: d,\r\n }\r\n }\r\n\r\n /// New insets with uniform values along each axis.\r\n #[inline]\r\n pub const fn uniform_xy(x: f64, y: f64) -> Insets {\r\n Insets {\r\n x0: x,\r\n y0: y,\r\n x1: x,\r\n y1: y,\r\n }\r\n }\r\n\r\n /// New insets. The ordering of the arguments is \"left, top, right, bottom\",\r\n /// assuming a y-down coordinate space.\r\n #[inline]\r\n pub const fn new(x0: f64, y0: f64, x1: f64, y1: f64) -> Insets {\r\n Insets { x0, y0, x1, y1 }\r\n }\r\n\r\n /// The total delta on the x-axis represented by these insets.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Insets;\r\n ///\r\n /// let insets = Insets::uniform_xy(3., 8.);\r\n /// assert_eq!(insets.x_value(), 6.);\r\n ///\r\n /// let insets = Insets::new(5., 0., -12., 0.,);\r\n /// assert_eq!(insets.x_value(), -7.);\r\n /// ```\r\n #[inline]\r\n pub fn x_value(self) -> f64 {\r\n self.x0 + self.x1\r\n }\r\n\r\n /// The total delta on the y-axis represented by these insets.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Insets;\r\n ///\r\n /// let insets = Insets::uniform_xy(3., 7.);\r\n /// assert_eq!(insets.y_value(), 14.);\r\n ///\r\n /// let insets = Insets::new(5., 10., -12., 4.,);\r\n /// assert_eq!(insets.y_value(), 14.);\r\n /// ```\r\n #[inline]\r\n pub fn y_value(self) -> f64 {\r\n self.y0 + self.y1\r\n }\r\n\r\n /// Returns the total delta represented by these insets as a [`Size`].\r\n ///\r\n /// This is equivalent to creating a [`Size`] from the values returned by\r\n /// [`x_value`] and [`y_value`].\r\n ///\r\n /// This function may return a size with negative values.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::{Insets, Size};\r\n ///\r\n /// let insets = Insets::new(11.1, -43.3, 3.333, -0.0);\r\n /// assert_eq!(insets.size(), Size::new(insets.x_value(), insets.y_value()));\r\n /// ```\r\n ///\r\n /// [`x_value`]: Insets::x_value\r\n /// [`y_value`]: Insets::y_value\r\n pub fn size(self) -> Size {\r\n Size::new(self.x_value(), self.y_value())\r\n }\r\n\r\n /// Return `true` iff all values are nonnegative.\r\n pub fn are_nonnegative(self) -> bool {\r\n let Insets { x0, y0, x1, y1 } = self;\r\n x0 >= 0.0 && y0 >= 0.0 && x1 >= 0.0 && y1 >= 0.0\r\n }\r\n\r\n /// Return new `Insets` with all negative values replaced with `0.0`.\r\n ///\r\n /// This is provided as a convenience for applications where negative insets\r\n /// are not meaningful.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Insets;\r\n ///\r\n /// let insets = Insets::new(-10., 3., -0.2, 4.);\r\n /// let nonnegative = insets.nonnegative();\r\n /// assert_eq!(nonnegative.x_value(), 0.0);\r\n /// assert_eq!(nonnegative.y_value(), 7.0);\r\n /// ```\r\n pub fn nonnegative(self) -> Insets {\r\n let Insets { x0, y0, x1, y1 } = self;\r\n Insets {\r\n x0: x0.max(0.0),\r\n y0: y0.max(0.0),\r\n x1: x1.max(0.0),\r\n y1: y1.max(0.0),\r\n }\r\n }\r\n\r\n /// Are these insets finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n self.x0.is_finite() && self.y0.is_finite() && self.x1.is_finite() && self.y1.is_finite()\r\n }\r\n\r\n /// Are these insets NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n self.x0.is_nan() || self.y0.is_nan() || self.x1.is_nan() || self.y1.is_nan()\r\n }\r\n}\r\n\r\nimpl Neg for Insets {\r\n type Output = Insets;\r\n\r\n #[inline]\r\n fn neg(self) -> Insets {\r\n Insets::new(-self.x0, -self.y0, -self.x1, -self.y1)\r\n }\r\n}\r\n\r\nimpl Add for Insets {\r\n type Output = Rect;\r\n\r\n #[inline]\r\n #[allow(clippy::suspicious_arithmetic_impl)]\r\n fn add(self, other: Rect) -> Rect {\r\n let other = other.abs();\r\n Rect::new(\r\n other.x0 - self.x0,\r\n other.y0 - self.y0,\r\n other.x1 + self.x1,\r\n other.y1 + self.y1,\r\n )\r\n }\r\n}\r\n\r\nimpl Add for Rect {\r\n type Output = Rect;\r\n\r\n #[inline]\r\n fn add(self, other: Insets) -> Rect {\r\n other + self\r\n }\r\n}\r\n\r\nimpl Sub for Insets {\r\n type Output = Rect;\r\n\r\n #[inline]\r\n fn sub(self, other: Rect) -> Rect {\r\n other + -self\r\n }\r\n}\r\n\r\nimpl Sub for Rect {\r\n type Output = Rect;\r\n\r\n #[inline]\r\n fn sub(self, other: Insets) -> Rect {\r\n other - self\r\n }\r\n}\r\n\r\nimpl From for Insets {\r\n fn from(src: f64) -> Insets {\r\n Insets::uniform(src)\r\n }\r\n}\r\n\r\nimpl From<(f64, f64)> for Insets {\r\n fn from(src: (f64, f64)) -> Insets {\r\n Insets::uniform_xy(src.0, src.1)\r\n }\r\n}\r\n\r\nimpl From<(f64, f64, f64, f64)> for Insets {\r\n fn from(src: (f64, f64, f64, f64)) -> Insets {\r\n Insets::new(src.0, src.1, src.2, src.3)\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/lib.rs", "sha256": "24bb8d407efa405b1b4d4efae668cf39f05a2d2777a981bc2201213d924f887c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! 2D geometry, with a focus on curves.\r\n//!\r\n//! The kurbo library contains data structures and algorithms for curves and\r\n//! vector paths. It was designed to serve the needs of 2D graphics applications,\r\n//! but it is intended to be general enough to be useful for other applications.\r\n//!\r\n//! Kurbo is designed to be used by [`Piet`], a crate for drawing 2D graphics,\r\n//! and is in turn used by [`Druid`], a cross-platform GUI toolkit.\r\n//!\r\n//! # Examples\r\n//!\r\n//! Basic UI-style geometry:\r\n//! ```\r\n//! use kurbo::{Insets, Point, Rect, Size, Vec2};\r\n//!\r\n//! let pt = Point::new(10.0, 10.0);\r\n//! let vector = Vec2::new(5.0, -5.0);\r\n//! let pt2 = pt + vector;\r\n//! assert_eq!(pt2, Point::new(15.0, 5.0));\r\n//!\r\n//! let rect = Rect::from_points(pt, pt2);\r\n//! assert_eq!(rect, Rect::from_origin_size((10.0, 5.0), (5.0, 5.0)));\r\n//!\r\n//! let insets = Insets::uniform(1.0);\r\n//! let inset_rect = rect - insets;\r\n//! assert_eq!(inset_rect.size(), Size::new(3.0, 3.0));\r\n//! ```\r\n//!\r\n//! Finding the closest position on a [`Shape`]'s perimeter to a [`Point`]:\r\n//!\r\n//! ```\r\n//! use kurbo::{Circle, ParamCurve, ParamCurveNearest, Point, Shape};\r\n//!\r\n//! const DESIRED_ACCURACY: f64 = 0.1;\r\n//!\r\n//! /// Given a shape and a point, returns the closest position on the shape's\r\n//! /// perimeter, or `None` if the shape is malformed.\r\n//! fn closest_perimeter_point(shape: impl Shape, pt: Point) -> Option {\r\n//! let mut best: Option<(Point, f64)> = None;\r\n//! for segment in shape.path_segments(DESIRED_ACCURACY) {\r\n//! let nearest = segment.nearest(pt, DESIRED_ACCURACY);\r\n//! if best.map(|(_, best_d)| nearest.distance_sq < best_d).unwrap_or(true) {\r\n//! best = Some((segment.eval(nearest.t), nearest.distance_sq))\r\n//! }\r\n//! }\r\n//! best.map(|(point, _)| point)\r\n//! }\r\n//!\r\n//! let circle = Circle::new((5.0, 5.0), 5.0);\r\n//! let hit_point = Point::new(5.0, -2.0);\r\n//! let expectation = Point::new(5.0, 0.0);\r\n//! let hit = closest_perimeter_point(circle, hit_point).unwrap();\r\n//! assert!(hit.distance(expectation) <= DESIRED_ACCURACY);\r\n//! ```\r\n//!\r\n//! # Features\r\n//!\r\n//! This crate either uses the standard library or the [`libm`] crate for\r\n//! math functionality. The `std` feature is enabled by default, but can be\r\n//! disabled, as long as the `libm` feature is enabled. This is useful for\r\n//! `no_std` environments. However, note that the `libm` crate is not as\r\n//! efficient as the standard library, and that this crate still uses the\r\n//! `alloc` crate regardless.\r\n//!\r\n//! [`Piet`]: https://docs.rs/piet\r\n//! [`Druid`]: https://docs.rs/druid\r\n//! [`libm`]: https://docs.rs/libm\r\n\r\n#![forbid(unsafe_code)]\r\n#![deny(missing_docs, clippy::trivially_copy_pass_by_ref)]\r\n#![warn(rustdoc::broken_intra_doc_links)]\r\n#![allow(\r\n clippy::unreadable_literal,\r\n clippy::many_single_char_names,\r\n clippy::excessive_precision,\r\n clippy::bool_to_int_with_if\r\n)]\r\n#![cfg_attr(all(not(feature = \"std\"), not(test)), no_std)]\r\n\r\n#[cfg(not(any(feature = \"std\", feature = \"libm\")))]\r\ncompile_error!(\"kurbo requires either the `std` or `libm` feature\");\r\n\r\nextern crate alloc;\r\n\r\nmod affine;\r\nmod arc;\r\nmod bezpath;\r\nmod circle;\r\npub mod common;\r\nmod cubicbez;\r\nmod ellipse;\r\nmod fit;\r\nmod insets;\r\nmod line;\r\nmod mindist;\r\npub mod offset;\r\nmod param_curve;\r\nmod point;\r\nmod quadbez;\r\nmod quadspline;\r\nmod rect;\r\nmod rounded_rect;\r\nmod rounded_rect_radii;\r\nmod shape;\r\npub mod simplify;\r\nmod size;\r\n#[cfg(feature = \"std\")]\r\nmod svg;\r\nmod translate_scale;\r\nmod vec2;\r\n\r\npub use crate::affine::*;\r\npub use crate::arc::*;\r\npub use crate::bezpath::*;\r\npub use crate::circle::*;\r\npub use crate::cubicbez::*;\r\npub use crate::ellipse::*;\r\npub use crate::fit::*;\r\npub use crate::insets::*;\r\npub use crate::line::*;\r\npub use crate::param_curve::*;\r\npub use crate::point::*;\r\npub use crate::quadbez::*;\r\npub use crate::quadspline::*;\r\npub use crate::rect::*;\r\npub use crate::rounded_rect::*;\r\npub use crate::rounded_rect_radii::*;\r\npub use crate::shape::*;\r\npub use crate::size::*;\r\n#[cfg(feature = \"std\")]\r\npub use crate::svg::*;\r\npub use crate::translate_scale::*;\r\npub use crate::vec2::*;\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/line.rs", "sha256": "f940db57e9fe5407433b3ea6c9dea6d001ca6874d08002a93945e60d3cd47d2c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Lines.\r\n\r\nuse core::ops::{Add, Mul, Range, Sub};\r\n\r\nuse arrayvec::ArrayVec;\r\n\r\nuse crate::{\r\n Affine, Nearest, ParamCurve, ParamCurveArclen, ParamCurveArea, ParamCurveCurvature,\r\n ParamCurveDeriv, ParamCurveExtrema, ParamCurveNearest, PathEl, Point, Rect, Shape, Vec2,\r\n DEFAULT_ACCURACY, MAX_EXTREMA,\r\n};\r\n\r\n/// A single line.\r\n#[derive(Clone, Copy, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct Line {\r\n /// The line's start point.\r\n pub p0: Point,\r\n /// The line's end point.\r\n pub p1: Point,\r\n}\r\n\r\nimpl Line {\r\n /// Create a new line.\r\n #[inline]\r\n pub fn new(p0: impl Into, p1: impl Into) -> Line {\r\n Line {\r\n p0: p0.into(),\r\n p1: p1.into(),\r\n }\r\n }\r\n\r\n /// The length of the line.\r\n #[inline]\r\n pub fn length(self) -> f64 {\r\n self.arclen(DEFAULT_ACCURACY)\r\n }\r\n\r\n /// Computes the point where two lines, if extended to infinity, would cross.\r\n pub fn crossing_point(self, other: Line) -> Option {\r\n let ab = self.p1 - self.p0;\r\n let cd = other.p1 - other.p0;\r\n let pcd = ab.cross(cd);\r\n if pcd == 0.0 {\r\n return None;\r\n }\r\n let h = ab.cross(self.p0 - other.p0) / pcd;\r\n Some(other.p0 + cd * h)\r\n }\r\n\r\n /// Is this line finite?\r\n #[inline]\r\n pub fn is_finite(self) -> bool {\r\n self.p0.is_finite() && self.p1.is_finite()\r\n }\r\n\r\n /// Is this line NaN?\r\n #[inline]\r\n pub fn is_nan(self) -> bool {\r\n self.p0.is_nan() || self.p1.is_nan()\r\n }\r\n}\r\n\r\nimpl ParamCurve for Line {\r\n #[inline]\r\n fn eval(&self, t: f64) -> Point {\r\n self.p0.lerp(self.p1, t)\r\n }\r\n\r\n #[inline]\r\n fn subsegment(&self, range: Range) -> Line {\r\n Line {\r\n p0: self.eval(range.start),\r\n p1: self.eval(range.end),\r\n }\r\n }\r\n\r\n #[inline]\r\n fn start(&self) -> Point {\r\n self.p0\r\n }\r\n\r\n #[inline]\r\n fn end(&self) -> Point {\r\n self.p1\r\n }\r\n}\r\n\r\nimpl ParamCurveDeriv for Line {\r\n type DerivResult = ConstPoint;\r\n\r\n #[inline]\r\n fn deriv(&self) -> ConstPoint {\r\n ConstPoint((self.p1 - self.p0).to_point())\r\n }\r\n}\r\n\r\nimpl ParamCurveArclen for Line {\r\n #[inline]\r\n fn arclen(&self, _accuracy: f64) -> f64 {\r\n (self.p1 - self.p0).hypot()\r\n }\r\n\r\n #[inline]\r\n fn inv_arclen(&self, arclen: f64, _accuracy: f64) -> f64 {\r\n arclen / (self.p1 - self.p0).hypot()\r\n }\r\n}\r\n\r\nimpl ParamCurveArea for Line {\r\n #[inline]\r\n fn signed_area(&self) -> f64 {\r\n self.p0.to_vec2().cross(self.p1.to_vec2()) * 0.5\r\n }\r\n}\r\n\r\nimpl ParamCurveNearest for Line {\r\n fn nearest(&self, p: Point, _accuracy: f64) -> Nearest {\r\n let d = self.p1 - self.p0;\r\n let dotp = d.dot(p - self.p0);\r\n let d_squared = d.dot(d);\r\n let (t, distance_sq) = if dotp <= 0.0 {\r\n (0.0, (p - self.p0).hypot2())\r\n } else if dotp >= d_squared {\r\n (1.0, (p - self.p1).hypot2())\r\n } else {\r\n let t = dotp / d_squared;\r\n let dist = (p - self.eval(t)).hypot2();\r\n (t, dist)\r\n };\r\n Nearest { distance_sq, t }\r\n }\r\n}\r\n\r\nimpl ParamCurveCurvature for Line {\r\n #[inline]\r\n fn curvature(&self, _t: f64) -> f64 {\r\n 0.0\r\n }\r\n}\r\n\r\nimpl ParamCurveExtrema for Line {\r\n #[inline]\r\n fn extrema(&self) -> ArrayVec {\r\n ArrayVec::new()\r\n }\r\n}\r\n\r\n/// A trivial \"curve\" that is just a constant.\r\n#[derive(Clone, Copy, Debug)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct ConstPoint(Point);\r\n\r\nimpl ConstPoint {\r\n /// Is this point finite?\r\n #[inline]\r\n pub fn is_finite(self) -> bool {\r\n self.0.is_finite()\r\n }\r\n\r\n /// Is this point NaN?\r\n #[inline]\r\n pub fn is_nan(self) -> bool {\r\n self.0.is_nan()\r\n }\r\n}\r\n\r\nimpl ParamCurve for ConstPoint {\r\n #[inline]\r\n fn eval(&self, _t: f64) -> Point {\r\n self.0\r\n }\r\n\r\n #[inline]\r\n fn subsegment(&self, _range: Range) -> ConstPoint {\r\n *self\r\n }\r\n}\r\n\r\nimpl ParamCurveDeriv for ConstPoint {\r\n type DerivResult = ConstPoint;\r\n\r\n #[inline]\r\n fn deriv(&self) -> ConstPoint {\r\n ConstPoint(Point::new(0.0, 0.0))\r\n }\r\n}\r\n\r\nimpl ParamCurveArclen for ConstPoint {\r\n #[inline]\r\n fn arclen(&self, _accuracy: f64) -> f64 {\r\n 0.0\r\n }\r\n\r\n #[inline]\r\n fn inv_arclen(&self, _arclen: f64, _accuracy: f64) -> f64 {\r\n 0.0\r\n }\r\n}\r\n\r\nimpl Mul for Affine {\r\n type Output = Line;\r\n\r\n #[inline]\r\n fn mul(self, other: Line) -> Line {\r\n Line {\r\n p0: self * other.p0,\r\n p1: self * other.p1,\r\n }\r\n }\r\n}\r\n\r\nimpl Add for Line {\r\n type Output = Line;\r\n\r\n #[inline]\r\n fn add(self, v: Vec2) -> Line {\r\n Line::new(self.p0 + v, self.p1 + v)\r\n }\r\n}\r\n\r\nimpl Sub for Line {\r\n type Output = Line;\r\n\r\n #[inline]\r\n fn sub(self, v: Vec2) -> Line {\r\n Line::new(self.p0 - v, self.p1 - v)\r\n }\r\n}\r\n\r\n/// An iterator yielding the path for a single line.\r\n#[doc(hidden)]\r\npub struct LinePathIter {\r\n line: Line,\r\n ix: usize,\r\n}\r\n\r\nimpl Shape for Line {\r\n type PathElementsIter<'iter> = LinePathIter;\r\n\r\n #[inline]\r\n fn path_elements(&self, _tolerance: f64) -> LinePathIter {\r\n LinePathIter { line: *self, ix: 0 }\r\n }\r\n\r\n /// Returning zero here is consistent with the contract (area is\r\n /// only meaningful for closed shapes), but an argument can be made\r\n /// that the contract should be tightened to include the Green's\r\n /// theorem contribution.\r\n fn area(&self) -> f64 {\r\n 0.0\r\n }\r\n\r\n #[inline]\r\n fn perimeter(&self, _accuracy: f64) -> f64 {\r\n (self.p1 - self.p0).hypot()\r\n }\r\n\r\n /// Same consideration as `area`.\r\n fn winding(&self, _pt: Point) -> i32 {\r\n 0\r\n }\r\n\r\n #[inline]\r\n fn bounding_box(&self) -> Rect {\r\n Rect::from_points(self.p0, self.p1)\r\n }\r\n\r\n #[inline]\r\n fn as_line(&self) -> Option {\r\n Some(*self)\r\n }\r\n}\r\n\r\nimpl Iterator for LinePathIter {\r\n type Item = PathEl;\r\n\r\n fn next(&mut self) -> Option {\r\n self.ix += 1;\r\n match self.ix {\r\n 1 => Some(PathEl::MoveTo(self.line.p0)),\r\n 2 => Some(PathEl::LineTo(self.line.p1)),\r\n _ => None,\r\n }\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{Line, ParamCurveArclen, Point};\r\n\r\n #[test]\r\n fn line_arclen() {\r\n let l = Line::new((0.0, 0.0), (1.0, 1.0));\r\n let true_len = 2.0f64.sqrt();\r\n let epsilon = 1e-9;\r\n assert!(l.arclen(epsilon) - true_len < epsilon);\r\n\r\n let t = l.inv_arclen(true_len / 3.0, epsilon);\r\n assert!((t - 1.0 / 3.0).abs() < epsilon);\r\n }\r\n\r\n #[test]\r\n fn line_is_finite() {\r\n assert!((Line {\r\n p0: Point { x: 0., y: 0. },\r\n p1: Point { x: 1., y: 1. }\r\n })\r\n .is_finite());\r\n\r\n assert!(!(Line {\r\n p0: Point { x: 0., y: 0. },\r\n p1: Point {\r\n x: f64::INFINITY,\r\n y: 1.\r\n }\r\n })\r\n .is_finite());\r\n\r\n assert!(!(Line {\r\n p0: Point { x: 0., y: 0. },\r\n p1: Point {\r\n x: 0.,\r\n y: f64::INFINITY\r\n }\r\n })\r\n .is_finite());\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/mindist.rs", "sha256": "8ba69c3e551c6e7ab5383dcfdb1055a680205078b818c18457083ecda801fca2", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2021 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Minimum distance between two Bézier curves\r\n//!\r\n//! This implements the algorithm in \"Computing the minimum distance between\r\n//! two Bézier curves\", Chen et al., *Journal of Computational and Applied\r\n//! Mathematics* 229(2009), 294-301\r\n\r\nuse crate::Vec2;\r\nuse core::cmp::Ordering;\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\npub(crate) fn min_dist_param(\r\n bez1: &[Vec2],\r\n bez2: &[Vec2],\r\n u: (f64, f64),\r\n v: (f64, f64),\r\n epsilon: f64,\r\n best_alpha: Option,\r\n) -> (f64, f64, f64) {\r\n assert!(!bez1.is_empty() && !bez2.is_empty());\r\n let n = bez1.len() - 1;\r\n let m = bez2.len() - 1;\r\n let (umin, umax) = u;\r\n let (vmin, vmax) = v;\r\n let umid = (umin + umax) / 2.0;\r\n let vmid = (vmin + vmax) / 2.0;\r\n let svalues: [(f64, f64, f64); 4] = [\r\n (S(umin, vmin, bez1, bez2), umin, vmin),\r\n (S(umin, vmax, bez1, bez2), umin, vmax),\r\n (S(umax, vmin, bez1, bez2), umax, vmin),\r\n (S(umax, vmax, bez1, bez2), umax, vmax),\r\n ];\r\n let alpha: f64 = svalues.iter().map(|(a, _, _)| *a).reduce(f64::min).unwrap();\r\n if let Some(best) = best_alpha {\r\n if alpha > best {\r\n return (alpha, umid, vmid);\r\n }\r\n }\r\n\r\n if (umax - umin).abs() < epsilon || (vmax - vmin).abs() < epsilon {\r\n return (alpha, umid, vmid);\r\n }\r\n\r\n // Property one: D(r>k) > alpha\r\n let mut is_outside = true;\r\n let mut min_drk = None;\r\n let mut min_ij = None;\r\n for r in 0..(2 * n) {\r\n for k in 0..(2 * m) {\r\n let d_rk = D_rk(r, k, bez1, bez2);\r\n if d_rk < alpha {\r\n is_outside = false;\r\n }\r\n if min_drk.is_none() || Some(d_rk) < min_drk {\r\n min_drk = Some(d_rk);\r\n min_ij = Some((r, k));\r\n }\r\n }\r\n }\r\n if is_outside {\r\n return (alpha, umid, vmid);\r\n }\r\n\r\n // Property two: boundary check\r\n let mut at_boundary0on_bez1 = true;\r\n let mut at_boundary1on_bez1 = true;\r\n let mut at_boundary0on_bez2 = true;\r\n let mut at_boundary1on_bez2 = true;\r\n for i in 0..2 * n {\r\n for j in 0..2 * m {\r\n let dij = D_rk(i, j, bez1, bez2);\r\n let dkj = D_rk(0, j, bez1, bez2);\r\n if dij < dkj {\r\n at_boundary0on_bez1 = false\r\n }\r\n let dkj = D_rk(2 * n, j, bez1, bez2);\r\n if dij < dkj {\r\n at_boundary1on_bez1 = false\r\n }\r\n let dkj = D_rk(i, 0, bez1, bez2);\r\n if dij < dkj {\r\n at_boundary0on_bez2 = false\r\n }\r\n let dkj = D_rk(i, 2 * n, bez1, bez2);\r\n if dij < dkj {\r\n at_boundary1on_bez2 = false\r\n }\r\n }\r\n }\r\n if at_boundary0on_bez1 && at_boundary0on_bez2 {\r\n return svalues[0];\r\n }\r\n if at_boundary0on_bez1 && at_boundary1on_bez2 {\r\n return svalues[1];\r\n }\r\n if at_boundary1on_bez1 && at_boundary0on_bez2 {\r\n return svalues[2];\r\n }\r\n if at_boundary1on_bez1 && at_boundary1on_bez2 {\r\n return svalues[3];\r\n }\r\n\r\n let (min_i, min_j) = min_ij.unwrap();\r\n let new_umid = umin + (umax - umin) * (min_i as f64 / (2 * n) as f64);\r\n let new_vmid = vmin + (vmax - vmin) * (min_j as f64 / (2 * m) as f64);\r\n\r\n // Subdivide\r\n let results = [\r\n min_dist_param(\r\n bez1,\r\n bez2,\r\n (umin, new_umid),\r\n (vmin, new_vmid),\r\n epsilon,\r\n Some(alpha),\r\n ),\r\n min_dist_param(\r\n bez1,\r\n bez2,\r\n (umin, new_umid),\r\n (new_vmid, vmax),\r\n epsilon,\r\n Some(alpha),\r\n ),\r\n min_dist_param(\r\n bez1,\r\n bez2,\r\n (new_umid, umax),\r\n (vmin, new_vmid),\r\n epsilon,\r\n Some(alpha),\r\n ),\r\n min_dist_param(\r\n bez1,\r\n bez2,\r\n (new_umid, umax),\r\n (new_vmid, vmax),\r\n epsilon,\r\n Some(alpha),\r\n ),\r\n ];\r\n\r\n *results\r\n .iter()\r\n .min_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(Ordering::Less))\r\n .unwrap()\r\n}\r\n\r\n// $ S(u,v)=\\sum_{r=0}^{2n} \\sum _{k=0}^{2m} D_{r,k} B_{2n}^r(u) B_{2m}^k(v) $\r\n\r\n#[allow(non_snake_case)]\r\nfn S(u: f64, v: f64, bez1: &[Vec2], bez2: &[Vec2]) -> f64 {\r\n let n = bez1.len() - 1;\r\n let m = bez2.len() - 1;\r\n let mut summand = 0.0;\r\n for r in 0..=2 * n {\r\n for k in 0..=2 * m {\r\n summand +=\r\n D_rk(r, k, bez1, bez2) * basis_function(2 * n, r, u) * basis_function(2 * m, k, v)\r\n }\r\n }\r\n summand\r\n}\r\n\r\n// $ C_{r,k} = ( \\sum_{i=\\theta}^\\upsilon P_i C_n^i C_n^{r-i} / C_{2n}^r ) \\cdot (\\sum_{j=\\sigma}^\\varsigma Q_j C_m^j C_m^{k-j} / C_{2m}^k ) $\r\n#[allow(non_snake_case)]\r\nfn C_rk(r: usize, k: usize, bez1: &[Vec2], bez2: &[Vec2]) -> f64 {\r\n let n = bez1.len() - 1;\r\n let upsilon = r.min(n);\r\n let theta = r - n.min(r);\r\n let mut left: Vec2 = Vec2::ZERO;\r\n for (i, &item) in bez1.iter().enumerate().take(upsilon + 1).skip(theta) {\r\n left += item * (choose(n, i) * choose(n, r - i)) as f64 / (choose(2 * n, r) as f64);\r\n }\r\n\r\n let m = bez2.len() - 1;\r\n let varsigma = k.min(m);\r\n let sigma = k - m.min(k);\r\n let mut right: Vec2 = Vec2::ZERO;\r\n for (j, &item) in bez2.iter().enumerate().take(varsigma + 1).skip(sigma) {\r\n right += item * (choose(m, j) * choose(m, k - j)) as f64 / (choose(2 * m, k) as f64);\r\n }\r\n left.dot(right)\r\n}\r\n\r\n// $ A_r=\\sum _{i=\\theta} ^\\upsilon (P_i \\cdot P_{r-i}) C_n^i C_n^{r-i} / C_{2n}^r $\r\n// ($ B_k $ is just the same as $ A_r $ but for the other curve.)\r\n\r\n#[allow(non_snake_case)]\r\nfn A_r(r: usize, p: &[Vec2]) -> f64 {\r\n let n = p.len() - 1;\r\n let upsilon = r.min(n);\r\n let theta = r - n.min(r);\r\n (theta..=upsilon)\r\n .map(|i| {\r\n let dot = p[i].dot(p[r - i]); // These are bounds checked by the sum limits\r\n let factor = (choose(n, i) * choose(n, r - i)) as f64 / (choose(2 * n, r) as f64);\r\n dot * factor\r\n })\r\n .sum()\r\n}\r\n\r\n#[allow(non_snake_case)]\r\nfn D_rk(r: usize, k: usize, bez1: &[Vec2], bez2: &[Vec2]) -> f64 {\r\n // In the paper, B_k is used for the second factor, but it's the same thing\r\n A_r(r, bez1) + A_r(k, bez2) - 2.0 * C_rk(r, k, bez1, bez2)\r\n}\r\n\r\n// Bezier basis function\r\nfn basis_function(n: usize, i: usize, u: f64) -> f64 {\r\n choose(n, i) as f64 * (1.0 - u).powi((n - i) as i32) * u.powi(i as i32)\r\n}\r\n\r\n// Binomial co-efficient, but returning zeros for values outside of domain\r\nfn choose(n: usize, k: usize) -> u32 {\r\n let mut n = n;\r\n if k > n {\r\n return 0;\r\n }\r\n let mut p = 1;\r\n for i in 1..=(n - k) {\r\n p *= n;\r\n p /= i;\r\n n -= 1;\r\n }\r\n p as u32\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::mindist::A_r;\r\n use crate::mindist::{choose, D_rk};\r\n use crate::{CubicBez, Line, PathSeg, Vec2};\r\n\r\n #[test]\r\n fn test_choose() {\r\n assert_eq!(choose(6, 0), 1);\r\n assert_eq!(choose(6, 1), 6);\r\n assert_eq!(choose(6, 2), 15);\r\n }\r\n\r\n #[test]\r\n fn test_d_rk() {\r\n let bez1 = vec![\r\n Vec2::new(129.0, 139.0),\r\n Vec2::new(190.0, 139.0),\r\n Vec2::new(201.0, 364.0),\r\n Vec2::new(90.0, 364.0),\r\n ];\r\n let bez2 = vec![\r\n Vec2::new(309.0, 159.0),\r\n Vec2::new(178.0, 159.0),\r\n Vec2::new(215.0, 408.0),\r\n Vec2::new(309.0, 408.0),\r\n ];\r\n let b = A_r(1, &bez2);\r\n assert!((b - 80283.0).abs() < 0.005, \"B_1(Q)={b:?}\");\r\n let d = D_rk(0, 1, &bez1, &bez2);\r\n assert!((d - 9220.0).abs() < 0.005, \"D={d:?}\");\r\n }\r\n\r\n #[test]\r\n fn test_mindist() {\r\n let bez1 = PathSeg::Cubic(CubicBez::new(\r\n (129.0, 139.0),\r\n (190.0, 139.0),\r\n (201.0, 364.0),\r\n (90.0, 364.0),\r\n ));\r\n let bez2 = PathSeg::Cubic(CubicBez::new(\r\n (309.0, 159.0),\r\n (178.0, 159.0),\r\n (215.0, 408.0),\r\n (309.0, 408.0),\r\n ));\r\n let mindist = bez1.min_dist(bez2, 0.001);\r\n assert!((mindist.distance - 50.9966).abs() < 0.5);\r\n }\r\n\r\n #[test]\r\n fn test_overflow() {\r\n let bez1 = PathSeg::Cubic(CubicBez::new(\r\n (232.0, 126.0),\r\n (134.0, 126.0),\r\n (139.0, 232.0),\r\n (141.0, 301.0),\r\n ));\r\n let bez2 = PathSeg::Line(Line::new((359.0, 416.0), (367.0, 755.0)));\r\n let mindist = bez1.min_dist(bez2, 0.001);\r\n assert!((mindist.distance - 246.4731222669117).abs() < 0.5);\r\n }\r\n\r\n #[test]\r\n fn test_out_of_order() {\r\n let bez1 = PathSeg::Cubic(CubicBez::new(\r\n (287.0, 182.0),\r\n (346.0, 277.0),\r\n (356.0, 299.0),\r\n (359.0, 416.0),\r\n ));\r\n let bez2 = PathSeg::Line(Line::new((141.0, 301.0), (152.0, 709.0)));\r\n let mindist1 = bez1.min_dist(bez2, 0.5);\r\n let mindist2 = bez2.min_dist(bez1, 0.5);\r\n assert!((mindist1.distance - mindist2.distance).abs() < 0.5);\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/offset.rs", "sha256": "4b21d1112f959eb3815b0ed6120db5784c7ac4d070929eecdf82ee59d7ea957b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 The Kurbo Authors\r\n//\r\n// Licensed under the Apache License, Version 2.0 (the \"License\");\r\n// you may not use this file except in compliance with the License.\r\n// You may obtain a copy of the License at\r\n//\r\n// https://www.apache.org/licenses/LICENSE-2.0\r\n//\r\n// Unless required by applicable law or agreed to in writing, software\r\n// distributed under the License is distributed on an \"AS IS\" BASIS,\r\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n// See the License for the specific language governing permissions and\r\n// limitations under the License.\r\n\r\n//! Computation of offset curves of cubic Béziers, based on a curve fitting\r\n//! approach.\r\n//!\r\n//! See the [Parallel curves of cubic Béziers] blog post for a discussion of how\r\n//! this algorithm works and what kind of results can be expected. In general, it\r\n//! is expected to perform much better than most published algorithms. The number\r\n//! of curve segments needed to attain a given accuracy scales as O(n^6) with\r\n//! accuracy.\r\n//!\r\n//! In general, to compute the offset curve (also known as parallel curve) of\r\n//! a cubic Bézier segment, create a [`CubicOffset`] struct with the curve\r\n//! segment and offset, then use [`fit_to_bezpath`] or [`fit_to_bezpath_opt`]\r\n//! depending on how much time to spend optimizing the resulting path.\r\n//!\r\n//! [`fit_to_bezpath`]: crate::fit_to_bezpath\r\n//! [`fit_to_bezpath_opt`]: crate::fit_to_bezpath_opt\r\n//! [Parallel curves of cubic Béziers]: https://raphlinus.github.io/curves/2022/09/09/parallel-beziers.html\r\nuse core::ops::Range;\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\nuse crate::{\r\n common::solve_itp, CubicBez, CurveFitSample, ParamCurve, ParamCurveDeriv, ParamCurveFit, Point,\r\n QuadBez, Vec2,\r\n};\r\n\r\n/// The offset curve of a cubic Bézier.\r\n///\r\n/// This is a representation of the offset curve of a cubic Bézier segment, for\r\n/// purposes of curve fitting.\r\n///\r\n/// See the [module-level documentation] for a bit more discussion of the approach,\r\n/// and how this struct is to be used.\r\n///\r\n/// [module-level documentation]: crate::offset\r\npub struct CubicOffset {\r\n /// Source curve.\r\n c: CubicBez,\r\n /// Derivative of source curve.\r\n q: QuadBez,\r\n /// Offset.\r\n d: f64,\r\n // c0 + c1 t + c2 t^2 is the cross product of second and first\r\n // derivatives of the underlying cubic, multiplied by offset (for\r\n // computing cusp).\r\n c0: f64,\r\n c1: f64,\r\n c2: f64,\r\n}\r\n\r\nimpl CubicOffset {\r\n /// Create a new curve from Bézier segment and offset.\r\n pub fn new(c: CubicBez, d: f64) -> Self {\r\n let q = c.deriv();\r\n let d0 = q.p0.to_vec2();\r\n let d1 = 2.0 * (q.p1 - q.p0);\r\n let d2 = q.p0.to_vec2() - 2.0 * q.p1.to_vec2() + q.p2.to_vec2();\r\n CubicOffset {\r\n c,\r\n q,\r\n d,\r\n c0: d * d1.cross(d0),\r\n c1: d * 2.0 * d2.cross(d0),\r\n c2: d * d2.cross(d1),\r\n }\r\n }\r\n\r\n fn eval_offset(&self, t: f64) -> Vec2 {\r\n let dp = self.q.eval(t).to_vec2();\r\n let norm = Vec2::new(-dp.y, dp.x);\r\n // TODO: deal with hypot = 0\r\n norm * self.d / dp.hypot()\r\n }\r\n\r\n fn eval(&self, t: f64) -> Point {\r\n // Point on source curve.\r\n self.c.eval(t) + self.eval_offset(t)\r\n }\r\n\r\n /// Evaluate derivative of curve.\r\n fn eval_deriv(&self, t: f64) -> Vec2 {\r\n self.cusp_sign(t) * self.q.eval(t).to_vec2()\r\n }\r\n\r\n // Compute a function which has a zero-crossing at cusps, and is\r\n // positive at low curvatures on the source curve.\r\n fn cusp_sign(&self, t: f64) -> f64 {\r\n let ds2 = self.q.eval(t).to_vec2().hypot2();\r\n ((self.c2 * t + self.c1) * t + self.c0) / (ds2 * ds2.sqrt()) + 1.0\r\n }\r\n}\r\n\r\nimpl ParamCurveFit for CubicOffset {\r\n fn sample_pt_tangent(&self, t: f64, sign: f64) -> CurveFitSample {\r\n let p = self.eval(t);\r\n const CUSP_EPS: f64 = 1e-8;\r\n let mut cusp = self.cusp_sign(t);\r\n if cusp.abs() < CUSP_EPS {\r\n // This is a numerical derivative, which is probably good enough\r\n // for all practical purposes, but an analytical derivative would\r\n // be more elegant.\r\n //\r\n // Also, we're not dealing with second or higher order cusps.\r\n cusp = sign * (self.cusp_sign(t + CUSP_EPS) - self.cusp_sign(t - CUSP_EPS));\r\n }\r\n let tangent = self.q.eval(t).to_vec2() * cusp.signum();\r\n CurveFitSample { p, tangent }\r\n }\r\n\r\n fn sample_pt_deriv(&self, t: f64) -> (Point, Vec2) {\r\n (self.eval(t), self.eval_deriv(t))\r\n }\r\n\r\n fn break_cusp(&self, range: Range) -> Option {\r\n const CUSP_EPS: f64 = 1e-8;\r\n // When an endpoint is on (or very near) a cusp, move just far enough\r\n // away from the cusp that we're confident we have the right sign.\r\n let break_cusp_help = |mut x, mut d| {\r\n let mut cusp = self.cusp_sign(x);\r\n while cusp.abs() < CUSP_EPS && d < 1.0 {\r\n x += d;\r\n let old_cusp = cusp;\r\n cusp = self.cusp_sign(x);\r\n if cusp.abs() > old_cusp.abs() {\r\n break;\r\n }\r\n d *= 2.0;\r\n }\r\n (x, cusp)\r\n };\r\n let (a, cusp0) = break_cusp_help(range.start, 1e-12);\r\n let (b, cusp1) = break_cusp_help(range.end, -1e-12);\r\n if a >= b || cusp0 * cusp1 >= 0.0 {\r\n // Discussion point: maybe we should search for double cusps in the interior\r\n // of the range.\r\n return None;\r\n }\r\n let s = cusp1.signum();\r\n let f = |t| s * self.cusp_sign(t);\r\n let k1 = 0.2 / (b - a);\r\n const ITP_EPS: f64 = 1e-12;\r\n let x = solve_itp(f, a, b, ITP_EPS, 1, k1, s * cusp0, s * cusp1);\r\n Some(x)\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/param_curve.rs", "sha256": "26cb79035285a4e164b17ead95848f4388b47f3b772c7e1ea1c18364c29fdf54", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A trait for curves parametrized by a scalar.\r\n\r\nuse core::ops::Range;\r\n\r\nuse arrayvec::ArrayVec;\r\n\r\nuse crate::{common, Point, Rect};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// A default value for methods that take an 'accuracy' argument.\r\n///\r\n/// This value is intended to be suitable for general-purpose use, such as\r\n/// 2d graphics.\r\npub const DEFAULT_ACCURACY: f64 = 1e-6;\r\n\r\n/// A curve parametrized by a scalar.\r\n///\r\n/// If the result is interpreted as a point, this represents a curve.\r\n/// But the result can be interpreted as a vector as well.\r\npub trait ParamCurve: Sized {\r\n /// Evaluate the curve at parameter `t`.\r\n ///\r\n /// Generally `t` is in the range [0..1].\r\n fn eval(&self, t: f64) -> Point;\r\n\r\n /// Get a subsegment of the curve for the given parameter range.\r\n fn subsegment(&self, range: Range) -> Self;\r\n\r\n /// Subdivide into (roughly) halves.\r\n #[inline]\r\n fn subdivide(&self) -> (Self, Self) {\r\n (self.subsegment(0.0..0.5), self.subsegment(0.5..1.0))\r\n }\r\n\r\n /// The start point.\r\n fn start(&self) -> Point {\r\n self.eval(0.0)\r\n }\r\n\r\n /// The end point.\r\n fn end(&self) -> Point {\r\n self.eval(1.0)\r\n }\r\n}\r\n\r\n// TODO: I might not want to have separate traits for all these.\r\n\r\n/// A differentiable parametrized curve.\r\npub trait ParamCurveDeriv {\r\n /// The parametric curve obtained by taking the derivative of this one.\r\n type DerivResult: ParamCurve;\r\n\r\n /// The derivative of the curve.\r\n ///\r\n /// Note that the type of the return value is somewhat inaccurate, as\r\n /// the derivative of a curve (mapping of param to point) is a mapping\r\n /// of param to vector. We choose to accept this rather than have a\r\n /// more complex type scheme.\r\n fn deriv(&self) -> Self::DerivResult;\r\n\r\n /// Estimate arclength using Gaussian quadrature.\r\n ///\r\n /// The coefficients are assumed to cover the range (-1..1), which is\r\n /// traditional.\r\n #[inline]\r\n fn gauss_arclen(&self, coeffs: &[(f64, f64)]) -> f64 {\r\n let d = self.deriv();\r\n coeffs\r\n .iter()\r\n .map(|(wi, xi)| wi * d.eval(0.5 * (xi + 1.0)).to_vec2().hypot())\r\n .sum::()\r\n * 0.5\r\n }\r\n}\r\n\r\n/// A parametrized curve that can have its arc length measured.\r\npub trait ParamCurveArclen: ParamCurve {\r\n /// The arc length of the curve.\r\n ///\r\n /// The result is accurate to the given accuracy (subject to\r\n /// roundoff errors for ridiculously low values). Compute time\r\n /// may vary with accuracy, if the curve needs to be subdivided.\r\n fn arclen(&self, accuracy: f64) -> f64;\r\n\r\n /// Solve for the parameter that has the given arc length from the start.\r\n ///\r\n /// This implementation uses the IPT method, as provided by\r\n /// [`common::solve_itp`]. This is as robust as bisection but\r\n /// typically converges faster. In addition, the method takes\r\n /// care to compute arc lengths of increasingly smaller segments\r\n /// of the curve, as that is likely faster than repeatedly\r\n /// computing the arc length of the segment starting at t=0.\r\n fn inv_arclen(&self, arclen: f64, accuracy: f64) -> f64 {\r\n if arclen <= 0.0 {\r\n return 0.0;\r\n }\r\n let total_arclen = self.arclen(accuracy);\r\n if arclen >= total_arclen {\r\n return 1.0;\r\n }\r\n let mut t_last = 0.0;\r\n let mut arclen_last = 0.0;\r\n let epsilon = accuracy / total_arclen;\r\n let n = 1.0 - epsilon.log2().ceil().min(0.0);\r\n let inner_accuracy = accuracy / n;\r\n let f = |t: f64| {\r\n let (range, dir) = if t > t_last {\r\n (t_last..t, 1.0)\r\n } else {\r\n (t..t_last, -1.0)\r\n };\r\n let arc = self.subsegment(range).arclen(inner_accuracy);\r\n arclen_last += arc * dir;\r\n t_last = t;\r\n arclen_last - arclen\r\n };\r\n common::solve_itp(f, 0.0, 1.0, epsilon, 1, 0.2, -arclen, total_arclen - arclen)\r\n }\r\n}\r\n\r\n/// A parametrized curve that can have its signed area measured.\r\npub trait ParamCurveArea {\r\n /// Compute the signed area under the curve.\r\n ///\r\n /// For a closed path, the signed area of the path is the sum of signed\r\n /// areas of the segments. This is a variant of the \"shoelace formula.\"\r\n /// See:\r\n /// and\r\n /// \r\n ///\r\n /// This can be computed exactly for Béziers thanks to Green's theorem,\r\n /// and also for simple curves such as circular arcs. For more exotic\r\n /// curves, it's probably best to subdivide to cubics. We leave that\r\n /// to the caller, which is why we don't give an accuracy param here.\r\n fn signed_area(&self) -> f64;\r\n}\r\n\r\n/// The nearest position on a curve to some point.\r\n///\r\n/// This is returned by [`ParamCurveNearest::nearest`]\r\n#[derive(Debug, Clone, Copy)]\r\npub struct Nearest {\r\n /// The square of the distance from the nearest position on the curve\r\n /// to the given point.\r\n pub distance_sq: f64,\r\n /// The position on the curve of the nearest point, as a parameter.\r\n ///\r\n /// To resolve this to a [`Point`], use [`ParamCurve::eval`].\r\n pub t: f64,\r\n}\r\n\r\n/// A parametrized curve that reports the nearest point.\r\npub trait ParamCurveNearest {\r\n /// Find the position on the curve that is nearest to the given point.\r\n ///\r\n /// This returns a [`Nearest`] struct that contains information about\r\n /// the position.\r\n fn nearest(&self, p: Point, accuracy: f64) -> Nearest;\r\n}\r\n\r\n/// A parametrized curve that reports its curvature.\r\npub trait ParamCurveCurvature: ParamCurveDeriv\r\nwhere\r\n Self::DerivResult: ParamCurveDeriv,\r\n{\r\n /// Compute the signed curvature at parameter `t`.\r\n #[inline]\r\n fn curvature(&self, t: f64) -> f64 {\r\n let deriv = self.deriv();\r\n let deriv2 = deriv.deriv();\r\n let d = deriv.eval(t).to_vec2();\r\n let d2 = deriv2.eval(t).to_vec2();\r\n // TODO: What's the convention for sign? I think it should match signed\r\n // area - a positive area curve should have positive curvature.\r\n d2.cross(d) * d.hypot2().powf(-1.5)\r\n }\r\n}\r\n\r\n/// The maximum number of extrema that can be reported in the `ParamCurveExtrema` trait.\r\n///\r\n/// This is 4 to support cubic Béziers. If other curves are used, they should be\r\n/// subdivided to limit the number of extrema.\r\npub const MAX_EXTREMA: usize = 4;\r\n\r\n/// A parametrized curve that reports its extrema.\r\npub trait ParamCurveExtrema: ParamCurve {\r\n /// Compute the extrema of the curve.\r\n ///\r\n /// Only extrema within the interior of the curve count.\r\n /// At most four extrema can be reported, which is sufficient for\r\n /// cubic Béziers.\r\n ///\r\n /// The extrema should be reported in increasing parameter order.\r\n fn extrema(&self) -> ArrayVec;\r\n\r\n /// Return parameter ranges, each of which is monotonic within the range.\r\n fn extrema_ranges(&self) -> ArrayVec, { MAX_EXTREMA + 1 }> {\r\n let mut result = ArrayVec::new();\r\n let mut t0 = 0.0;\r\n for t in self.extrema() {\r\n result.push(t0..t);\r\n t0 = t;\r\n }\r\n result.push(t0..1.0);\r\n result\r\n }\r\n\r\n /// The smallest rectangle that encloses the curve in the range (0..1).\r\n fn bounding_box(&self) -> Rect {\r\n let mut bbox = Rect::from_points(self.start(), self.end());\r\n for t in self.extrema() {\r\n bbox = bbox.union_pt(self.eval(t))\r\n }\r\n bbox\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/point.rs", "sha256": "42977c1c6a9be6d591e46456d48d648d41e1259b3877949a07566b9d0d0250ee", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A 2D point.\r\n\r\nuse core::fmt;\r\nuse core::ops::{Add, AddAssign, Sub, SubAssign};\r\n\r\nuse crate::common::FloatExt;\r\nuse crate::Vec2;\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// A 2D point.\r\n///\r\n/// This type represents a point in 2D space. It has the same layout as [`Vec2`][crate::Vec2], but\r\n/// its meaning is different: `Vec2` represents a change in location (for example velocity).\r\n///\r\n/// In general, `kurbo` overloads math operators where it makes sense, for example implementing\r\n/// `Affine * Point` as the point under the affine transformation. However `Point + Point` and\r\n/// `f64 * Point` are not implemented, because the operations do not make geometric sense. If you\r\n/// need to apply these operations, then 1) check what you're doing makes geometric sense, then 2)\r\n/// use [`Point::to_vec2`] to convert the point to a `Vec2`.\r\n#[derive(Clone, Copy, Default, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct Point {\r\n /// The x coordinate.\r\n pub x: f64,\r\n /// The y coordinate.\r\n pub y: f64,\r\n}\r\n\r\nimpl Point {\r\n /// The point (0, 0).\r\n pub const ZERO: Point = Point::new(0., 0.);\r\n\r\n /// The point at the origin; (0, 0).\r\n pub const ORIGIN: Point = Point::new(0., 0.);\r\n\r\n /// Create a new `Point` with the provided `x` and `y` coordinates.\r\n #[inline]\r\n pub const fn new(x: f64, y: f64) -> Self {\r\n Point { x, y }\r\n }\r\n\r\n /// Convert this point into a `Vec2`.\r\n #[inline]\r\n pub const fn to_vec2(self) -> Vec2 {\r\n Vec2::new(self.x, self.y)\r\n }\r\n\r\n /// Linearly interpolate between two points.\r\n #[inline]\r\n pub fn lerp(self, other: Point, t: f64) -> Point {\r\n self.to_vec2().lerp(other.to_vec2(), t).to_point()\r\n }\r\n\r\n /// Determine the midpoint of two points.\r\n #[inline]\r\n pub fn midpoint(self, other: Point) -> Point {\r\n Point::new(0.5 * (self.x + other.x), 0.5 * (self.y + other.y))\r\n }\r\n\r\n /// Euclidean distance.\r\n #[inline]\r\n pub fn distance(self, other: Point) -> f64 {\r\n (self - other).hypot()\r\n }\r\n\r\n /// Squared Euclidean distance.\r\n #[inline]\r\n pub fn distance_squared(self, other: Point) -> f64 {\r\n (self - other).hypot2()\r\n }\r\n\r\n /// Returns a new `Point`,\r\n /// with `x` and `y` rounded to the nearest integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Point;\r\n /// let a = Point::new(3.3, 3.6).round();\r\n /// let b = Point::new(3.0, -3.1).round();\r\n /// assert_eq!(a.x, 3.0);\r\n /// assert_eq!(a.y, 4.0);\r\n /// assert_eq!(b.x, 3.0);\r\n /// assert_eq!(b.y, -3.0);\r\n /// ```\r\n #[inline]\r\n pub fn round(self) -> Point {\r\n Point::new(self.x.round(), self.y.round())\r\n }\r\n\r\n /// Returns a new `Point`,\r\n /// with `x` and `y` rounded up to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Point;\r\n /// let a = Point::new(3.3, 3.6).ceil();\r\n /// let b = Point::new(3.0, -3.1).ceil();\r\n /// assert_eq!(a.x, 4.0);\r\n /// assert_eq!(a.y, 4.0);\r\n /// assert_eq!(b.x, 3.0);\r\n /// assert_eq!(b.y, -3.0);\r\n /// ```\r\n #[inline]\r\n pub fn ceil(self) -> Point {\r\n Point::new(self.x.ceil(), self.y.ceil())\r\n }\r\n\r\n /// Returns a new `Point`,\r\n /// with `x` and `y` rounded down to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Point;\r\n /// let a = Point::new(3.3, 3.6).floor();\r\n /// let b = Point::new(3.0, -3.1).floor();\r\n /// assert_eq!(a.x, 3.0);\r\n /// assert_eq!(a.y, 3.0);\r\n /// assert_eq!(b.x, 3.0);\r\n /// assert_eq!(b.y, -4.0);\r\n /// ```\r\n #[inline]\r\n pub fn floor(self) -> Point {\r\n Point::new(self.x.floor(), self.y.floor())\r\n }\r\n\r\n /// Returns a new `Point`,\r\n /// with `x` and `y` rounded away from zero to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Point;\r\n /// let a = Point::new(3.3, 3.6).expand();\r\n /// let b = Point::new(3.0, -3.1).expand();\r\n /// assert_eq!(a.x, 4.0);\r\n /// assert_eq!(a.y, 4.0);\r\n /// assert_eq!(b.x, 3.0);\r\n /// assert_eq!(b.y, -4.0);\r\n /// ```\r\n #[inline]\r\n pub fn expand(self) -> Point {\r\n Point::new(self.x.expand(), self.y.expand())\r\n }\r\n\r\n /// Returns a new `Point`,\r\n /// with `x` and `y` rounded towards zero to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Point;\r\n /// let a = Point::new(3.3, 3.6).trunc();\r\n /// let b = Point::new(3.0, -3.1).trunc();\r\n /// assert_eq!(a.x, 3.0);\r\n /// assert_eq!(a.y, 3.0);\r\n /// assert_eq!(b.x, 3.0);\r\n /// assert_eq!(b.y, -3.0);\r\n /// ```\r\n #[inline]\r\n pub fn trunc(self) -> Point {\r\n Point::new(self.x.trunc(), self.y.trunc())\r\n }\r\n\r\n /// Is this point finite?\r\n #[inline]\r\n pub fn is_finite(self) -> bool {\r\n self.x.is_finite() && self.y.is_finite()\r\n }\r\n\r\n /// Is this point NaN?\r\n #[inline]\r\n pub fn is_nan(self) -> bool {\r\n self.x.is_nan() || self.y.is_nan()\r\n }\r\n}\r\n\r\nimpl From<(f64, f64)> for Point {\r\n #[inline]\r\n fn from(v: (f64, f64)) -> Point {\r\n Point { x: v.0, y: v.1 }\r\n }\r\n}\r\n\r\nimpl From for (f64, f64) {\r\n #[inline]\r\n fn from(v: Point) -> (f64, f64) {\r\n (v.x, v.y)\r\n }\r\n}\r\n\r\nimpl Add for Point {\r\n type Output = Point;\r\n\r\n #[inline]\r\n fn add(self, other: Vec2) -> Self {\r\n Point::new(self.x + other.x, self.y + other.y)\r\n }\r\n}\r\n\r\nimpl AddAssign for Point {\r\n #[inline]\r\n fn add_assign(&mut self, other: Vec2) {\r\n *self = Point::new(self.x + other.x, self.y + other.y)\r\n }\r\n}\r\n\r\nimpl Sub for Point {\r\n type Output = Point;\r\n\r\n #[inline]\r\n fn sub(self, other: Vec2) -> Self {\r\n Point::new(self.x - other.x, self.y - other.y)\r\n }\r\n}\r\n\r\nimpl SubAssign for Point {\r\n #[inline]\r\n fn sub_assign(&mut self, other: Vec2) {\r\n *self = Point::new(self.x - other.x, self.y - other.y)\r\n }\r\n}\r\n\r\nimpl Add<(f64, f64)> for Point {\r\n type Output = Point;\r\n\r\n #[inline]\r\n fn add(self, (x, y): (f64, f64)) -> Self {\r\n Point::new(self.x + x, self.y + y)\r\n }\r\n}\r\n\r\nimpl AddAssign<(f64, f64)> for Point {\r\n #[inline]\r\n fn add_assign(&mut self, (x, y): (f64, f64)) {\r\n *self = Point::new(self.x + x, self.y + y)\r\n }\r\n}\r\n\r\nimpl Sub<(f64, f64)> for Point {\r\n type Output = Point;\r\n\r\n #[inline]\r\n fn sub(self, (x, y): (f64, f64)) -> Self {\r\n Point::new(self.x - x, self.y - y)\r\n }\r\n}\r\n\r\nimpl SubAssign<(f64, f64)> for Point {\r\n #[inline]\r\n fn sub_assign(&mut self, (x, y): (f64, f64)) {\r\n *self = Point::new(self.x - x, self.y - y)\r\n }\r\n}\r\n\r\nimpl Sub for Point {\r\n type Output = Vec2;\r\n\r\n #[inline]\r\n fn sub(self, other: Point) -> Vec2 {\r\n Vec2::new(self.x - other.x, self.y - other.y)\r\n }\r\n}\r\n\r\nimpl fmt::Debug for Point {\r\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\r\n write!(f, \"({:?}, {:?})\", self.x, self.y)\r\n }\r\n}\r\n\r\nimpl fmt::Display for Point {\r\n fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\r\n write!(formatter, \"(\")?;\r\n fmt::Display::fmt(&self.x, formatter)?;\r\n write!(formatter, \", \")?;\r\n fmt::Display::fmt(&self.y, formatter)?;\r\n write!(formatter, \")\")\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use super::*;\r\n #[test]\r\n fn point_arithmetic() {\r\n assert_eq!(\r\n Point::new(0., 0.) - Vec2::new(10., 0.),\r\n Point::new(-10., 0.)\r\n );\r\n assert_eq!(\r\n Point::new(0., 0.) - Point::new(-5., 101.),\r\n Vec2::new(5., -101.)\r\n );\r\n }\r\n\r\n #[test]\r\n #[allow(clippy::float_cmp)]\r\n fn distance() {\r\n let p1 = Point::new(0., 10.);\r\n let p2 = Point::new(0., 5.);\r\n assert_eq!(p1.distance(p2), 5.);\r\n\r\n let p1 = Point::new(-11., 1.);\r\n let p2 = Point::new(-7., -2.);\r\n assert_eq!(p1.distance(p2), 5.);\r\n }\r\n\r\n #[test]\r\n fn display() {\r\n let p = Point::new(0.12345, 9.87654);\r\n assert_eq!(format!(\"{p}\"), \"(0.12345, 9.87654)\");\r\n\r\n let p = Point::new(0.12345, 9.87654);\r\n assert_eq!(format!(\"{p:.2}\"), \"(0.12, 9.88)\");\r\n }\r\n}\r\n\r\n#[cfg(feature = \"mint\")]\r\nimpl From for mint::Point2 {\r\n #[inline]\r\n fn from(p: Point) -> mint::Point2 {\r\n mint::Point2 { x: p.x, y: p.y }\r\n }\r\n}\r\n\r\n#[cfg(feature = \"mint\")]\r\nimpl From> for Point {\r\n #[inline]\r\n fn from(p: mint::Point2) -> Point {\r\n Point { x: p.x, y: p.y }\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/quadbez.rs", "sha256": "deb6971c8a0403ac9a1a9a84f40b0c630b056f9edd75aea74b062eae555b9903", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Quadratic Bézier segments.\r\n\r\nuse core::ops::{Mul, Range};\r\n\r\nuse arrayvec::ArrayVec;\r\n\r\nuse crate::common::solve_cubic;\r\nuse crate::MAX_EXTREMA;\r\nuse crate::{\r\n Affine, CubicBez, Line, Nearest, ParamCurve, ParamCurveArclen, ParamCurveArea,\r\n ParamCurveCurvature, ParamCurveDeriv, ParamCurveExtrema, ParamCurveNearest, PathEl, Point,\r\n Rect, Shape,\r\n};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// A single quadratic Bézier segment.\r\n#[derive(Clone, Copy, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\n#[allow(missing_docs)]\r\npub struct QuadBez {\r\n pub p0: Point,\r\n pub p1: Point,\r\n pub p2: Point,\r\n}\r\n\r\nimpl QuadBez {\r\n /// Create a new quadratic Bézier segment.\r\n #[inline]\r\n pub fn new>(p0: V, p1: V, p2: V) -> QuadBez {\r\n QuadBez {\r\n p0: p0.into(),\r\n p1: p1.into(),\r\n p2: p2.into(),\r\n }\r\n }\r\n\r\n /// Raise the order by 1.\r\n ///\r\n /// Returns a cubic Bézier segment that exactly represents this quadratic.\r\n #[inline]\r\n pub fn raise(&self) -> CubicBez {\r\n CubicBez::new(\r\n self.p0,\r\n self.p0 + (2.0 / 3.0) * (self.p1 - self.p0),\r\n self.p2 + (2.0 / 3.0) * (self.p1 - self.p2),\r\n self.p2,\r\n )\r\n }\r\n\r\n /// Estimate the number of subdivisions for flattening.\r\n pub(crate) fn estimate_subdiv(&self, sqrt_tol: f64) -> FlattenParams {\r\n // Determine transformation to $y = x^2$ parabola.\r\n let d01 = self.p1 - self.p0;\r\n let d12 = self.p2 - self.p1;\r\n let dd = d01 - d12;\r\n let cross = (self.p2 - self.p0).cross(dd);\r\n let x0 = d01.dot(dd) * cross.recip();\r\n let x2 = d12.dot(dd) * cross.recip();\r\n let scale = (cross / (dd.hypot() * (x2 - x0))).abs();\r\n\r\n // Compute number of subdivisions needed.\r\n let a0 = approx_parabola_integral(x0);\r\n let a2 = approx_parabola_integral(x2);\r\n let val = if scale.is_finite() {\r\n let da = (a2 - a0).abs();\r\n let sqrt_scale = scale.sqrt();\r\n if x0.signum() == x2.signum() {\r\n da * sqrt_scale\r\n } else {\r\n // Handle cusp case (segment contains curvature maximum)\r\n let xmin = sqrt_tol / sqrt_scale;\r\n sqrt_tol * da / approx_parabola_integral(xmin)\r\n }\r\n } else {\r\n 0.0\r\n };\r\n let u0 = approx_parabola_inv_integral(a0);\r\n let u2 = approx_parabola_inv_integral(a2);\r\n let uscale = (u2 - u0).recip();\r\n FlattenParams {\r\n a0,\r\n a2,\r\n u0,\r\n uscale,\r\n val,\r\n }\r\n }\r\n\r\n // Maps a value from 0..1 to 0..1.\r\n pub(crate) fn determine_subdiv_t(&self, params: &FlattenParams, x: f64) -> f64 {\r\n let a = params.a0 + (params.a2 - params.a0) * x;\r\n let u = approx_parabola_inv_integral(a);\r\n (u - params.u0) * params.uscale\r\n }\r\n\r\n /// Is this quadratic Bezier curve finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n self.p0.is_finite() && self.p1.is_finite() && self.p2.is_finite()\r\n }\r\n\r\n /// Is this quadratic Bezier curve NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n self.p0.is_nan() || self.p1.is_nan() || self.p2.is_nan()\r\n }\r\n}\r\n\r\n/// An iterator for quadratic beziers.\r\npub struct QuadBezIter {\r\n quad: QuadBez,\r\n ix: usize,\r\n}\r\n\r\nimpl Shape for QuadBez {\r\n type PathElementsIter<'iter> = QuadBezIter;\r\n\r\n #[inline]\r\n fn path_elements(&self, _tolerance: f64) -> QuadBezIter {\r\n QuadBezIter { quad: *self, ix: 0 }\r\n }\r\n\r\n fn area(&self) -> f64 {\r\n 0.0\r\n }\r\n\r\n #[inline]\r\n fn perimeter(&self, accuracy: f64) -> f64 {\r\n self.arclen(accuracy)\r\n }\r\n\r\n fn winding(&self, _pt: Point) -> i32 {\r\n 0\r\n }\r\n\r\n #[inline]\r\n fn bounding_box(&self) -> Rect {\r\n ParamCurveExtrema::bounding_box(self)\r\n }\r\n}\r\n\r\nimpl Iterator for QuadBezIter {\r\n type Item = PathEl;\r\n\r\n fn next(&mut self) -> Option {\r\n self.ix += 1;\r\n match self.ix {\r\n 1 => Some(PathEl::MoveTo(self.quad.p0)),\r\n 2 => Some(PathEl::QuadTo(self.quad.p1, self.quad.p2)),\r\n _ => None,\r\n }\r\n }\r\n}\r\n\r\npub(crate) struct FlattenParams {\r\n a0: f64,\r\n a2: f64,\r\n u0: f64,\r\n uscale: f64,\r\n /// The number of subdivisions * 2 * sqrt_tol.\r\n pub(crate) val: f64,\r\n}\r\n\r\n/// An approximation to $\\int (1 + 4x^2) ^ -0.25 dx$\r\n///\r\n/// This is used for flattening curves.\r\nfn approx_parabola_integral(x: f64) -> f64 {\r\n const D: f64 = 0.67;\r\n x / (1.0 - D + (D.powi(4) + 0.25 * x * x).sqrt().sqrt())\r\n}\r\n\r\n/// An approximation to the inverse parabola integral.\r\nfn approx_parabola_inv_integral(x: f64) -> f64 {\r\n const B: f64 = 0.39;\r\n x * (1.0 - B + (B * B + 0.25 * x * x).sqrt())\r\n}\r\n\r\nimpl ParamCurve for QuadBez {\r\n #[inline]\r\n fn eval(&self, t: f64) -> Point {\r\n let mt = 1.0 - t;\r\n (self.p0.to_vec2() * (mt * mt)\r\n + (self.p1.to_vec2() * (mt * 2.0) + self.p2.to_vec2() * t) * t)\r\n .to_point()\r\n }\r\n\r\n fn subsegment(&self, range: Range) -> QuadBez {\r\n let (t0, t1) = (range.start, range.end);\r\n let p0 = self.eval(t0);\r\n let p2 = self.eval(t1);\r\n let p1 = p0 + (self.p1 - self.p0).lerp(self.p2 - self.p1, t0) * (t1 - t0);\r\n QuadBez { p0, p1, p2 }\r\n }\r\n\r\n /// Subdivide into halves, using de Casteljau.\r\n #[inline]\r\n fn subdivide(&self) -> (QuadBez, QuadBez) {\r\n let pm = self.eval(0.5);\r\n (\r\n QuadBez::new(self.p0, self.p0.midpoint(self.p1), pm),\r\n QuadBez::new(pm, self.p1.midpoint(self.p2), self.p2),\r\n )\r\n }\r\n\r\n #[inline]\r\n fn start(&self) -> Point {\r\n self.p0\r\n }\r\n\r\n #[inline]\r\n fn end(&self) -> Point {\r\n self.p2\r\n }\r\n}\r\n\r\nimpl ParamCurveDeriv for QuadBez {\r\n type DerivResult = Line;\r\n\r\n #[inline]\r\n fn deriv(&self) -> Line {\r\n Line::new(\r\n (2.0 * (self.p1.to_vec2() - self.p0.to_vec2())).to_point(),\r\n (2.0 * (self.p2.to_vec2() - self.p1.to_vec2())).to_point(),\r\n )\r\n }\r\n}\r\n\r\nimpl ParamCurveArclen for QuadBez {\r\n /// Arclength of a quadratic Bézier segment.\r\n ///\r\n /// This computation is based on an analytical formula. Since that formula suffers\r\n /// from numerical instability when the curve is very close to a straight line, we\r\n /// detect that case and fall back to Legendre-Gauss quadrature.\r\n ///\r\n /// Accuracy should be better than 1e-13 over the entire range.\r\n ///\r\n /// Adapted from \r\n /// with permission.\r\n fn arclen(&self, _accuracy: f64) -> f64 {\r\n let d2 = self.p0.to_vec2() - 2.0 * self.p1.to_vec2() + self.p2.to_vec2();\r\n let a = d2.hypot2();\r\n let d1 = self.p1 - self.p0;\r\n let c = d1.hypot2();\r\n if a < 5e-4 * c {\r\n // This case happens for nearly straight Béziers.\r\n //\r\n // Calculate arclength using Legendre-Gauss quadrature using formula from Behdad\r\n // in https://github.com/Pomax/BezierInfo-2/issues/77\r\n let v0 = (-0.492943519233745 * self.p0.to_vec2()\r\n + 0.430331482911935 * self.p1.to_vec2()\r\n + 0.0626120363218102 * self.p2.to_vec2())\r\n .hypot();\r\n let v1 = ((self.p2 - self.p0) * 0.4444444444444444).hypot();\r\n let v2 = (-0.0626120363218102 * self.p0.to_vec2()\r\n - 0.430331482911935 * self.p1.to_vec2()\r\n + 0.492943519233745 * self.p2.to_vec2())\r\n .hypot();\r\n return v0 + v1 + v2;\r\n }\r\n let b = 2.0 * d2.dot(d1);\r\n\r\n let sabc = (a + b + c).sqrt();\r\n let a2 = a.powf(-0.5);\r\n let a32 = a2.powi(3);\r\n let c2 = 2.0 * c.sqrt();\r\n let ba_c2 = b * a2 + c2;\r\n\r\n let v0 = 0.25 * a2 * a2 * b * (2.0 * sabc - c2) + sabc;\r\n // TODO: justify and fine-tune this exact constant.\r\n if ba_c2 < 1e-13 {\r\n // This case happens for Béziers with a sharp kink.\r\n v0\r\n } else {\r\n v0 + 0.25\r\n * a32\r\n * (4.0 * c * a - b * b)\r\n * (((2.0 * a + b) * a2 + 2.0 * sabc) / ba_c2).ln()\r\n }\r\n }\r\n}\r\n\r\nimpl ParamCurveArea for QuadBez {\r\n #[inline]\r\n fn signed_area(&self) -> f64 {\r\n (self.p0.x * (2.0 * self.p1.y + self.p2.y) + 2.0 * self.p1.x * (self.p2.y - self.p0.y)\r\n - self.p2.x * (self.p0.y + 2.0 * self.p1.y))\r\n * (1.0 / 6.0)\r\n }\r\n}\r\n\r\nimpl ParamCurveNearest for QuadBez {\r\n /// Find the nearest point, using analytical algorithm based on cubic root finding.\r\n fn nearest(&self, p: Point, _accuracy: f64) -> Nearest {\r\n fn eval_t(p: Point, t_best: &mut f64, r_best: &mut Option, t: f64, p0: Point) {\r\n let r = (p0 - p).hypot2();\r\n if r_best.map(|r_best| r < r_best).unwrap_or(true) {\r\n *r_best = Some(r);\r\n *t_best = t;\r\n }\r\n }\r\n fn try_t(\r\n q: &QuadBez,\r\n p: Point,\r\n t_best: &mut f64,\r\n r_best: &mut Option,\r\n t: f64,\r\n ) -> bool {\r\n if !(0.0..=1.0).contains(&t) {\r\n return true;\r\n }\r\n eval_t(p, t_best, r_best, t, q.eval(t));\r\n false\r\n }\r\n let d0 = self.p1 - self.p0;\r\n let d1 = self.p0.to_vec2() + self.p2.to_vec2() - 2.0 * self.p1.to_vec2();\r\n let d = self.p0 - p;\r\n let c0 = d.dot(d0);\r\n let c1 = 2.0 * d0.hypot2() + d.dot(d1);\r\n let c2 = 3.0 * d1.dot(d0);\r\n let c3 = d1.hypot2();\r\n let roots = solve_cubic(c0, c1, c2, c3);\r\n let mut r_best = None;\r\n let mut t_best = 0.0;\r\n let mut need_ends = false;\r\n for &t in &roots {\r\n need_ends |= try_t(self, p, &mut t_best, &mut r_best, t);\r\n }\r\n if need_ends {\r\n eval_t(p, &mut t_best, &mut r_best, 0.0, self.p0);\r\n eval_t(p, &mut t_best, &mut r_best, 1.0, self.p2);\r\n }\r\n\r\n Nearest {\r\n t: t_best,\r\n distance_sq: r_best.unwrap(),\r\n }\r\n }\r\n}\r\n\r\nimpl ParamCurveCurvature for QuadBez {}\r\n\r\nimpl ParamCurveExtrema for QuadBez {\r\n fn extrema(&self) -> ArrayVec {\r\n let mut result = ArrayVec::new();\r\n let d0 = self.p1 - self.p0;\r\n let d1 = self.p2 - self.p1;\r\n let dd = d1 - d0;\r\n if dd.x != 0.0 {\r\n let t = -d0.x / dd.x;\r\n if t > 0.0 && t < 1.0 {\r\n result.push(t);\r\n }\r\n }\r\n if dd.y != 0.0 {\r\n let t = -d0.y / dd.y;\r\n if t > 0.0 && t < 1.0 {\r\n result.push(t);\r\n if result.len() == 2 && result[0] > t {\r\n result.swap(0, 1);\r\n }\r\n }\r\n }\r\n result\r\n }\r\n}\r\n\r\nimpl Mul for Affine {\r\n type Output = QuadBez;\r\n\r\n #[inline]\r\n fn mul(self, other: QuadBez) -> QuadBez {\r\n QuadBez {\r\n p0: self * other.p0,\r\n p1: self * other.p1,\r\n p2: self * other.p2,\r\n }\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{\r\n Affine, Nearest, ParamCurve, ParamCurveArclen, ParamCurveArea, ParamCurveDeriv,\r\n ParamCurveExtrema, ParamCurveNearest, Point, QuadBez,\r\n };\r\n\r\n fn assert_near(p0: Point, p1: Point, epsilon: f64) {\r\n assert!((p1 - p0).hypot() < epsilon, \"{p0:?} != {p1:?}\");\r\n }\r\n\r\n #[test]\r\n fn quadbez_deriv() {\r\n let q = QuadBez::new((0.0, 0.0), (0.0, 0.5), (1.0, 1.0));\r\n let deriv = q.deriv();\r\n\r\n let n = 10;\r\n for i in 0..=n {\r\n let t = (i as f64) * (n as f64).recip();\r\n let delta = 1e-6;\r\n let p = q.eval(t);\r\n let p1 = q.eval(t + delta);\r\n let d_approx = (p1 - p) * delta.recip();\r\n let d = deriv.eval(t).to_vec2();\r\n assert!((d - d_approx).hypot() < delta * 2.0);\r\n }\r\n }\r\n\r\n #[test]\r\n fn quadbez_arclen() {\r\n let q = QuadBez::new((0.0, 0.0), (0.0, 0.5), (1.0, 1.0));\r\n let true_arclen = 0.5 * 5.0f64.sqrt() + 0.25 * (2.0 + 5.0f64.sqrt()).ln();\r\n for i in 0..12 {\r\n let accuracy = 0.1f64.powi(i);\r\n let est = q.arclen(accuracy);\r\n let error = est - true_arclen;\r\n assert!(error.abs() < accuracy, \"{est} != {true_arclen}\");\r\n }\r\n }\r\n\r\n #[test]\r\n fn quadbez_arclen_pathological() {\r\n let q = QuadBez::new((-1.0, 0.0), (1.03, 0.0), (1.0, 0.0));\r\n let true_arclen = 2.0008737864167325; // A rough empirical calculation\r\n let accuracy = 1e-11;\r\n let est = q.arclen(accuracy);\r\n assert!(\r\n (est - true_arclen).abs() < accuracy,\r\n \"{est} != {true_arclen}\"\r\n );\r\n }\r\n\r\n #[test]\r\n fn quadbez_subsegment() {\r\n let q = QuadBez::new((3.1, 4.1), (5.9, 2.6), (5.3, 5.8));\r\n let t0 = 0.1;\r\n let t1 = 0.8;\r\n let qs = q.subsegment(t0..t1);\r\n let epsilon = 1e-12;\r\n let n = 10;\r\n for i in 0..=n {\r\n let t = (i as f64) * (n as f64).recip();\r\n let ts = t0 + t * (t1 - t0);\r\n assert_near(q.eval(ts), qs.eval(t), epsilon);\r\n }\r\n }\r\n\r\n #[test]\r\n fn quadbez_raise() {\r\n let q = QuadBez::new((3.1, 4.1), (5.9, 2.6), (5.3, 5.8));\r\n let c = q.raise();\r\n let qd = q.deriv();\r\n let cd = c.deriv();\r\n let epsilon = 1e-12;\r\n let n = 10;\r\n for i in 0..=n {\r\n let t = (i as f64) * (n as f64).recip();\r\n assert_near(q.eval(t), c.eval(t), epsilon);\r\n assert_near(qd.eval(t), cd.eval(t), epsilon);\r\n }\r\n }\r\n\r\n #[test]\r\n fn quadbez_signed_area() {\r\n // y = 1 - x^2\r\n let q = QuadBez::new((1.0, 0.0), (0.5, 1.0), (0.0, 1.0));\r\n let epsilon = 1e-12;\r\n assert!((q.signed_area() - 2.0 / 3.0).abs() < epsilon);\r\n assert!(((Affine::rotate(0.5) * q).signed_area() - 2.0 / 3.0).abs() < epsilon);\r\n assert!(((Affine::translate((0.0, 1.0)) * q).signed_area() - 3.5 / 3.0).abs() < epsilon);\r\n assert!(((Affine::translate((1.0, 0.0)) * q).signed_area() - 3.5 / 3.0).abs() < epsilon);\r\n }\r\n\r\n fn verify(result: Nearest, expected: f64) {\r\n assert!(\r\n (result.t - expected).abs() < 1e-6,\r\n \"got {result:?} expected {expected}\"\r\n );\r\n }\r\n\r\n #[test]\r\n fn quadbez_nearest() {\r\n // y = x^2\r\n let q = QuadBez::new((-1.0, 1.0), (0.0, -1.0), (1.0, 1.0));\r\n verify(q.nearest((0.0, 0.0).into(), 1e-3), 0.5);\r\n verify(q.nearest((0.0, 0.1).into(), 1e-3), 0.5);\r\n verify(q.nearest((0.0, -0.1).into(), 1e-3), 0.5);\r\n verify(q.nearest((0.5, 0.25).into(), 1e-3), 0.75);\r\n verify(q.nearest((1.0, 1.0).into(), 1e-3), 1.0);\r\n verify(q.nearest((1.1, 1.1).into(), 1e-3), 1.0);\r\n verify(q.nearest((-1.1, 1.1).into(), 1e-3), 0.0);\r\n let a = Affine::rotate(0.5);\r\n verify((a * q).nearest(a * Point::new(0.5, 0.25), 1e-3), 0.75);\r\n }\r\n\r\n // This test exposes a degenerate case in the solver used internally\r\n // by the \"nearest\" calculation - the cubic term is zero.\r\n #[test]\r\n fn quadbez_nearest_low_order() {\r\n let q = QuadBez::new((-1.0, 0.0), (0.0, 0.0), (1.0, 0.0));\r\n\r\n verify(q.nearest((0.0, 0.0).into(), 1e-3), 0.5);\r\n verify(q.nearest((0.0, 1.0).into(), 1e-3), 0.5);\r\n }\r\n\r\n #[test]\r\n fn quadbez_extrema() {\r\n // y = x^2\r\n let q = QuadBez::new((-1.0, 1.0), (0.0, -1.0), (1.0, 1.0));\r\n let extrema = q.extrema();\r\n assert_eq!(extrema.len(), 1);\r\n assert!((extrema[0] - 0.5).abs() < 1e-6);\r\n\r\n let q = QuadBez::new((0.0, 0.5), (1.0, 1.0), (0.5, 0.0));\r\n let extrema = q.extrema();\r\n assert_eq!(extrema.len(), 2);\r\n assert!((extrema[0] - 1.0 / 3.0).abs() < 1e-6);\r\n assert!((extrema[1] - 2.0 / 3.0).abs() < 1e-6);\r\n\r\n // Reverse direction\r\n let q = QuadBez::new((0.5, 0.0), (1.0, 1.0), (0.0, 0.5));\r\n let extrema = q.extrema();\r\n assert_eq!(extrema.len(), 2);\r\n assert!((extrema[0] - 1.0 / 3.0).abs() < 1e-6);\r\n assert!((extrema[1] - 2.0 / 3.0).abs() < 1e-6);\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/quadspline.rs", "sha256": "6da289e294f4b86809d9c0354d5d7c1b999c14368106172ea4025e55cab97cd8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2021 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Quadratic Bézier splines.\r\nuse crate::Point;\r\n\r\nuse crate::QuadBez;\r\nuse alloc::vec::Vec;\r\n\r\n/// A quadratic Bézier spline in [B-spline](https://en.wikipedia.org/wiki/B-spline) format.\r\n#[derive(Clone, Debug, PartialEq)]\r\npub struct QuadSpline(Vec);\r\n\r\nimpl QuadSpline {\r\n /// Construct a new `QuadSpline` from an array of [`Point`]s.\r\n #[inline]\r\n pub fn new(points: Vec) -> Self {\r\n Self(points)\r\n }\r\n\r\n /// Return the spline's control [`Point`]s.\r\n #[inline]\r\n pub fn points(&self) -> &[Point] {\r\n &self.0\r\n }\r\n\r\n /// Return an iterator over the implied [`QuadBez`] sequence.\r\n ///\r\n /// The returned quads are guaranteed to be G1 continuous.\r\n #[inline]\r\n pub fn to_quads(&self) -> impl Iterator + '_ {\r\n ToQuadBez {\r\n idx: 0,\r\n points: &self.0,\r\n }\r\n }\r\n}\r\n\r\nstruct ToQuadBez<'a> {\r\n idx: usize,\r\n points: &'a Vec,\r\n}\r\n\r\nimpl<'a> Iterator for ToQuadBez<'a> {\r\n type Item = QuadBez;\r\n\r\n fn next(&mut self) -> Option {\r\n let [mut p0, p1, mut p2]: [Point; 3] =\r\n self.points.get(self.idx..=self.idx + 2)?.try_into().ok()?;\r\n\r\n if self.idx != 0 {\r\n p0 = p0.midpoint(p1);\r\n }\r\n if self.idx + 2 < self.points.len() - 1 {\r\n p2 = p1.midpoint(p2);\r\n }\r\n\r\n self.idx += 1;\r\n\r\n Some(QuadBez { p0, p1, p2 })\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{Point, QuadBez, QuadSpline};\r\n\r\n #[test]\r\n pub fn no_points_no_quads() {\r\n assert!(QuadSpline::new(Vec::new()).to_quads().next().is_none());\r\n }\r\n\r\n #[test]\r\n pub fn one_point_no_quads() {\r\n assert!(QuadSpline::new(vec![Point::new(1.0, 1.0)])\r\n .to_quads()\r\n .next()\r\n .is_none());\r\n }\r\n\r\n #[test]\r\n pub fn two_points_no_quads() {\r\n assert!(\r\n QuadSpline::new(vec![Point::new(1.0, 1.0), Point::new(1.0, 1.0)])\r\n .to_quads()\r\n .next()\r\n .is_none()\r\n );\r\n }\r\n\r\n #[test]\r\n pub fn three_points_same_quad() {\r\n let p0 = Point::new(1.0, 1.0);\r\n let p1 = Point::new(2.0, 2.0);\r\n let p2 = Point::new(3.0, 3.0);\r\n assert_eq!(\r\n vec![QuadBez { p0, p1, p2 }],\r\n QuadSpline::new(vec![p0, p1, p2])\r\n .to_quads()\r\n .collect::>()\r\n );\r\n }\r\n\r\n #[test]\r\n pub fn four_points_implicit_on_curve() {\r\n let p0 = Point::new(1.0, 1.0);\r\n let p1 = Point::new(3.0, 3.0);\r\n let p2 = Point::new(5.0, 5.0);\r\n let p3 = Point::new(8.0, 8.0);\r\n assert_eq!(\r\n vec![\r\n QuadBez {\r\n p0,\r\n p1,\r\n p2: p1.midpoint(p2)\r\n },\r\n QuadBez {\r\n p0: p1.midpoint(p2),\r\n p1: p2,\r\n p2: p3\r\n }\r\n ],\r\n QuadSpline::new(vec![p0, p1, p2, p3])\r\n .to_quads()\r\n .collect::>()\r\n );\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/rect.rs", "sha256": "57f99765f89f7b96a0204dfda087ad539eb2dafcc68afcad100a176b89522692", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A rectangle.\r\n\r\nuse core::fmt;\r\nuse core::ops::{Add, Sub};\r\n\r\nuse crate::{Ellipse, Insets, PathEl, Point, RoundedRect, RoundedRectRadii, Shape, Size, Vec2};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// A rectangle.\r\n#[derive(Clone, Copy, Default, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct Rect {\r\n /// The minimum x coordinate (left edge).\r\n pub x0: f64,\r\n /// The minimum y coordinate (top edge in y-down spaces).\r\n pub y0: f64,\r\n /// The maximum x coordinate (right edge).\r\n pub x1: f64,\r\n /// The maximum y coordinate (bottom edge in y-down spaces).\r\n pub y1: f64,\r\n}\r\n\r\nimpl Rect {\r\n /// The empty rectangle at the origin.\r\n pub const ZERO: Rect = Rect::new(0., 0., 0., 0.);\r\n\r\n /// A new rectangle from minimum and maximum coordinates.\r\n #[inline]\r\n pub const fn new(x0: f64, y0: f64, x1: f64, y1: f64) -> Rect {\r\n Rect { x0, y0, x1, y1 }\r\n }\r\n\r\n /// A new rectangle from two points.\r\n ///\r\n /// The result will have non-negative width and height.\r\n #[inline]\r\n pub fn from_points(p0: impl Into, p1: impl Into) -> Rect {\r\n let p0 = p0.into();\r\n let p1 = p1.into();\r\n Rect::new(p0.x, p0.y, p1.x, p1.y).abs()\r\n }\r\n\r\n /// A new rectangle from origin and size.\r\n ///\r\n /// The result will have non-negative width and height.\r\n #[inline]\r\n pub fn from_origin_size(origin: impl Into, size: impl Into) -> Rect {\r\n let origin = origin.into();\r\n Rect::from_points(origin, origin + size.into().to_vec2())\r\n }\r\n\r\n /// A new rectangle from center and size.\r\n #[inline]\r\n pub fn from_center_size(center: impl Into, size: impl Into) -> Rect {\r\n let center = center.into();\r\n let size = 0.5 * size.into();\r\n Rect::new(\r\n center.x - size.width,\r\n center.y - size.height,\r\n center.x + size.width,\r\n center.y + size.height,\r\n )\r\n }\r\n\r\n /// Create a new `Rect` with the same size as `self` and a new origin.\r\n #[inline]\r\n pub fn with_origin(self, origin: impl Into) -> Rect {\r\n Rect::from_origin_size(origin, self.size())\r\n }\r\n\r\n /// Create a new `Rect` with the same origin as `self` and a new size.\r\n #[inline]\r\n pub fn with_size(self, size: impl Into) -> Rect {\r\n Rect::from_origin_size(self.origin(), size)\r\n }\r\n\r\n /// Create a new `Rect` by applying the [`Insets`].\r\n ///\r\n /// This will not preserve negative width and height.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Rect;\r\n /// let inset_rect = Rect::new(0., 0., 10., 10.,).inset(2.);\r\n /// assert_eq!(inset_rect.width(), 14.0);\r\n /// assert_eq!(inset_rect.x0, -2.0);\r\n /// assert_eq!(inset_rect.x1, 12.0);\r\n /// ```\r\n #[inline]\r\n pub fn inset(self, insets: impl Into) -> Rect {\r\n self + insets.into()\r\n }\r\n\r\n /// The width of the rectangle.\r\n ///\r\n /// Note: nothing forbids negative width.\r\n #[inline]\r\n pub fn width(&self) -> f64 {\r\n self.x1 - self.x0\r\n }\r\n\r\n /// The height of the rectangle.\r\n ///\r\n /// Note: nothing forbids negative height.\r\n #[inline]\r\n pub fn height(&self) -> f64 {\r\n self.y1 - self.y0\r\n }\r\n\r\n /// Returns the minimum value for the x-coordinate of the rectangle.\r\n #[inline]\r\n pub fn min_x(&self) -> f64 {\r\n self.x0.min(self.x1)\r\n }\r\n\r\n /// Returns the maximum value for the x-coordinate of the rectangle.\r\n #[inline]\r\n pub fn max_x(&self) -> f64 {\r\n self.x0.max(self.x1)\r\n }\r\n\r\n /// Returns the minimum value for the y-coordinate of the rectangle.\r\n #[inline]\r\n pub fn min_y(&self) -> f64 {\r\n self.y0.min(self.y1)\r\n }\r\n\r\n /// Returns the maximum value for the y-coordinate of the rectangle.\r\n #[inline]\r\n pub fn max_y(&self) -> f64 {\r\n self.y0.max(self.y1)\r\n }\r\n\r\n /// The origin of the rectangle.\r\n ///\r\n /// This is the top left corner in a y-down space and with\r\n /// non-negative width and height.\r\n #[inline]\r\n pub fn origin(&self) -> Point {\r\n Point::new(self.x0, self.y0)\r\n }\r\n\r\n /// The size of the rectangle.\r\n #[inline]\r\n pub fn size(&self) -> Size {\r\n Size::new(self.width(), self.height())\r\n }\r\n\r\n /// The area of the rectangle.\r\n #[inline]\r\n pub fn area(&self) -> f64 {\r\n self.width() * self.height()\r\n }\r\n\r\n /// Whether this rectangle has zero area.\r\n ///\r\n /// Note: a rectangle with negative area is not considered empty.\r\n #[inline]\r\n pub fn is_empty(&self) -> bool {\r\n self.area() == 0.0\r\n }\r\n\r\n /// The center point of the rectangle.\r\n #[inline]\r\n pub fn center(&self) -> Point {\r\n Point::new(0.5 * (self.x0 + self.x1), 0.5 * (self.y0 + self.y1))\r\n }\r\n\r\n /// Returns `true` if `point` lies within `self`.\r\n #[inline]\r\n pub fn contains(&self, point: Point) -> bool {\r\n point.x >= self.x0 && point.x < self.x1 && point.y >= self.y0 && point.y < self.y1\r\n }\r\n\r\n /// Take absolute value of width and height.\r\n ///\r\n /// The resulting rect has the same extents as the original, but is\r\n /// guaranteed to have non-negative width and height.\r\n #[inline]\r\n pub fn abs(&self) -> Rect {\r\n let Rect { x0, y0, x1, y1 } = *self;\r\n Rect::new(x0.min(x1), y0.min(y1), x0.max(x1), y0.max(y1))\r\n }\r\n\r\n /// The smallest rectangle enclosing two rectangles.\r\n ///\r\n /// Results are valid only if width and height are non-negative.\r\n #[inline]\r\n pub fn union(&self, other: Rect) -> Rect {\r\n Rect::new(\r\n self.x0.min(other.x0),\r\n self.y0.min(other.y0),\r\n self.x1.max(other.x1),\r\n self.y1.max(other.y1),\r\n )\r\n }\r\n\r\n /// Compute the union with one point.\r\n ///\r\n /// This method includes the perimeter of zero-area rectangles.\r\n /// Thus, a succession of `union_pt` operations on a series of\r\n /// points yields their enclosing rectangle.\r\n ///\r\n /// Results are valid only if width and height are non-negative.\r\n pub fn union_pt(&self, pt: Point) -> Rect {\r\n Rect::new(\r\n self.x0.min(pt.x),\r\n self.y0.min(pt.y),\r\n self.x1.max(pt.x),\r\n self.y1.max(pt.y),\r\n )\r\n }\r\n\r\n /// The intersection of two rectangles.\r\n ///\r\n /// The result is zero-area if either input has negative width or\r\n /// height. The result always has non-negative width and height.\r\n #[inline]\r\n pub fn intersect(&self, other: Rect) -> Rect {\r\n let x0 = self.x0.max(other.x0);\r\n let y0 = self.y0.max(other.y0);\r\n let x1 = self.x1.min(other.x1);\r\n let y1 = self.y1.min(other.y1);\r\n Rect::new(x0, y0, x1.max(x0), y1.max(y0))\r\n }\r\n\r\n /// Expand a rectangle by a constant amount in both directions.\r\n ///\r\n /// The logic simply applies the amount in each direction. If rectangle\r\n /// area or added dimensions are negative, this could give odd results.\r\n pub fn inflate(&self, width: f64, height: f64) -> Rect {\r\n Rect::new(\r\n self.x0 - width,\r\n self.y0 - height,\r\n self.x1 + width,\r\n self.y1 + height,\r\n )\r\n }\r\n\r\n /// Returns a new `Rect`,\r\n /// with each coordinate value rounded to the nearest integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Rect;\r\n /// let rect = Rect::new(3.3, 3.6, 3.0, -3.1).round();\r\n /// assert_eq!(rect.x0, 3.0);\r\n /// assert_eq!(rect.y0, 4.0);\r\n /// assert_eq!(rect.x1, 3.0);\r\n /// assert_eq!(rect.y1, -3.0);\r\n /// ```\r\n #[inline]\r\n pub fn round(self) -> Rect {\r\n Rect::new(\r\n self.x0.round(),\r\n self.y0.round(),\r\n self.x1.round(),\r\n self.y1.round(),\r\n )\r\n }\r\n\r\n /// Returns a new `Rect`,\r\n /// with each coordinate value rounded up to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Rect;\r\n /// let rect = Rect::new(3.3, 3.6, 3.0, -3.1).ceil();\r\n /// assert_eq!(rect.x0, 4.0);\r\n /// assert_eq!(rect.y0, 4.0);\r\n /// assert_eq!(rect.x1, 3.0);\r\n /// assert_eq!(rect.y1, -3.0);\r\n /// ```\r\n #[inline]\r\n pub fn ceil(self) -> Rect {\r\n Rect::new(\r\n self.x0.ceil(),\r\n self.y0.ceil(),\r\n self.x1.ceil(),\r\n self.y1.ceil(),\r\n )\r\n }\r\n\r\n /// Returns a new `Rect`,\r\n /// with each coordinate value rounded down to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Rect;\r\n /// let rect = Rect::new(3.3, 3.6, 3.0, -3.1).floor();\r\n /// assert_eq!(rect.x0, 3.0);\r\n /// assert_eq!(rect.y0, 3.0);\r\n /// assert_eq!(rect.x1, 3.0);\r\n /// assert_eq!(rect.y1, -4.0);\r\n /// ```\r\n #[inline]\r\n pub fn floor(self) -> Rect {\r\n Rect::new(\r\n self.x0.floor(),\r\n self.y0.floor(),\r\n self.x1.floor(),\r\n self.y1.floor(),\r\n )\r\n }\r\n\r\n /// Returns a new `Rect`,\r\n /// with each coordinate value rounded away from the center of the `Rect`\r\n /// to the nearest integer, unless they are already an integer.\r\n /// That is to say this function will return the smallest possible `Rect`\r\n /// with integer coordinates that is a superset of `self`.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Rect;\r\n ///\r\n /// // In positive space\r\n /// let rect = Rect::new(3.3, 3.6, 5.6, 4.1).expand();\r\n /// assert_eq!(rect.x0, 3.0);\r\n /// assert_eq!(rect.y0, 3.0);\r\n /// assert_eq!(rect.x1, 6.0);\r\n /// assert_eq!(rect.y1, 5.0);\r\n ///\r\n /// // In both positive and negative space\r\n /// let rect = Rect::new(-3.3, -3.6, 5.6, 4.1).expand();\r\n /// assert_eq!(rect.x0, -4.0);\r\n /// assert_eq!(rect.y0, -4.0);\r\n /// assert_eq!(rect.x1, 6.0);\r\n /// assert_eq!(rect.y1, 5.0);\r\n ///\r\n /// // In negative space\r\n /// let rect = Rect::new(-5.6, -4.1, -3.3, -3.6).expand();\r\n /// assert_eq!(rect.x0, -6.0);\r\n /// assert_eq!(rect.y0, -5.0);\r\n /// assert_eq!(rect.x1, -3.0);\r\n /// assert_eq!(rect.y1, -3.0);\r\n ///\r\n /// // Inverse orientation\r\n /// let rect = Rect::new(5.6, -3.6, 3.3, -4.1).expand();\r\n /// assert_eq!(rect.x0, 6.0);\r\n /// assert_eq!(rect.y0, -3.0);\r\n /// assert_eq!(rect.x1, 3.0);\r\n /// assert_eq!(rect.y1, -5.0);\r\n /// ```\r\n #[inline]\r\n pub fn expand(self) -> Rect {\r\n // The compiler optimizer will remove the if branching.\r\n let (x0, x1) = if self.x0 < self.x1 {\r\n (self.x0.floor(), self.x1.ceil())\r\n } else {\r\n (self.x0.ceil(), self.x1.floor())\r\n };\r\n let (y0, y1) = if self.y0 < self.y1 {\r\n (self.y0.floor(), self.y1.ceil())\r\n } else {\r\n (self.y0.ceil(), self.y1.floor())\r\n };\r\n Rect::new(x0, y0, x1, y1)\r\n }\r\n\r\n /// Returns a new `Rect`,\r\n /// with each coordinate value rounded towards the center of the `Rect`\r\n /// to the nearest integer, unless they are already an integer.\r\n /// That is to say this function will return the biggest possible `Rect`\r\n /// with integer coordinates that is a subset of `self`.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Rect;\r\n ///\r\n /// // In positive space\r\n /// let rect = Rect::new(3.3, 3.6, 5.6, 4.1).trunc();\r\n /// assert_eq!(rect.x0, 4.0);\r\n /// assert_eq!(rect.y0, 4.0);\r\n /// assert_eq!(rect.x1, 5.0);\r\n /// assert_eq!(rect.y1, 4.0);\r\n ///\r\n /// // In both positive and negative space\r\n /// let rect = Rect::new(-3.3, -3.6, 5.6, 4.1).trunc();\r\n /// assert_eq!(rect.x0, -3.0);\r\n /// assert_eq!(rect.y0, -3.0);\r\n /// assert_eq!(rect.x1, 5.0);\r\n /// assert_eq!(rect.y1, 4.0);\r\n ///\r\n /// // In negative space\r\n /// let rect = Rect::new(-5.6, -4.1, -3.3, -3.6).trunc();\r\n /// assert_eq!(rect.x0, -5.0);\r\n /// assert_eq!(rect.y0, -4.0);\r\n /// assert_eq!(rect.x1, -4.0);\r\n /// assert_eq!(rect.y1, -4.0);\r\n ///\r\n /// // Inverse orientation\r\n /// let rect = Rect::new(5.6, -3.6, 3.3, -4.1).trunc();\r\n /// assert_eq!(rect.x0, 5.0);\r\n /// assert_eq!(rect.y0, -4.0);\r\n /// assert_eq!(rect.x1, 4.0);\r\n /// assert_eq!(rect.y1, -4.0);\r\n /// ```\r\n #[inline]\r\n pub fn trunc(self) -> Rect {\r\n // The compiler optimizer will remove the if branching.\r\n let (x0, x1) = if self.x0 < self.x1 {\r\n (self.x0.ceil(), self.x1.floor())\r\n } else {\r\n (self.x0.floor(), self.x1.ceil())\r\n };\r\n let (y0, y1) = if self.y0 < self.y1 {\r\n (self.y0.ceil(), self.y1.floor())\r\n } else {\r\n (self.y0.floor(), self.y1.ceil())\r\n };\r\n Rect::new(x0, y0, x1, y1)\r\n }\r\n\r\n /// Scales the `Rect` by `factor` with respect to the origin (the point `(0, 0)`).\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Rect;\r\n ///\r\n /// let rect = Rect::new(2., 2., 4., 6.).scale_from_origin(2.);\r\n /// assert_eq!(rect.x0, 4.);\r\n /// assert_eq!(rect.x1, 8.);\r\n /// ```\r\n #[inline]\r\n pub fn scale_from_origin(self, factor: f64) -> Rect {\r\n Rect {\r\n x0: self.x0 * factor,\r\n y0: self.y0 * factor,\r\n x1: self.x1 * factor,\r\n y1: self.y1 * factor,\r\n }\r\n }\r\n\r\n /// Creates a new [`RoundedRect`] from this `Rect` and the provided\r\n /// corner radius.\r\n #[inline]\r\n pub fn to_rounded_rect(self, radii: impl Into) -> RoundedRect {\r\n RoundedRect::from_rect(self, radii)\r\n }\r\n\r\n /// Returns the [`Ellipse`] that is bounded by this `Rect`.\r\n #[inline]\r\n pub fn to_ellipse(self) -> Ellipse {\r\n Ellipse::from_rect(self)\r\n }\r\n\r\n /// The aspect ratio of the `Rect`.\r\n ///\r\n /// This is defined as the height divided by the width. It measures the\r\n /// \"squareness\" of the rectangle (a value of `1` is square).\r\n ///\r\n /// If the width is `0` the output will be `sign(y1 - y0) * infinity`.\r\n ///\r\n /// If The width and height are `0`, the result will be `NaN`.\r\n #[inline]\r\n pub fn aspect_ratio(&self) -> f64 {\r\n self.size().aspect_ratio()\r\n }\r\n\r\n /// Returns the largest possible `Rect` that is fully contained in `self`\r\n /// with the given `aspect_ratio`.\r\n ///\r\n /// The aspect ratio is specified fractionally, as `height / width`.\r\n ///\r\n /// The resulting rectangle will be centered if it is smaller than the\r\n /// input rectangle.\r\n ///\r\n /// For the special case where the aspect ratio is `1.0`, the resulting\r\n /// `Rect` will be square.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// # use kurbo::Rect;\r\n /// let outer = Rect::new(0.0, 0.0, 10.0, 20.0);\r\n /// let inner = outer.contained_rect_with_aspect_ratio(1.0);\r\n /// // The new `Rect` is a square centered at the center of `outer`.\r\n /// assert_eq!(inner, Rect::new(0.0, 5.0, 10.0, 15.0));\r\n /// ```\r\n ///\r\n pub fn contained_rect_with_aspect_ratio(&self, aspect_ratio: f64) -> Rect {\r\n let (width, height) = (self.width(), self.height());\r\n let self_aspect = height / width;\r\n\r\n // TODO the parameter `1e-9` was chosen quickly and may not be optimal.\r\n if (self_aspect - aspect_ratio).abs() < 1e-9 {\r\n // short circuit\r\n *self\r\n } else if self_aspect.abs() < aspect_ratio.abs() {\r\n // shrink x to fit\r\n let new_width = height * aspect_ratio.recip();\r\n let gap = (width - new_width) * 0.5;\r\n let x0 = self.x0 + gap;\r\n let x1 = self.x1 - gap;\r\n Rect::new(x0, self.y0, x1, self.y1)\r\n } else {\r\n // shrink y to fit\r\n let new_height = width * aspect_ratio;\r\n let gap = (height - new_height) * 0.5;\r\n let y0 = self.y0 + gap;\r\n let y1 = self.y1 - gap;\r\n Rect::new(self.x0, y0, self.x1, y1)\r\n }\r\n }\r\n\r\n /// Is this rectangle finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n self.x0.is_finite() && self.x1.is_finite() && self.y0.is_finite() && self.y1.is_finite()\r\n }\r\n\r\n /// Is this rectangle NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n self.x0.is_nan() || self.y0.is_nan() || self.x1.is_nan() || self.y1.is_nan()\r\n }\r\n}\r\n\r\nimpl From<(Point, Point)> for Rect {\r\n fn from(points: (Point, Point)) -> Rect {\r\n Rect::from_points(points.0, points.1)\r\n }\r\n}\r\n\r\nimpl From<(Point, Size)> for Rect {\r\n fn from(params: (Point, Size)) -> Rect {\r\n Rect::from_origin_size(params.0, params.1)\r\n }\r\n}\r\n\r\nimpl Add for Rect {\r\n type Output = Rect;\r\n\r\n #[inline]\r\n fn add(self, v: Vec2) -> Rect {\r\n Rect::new(self.x0 + v.x, self.y0 + v.y, self.x1 + v.x, self.y1 + v.y)\r\n }\r\n}\r\n\r\nimpl Sub for Rect {\r\n type Output = Rect;\r\n\r\n #[inline]\r\n fn sub(self, v: Vec2) -> Rect {\r\n Rect::new(self.x0 - v.x, self.y0 - v.y, self.x1 - v.x, self.y1 - v.y)\r\n }\r\n}\r\n\r\nimpl Sub for Rect {\r\n type Output = Insets;\r\n\r\n #[inline]\r\n fn sub(self, other: Rect) -> Insets {\r\n let x0 = other.x0 - self.x0;\r\n let y0 = other.y0 - self.y0;\r\n let x1 = self.x1 - other.x1;\r\n let y1 = self.y1 - other.y1;\r\n Insets { x0, y0, x1, y1 }\r\n }\r\n}\r\n\r\n#[doc(hidden)]\r\npub struct RectPathIter {\r\n rect: Rect,\r\n ix: usize,\r\n}\r\n\r\nimpl Shape for Rect {\r\n type PathElementsIter<'iter> = RectPathIter;\r\n\r\n fn path_elements(&self, _tolerance: f64) -> RectPathIter {\r\n RectPathIter { rect: *self, ix: 0 }\r\n }\r\n\r\n // It's a bit of duplication having both this and the impl method, but\r\n // removing that would require using the trait. We'll leave it for now.\r\n #[inline]\r\n fn area(&self) -> f64 {\r\n Rect::area(self)\r\n }\r\n\r\n #[inline]\r\n fn perimeter(&self, _accuracy: f64) -> f64 {\r\n 2.0 * (self.width().abs() + self.height().abs())\r\n }\r\n\r\n /// Note: this function is carefully designed so that if the plane is\r\n /// tiled with rectangles, the winding number will be nonzero for exactly\r\n /// one of them.\r\n #[inline]\r\n fn winding(&self, pt: Point) -> i32 {\r\n let xmin = self.x0.min(self.x1);\r\n let xmax = self.x0.max(self.x1);\r\n let ymin = self.y0.min(self.y1);\r\n let ymax = self.y0.max(self.y1);\r\n if pt.x >= xmin && pt.x < xmax && pt.y >= ymin && pt.y < ymax {\r\n if (self.x1 > self.x0) ^ (self.y1 > self.y0) {\r\n -1\r\n } else {\r\n 1\r\n }\r\n } else {\r\n 0\r\n }\r\n }\r\n\r\n #[inline]\r\n fn bounding_box(&self) -> Rect {\r\n self.abs()\r\n }\r\n\r\n #[inline]\r\n fn as_rect(&self) -> Option {\r\n Some(*self)\r\n }\r\n\r\n #[inline]\r\n fn contains(&self, pt: Point) -> bool {\r\n self.contains(pt)\r\n }\r\n}\r\n\r\n// This is clockwise in a y-down coordinate system for positive area.\r\nimpl Iterator for RectPathIter {\r\n type Item = PathEl;\r\n\r\n fn next(&mut self) -> Option {\r\n self.ix += 1;\r\n match self.ix {\r\n 1 => Some(PathEl::MoveTo(Point::new(self.rect.x0, self.rect.y0))),\r\n 2 => Some(PathEl::LineTo(Point::new(self.rect.x1, self.rect.y0))),\r\n 3 => Some(PathEl::LineTo(Point::new(self.rect.x1, self.rect.y1))),\r\n 4 => Some(PathEl::LineTo(Point::new(self.rect.x0, self.rect.y1))),\r\n 5 => Some(PathEl::ClosePath),\r\n _ => None,\r\n }\r\n }\r\n}\r\n\r\nimpl fmt::Debug for Rect {\r\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\r\n if f.alternate() {\r\n write!(\r\n f,\r\n \"Rect {{ origin: {:?}, size: {:?} }}\",\r\n self.origin(),\r\n self.size()\r\n )\r\n } else {\r\n write!(\r\n f,\r\n \"Rect {{ x0: {:?}, y0: {:?}, x1: {:?}, y1: {:?} }}\",\r\n self.x0, self.y0, self.x1, self.y1\r\n )\r\n }\r\n }\r\n}\r\n\r\nimpl fmt::Display for Rect {\r\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\r\n write!(f, \"Rect {{ \")?;\r\n fmt::Display::fmt(&self.origin(), f)?;\r\n write!(f, \" \")?;\r\n fmt::Display::fmt(&self.size(), f)?;\r\n write!(f, \" }}\")\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{Point, Rect, Shape};\r\n\r\n fn assert_approx_eq(x: f64, y: f64) {\r\n assert!((x - y).abs() < 1e-7);\r\n }\r\n\r\n #[test]\r\n fn area_sign() {\r\n let r = Rect::new(0.0, 0.0, 10.0, 10.0);\r\n let center = r.center();\r\n assert_approx_eq(r.area(), 100.0);\r\n\r\n assert_eq!(r.winding(center), 1);\r\n\r\n let p = r.to_path(1e-9);\r\n assert_approx_eq(r.area(), p.area());\r\n assert_eq!(r.winding(center), p.winding(center));\r\n\r\n let r_flip = Rect::new(0.0, 10.0, 10.0, 0.0);\r\n assert_approx_eq(r_flip.area(), -100.0);\r\n\r\n assert_eq!(r_flip.winding(Point::new(5.0, 5.0)), -1);\r\n let p_flip = r_flip.to_path(1e-9);\r\n assert_approx_eq(r_flip.area(), p_flip.area());\r\n assert_eq!(r_flip.winding(center), p_flip.winding(center));\r\n }\r\n\r\n #[test]\r\n fn display() {\r\n let r = Rect::from_origin_size((10., 12.23214), (22.222222222, 23.1));\r\n assert_eq!(\r\n format!(\"{r}\"),\r\n \"Rect { (10, 12.23214) (22.222222222×23.1) }\"\r\n );\r\n assert_eq!(format!(\"{r:.2}\"), \"Rect { (10.00, 12.23) (22.22×23.10) }\");\r\n }\r\n\r\n /* TODO uncomment when a (possibly approximate) equality test has been decided on\r\n #[test]\r\n fn rect_from_center_size() {\r\n assert_eq!(\r\n Rect::from_center_size(Point::new(3.0, 2.0), Size::new(2.0, 4.0)),\r\n Rect::new(2.0, 0.0, 4.0, 4.0)\r\n );\r\n }\r\n */\r\n\r\n #[test]\r\n fn contained_rect_with_aspect_ratio() {\r\n fn case(outer: [f64; 4], aspect_ratio: f64, expected: [f64; 4]) {\r\n let outer = Rect::new(outer[0], outer[1], outer[2], outer[3]);\r\n let expected = Rect::new(expected[0], expected[1], expected[2], expected[3]);\r\n assert_eq!(\r\n outer.contained_rect_with_aspect_ratio(aspect_ratio),\r\n expected\r\n );\r\n }\r\n // squares (different point orderings)\r\n case([0.0, 0.0, 10.0, 20.0], 1.0, [0.0, 5.0, 10.0, 15.0]);\r\n case([0.0, 20.0, 10.0, 0.0], 1.0, [0.0, 5.0, 10.0, 15.0]);\r\n case([10.0, 0.0, 0.0, 20.0], 1.0, [10.0, 15.0, 0.0, 5.0]);\r\n case([10.0, 20.0, 0.0, 0.0], 1.0, [10.0, 15.0, 0.0, 5.0]);\r\n // non-square\r\n case([0.0, 0.0, 10.0, 20.0], 0.5, [0.0, 7.5, 10.0, 12.5]);\r\n // same aspect ratio\r\n case([0.0, 0.0, 10.0, 20.0], 2.0, [0.0, 0.0, 10.0, 20.0]);\r\n // negative aspect ratio\r\n case([0.0, 0.0, 10.0, 20.0], -1.0, [0.0, 15.0, 10.0, 5.0]);\r\n // infinite aspect ratio\r\n case([0.0, 0.0, 10.0, 20.0], f64::INFINITY, [5.0, 0.0, 5.0, 20.0]);\r\n // zero aspect ratio\r\n case([0.0, 0.0, 10.0, 20.0], 0.0, [0.0, 10.0, 10.0, 10.0]);\r\n // zero width rect\r\n case([0.0, 0.0, 0.0, 20.0], 1.0, [0.0, 10.0, 0.0, 10.0]);\r\n // many zeros\r\n case([0.0, 0.0, 0.0, 20.0], 0.0, [0.0, 10.0, 0.0, 10.0]);\r\n // everything zero\r\n case([0.0, 0.0, 0.0, 0.0], 0.0, [0.0, 0.0, 0.0, 0.0]);\r\n }\r\n\r\n #[test]\r\n fn aspect_ratio() {\r\n let test = Rect::new(0.0, 0.0, 1.0, 1.0);\r\n assert!((test.aspect_ratio() - 1.0).abs() < 1e-6);\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/rounded_rect.rs", "sha256": "0e96575f7017f05a7507d2024b931234a5815d2d97cfb347afc9db8f677690de", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A rectangle with rounded corners.\r\n\r\nuse core::f64::consts::{FRAC_PI_2, FRAC_PI_4};\r\nuse core::ops::{Add, Sub};\r\n\r\nuse crate::{arc::ArcAppendIter, Arc, PathEl, Point, Rect, RoundedRectRadii, Shape, Size, Vec2};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// A rectangle with equally rounded corners.\r\n///\r\n/// By construction the rounded rectangle will have\r\n/// non-negative dimensions and radii clamped to half size of the rect.\r\n///\r\n/// The easiest way to create a `RoundedRect` is often to create a [`Rect`],\r\n/// and then call [`to_rounded_rect`].\r\n///\r\n/// ```\r\n/// use kurbo::{RoundedRect, RoundedRectRadii};\r\n///\r\n/// // Create a rounded rectangle with a single radius for all corners:\r\n/// RoundedRect::new(0.0, 0.0, 10.0, 10.0, 5.0);\r\n///\r\n/// // Or, specify different radii for each corner, clockwise from the top-left:\r\n/// RoundedRect::new(0.0, 0.0, 10.0, 10.0, (1.0, 2.0, 3.0, 4.0));\r\n/// ```\r\n///\r\n/// [`to_rounded_rect`]: Rect::to_rounded_rect\r\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct RoundedRect {\r\n /// Coordinates of the rectangle.\r\n rect: Rect,\r\n /// Radius of all four corners.\r\n radii: RoundedRectRadii,\r\n}\r\n\r\nimpl RoundedRect {\r\n /// A new rectangle from minimum and maximum coordinates.\r\n ///\r\n /// The result will have non-negative width, height and radii.\r\n #[inline]\r\n pub fn new(\r\n x0: f64,\r\n y0: f64,\r\n x1: f64,\r\n y1: f64,\r\n radii: impl Into,\r\n ) -> RoundedRect {\r\n RoundedRect::from_rect(Rect::new(x0, y0, x1, y1), radii)\r\n }\r\n\r\n /// A new rounded rectangle from a rectangle and corner radii.\r\n ///\r\n /// The result will have non-negative width, height and radii.\r\n ///\r\n /// See also [`Rect::to_rounded_rect`], which offers the same utility.\r\n #[inline]\r\n pub fn from_rect(rect: Rect, radii: impl Into) -> RoundedRect {\r\n let rect = rect.abs();\r\n let shortest_side_length = (rect.width()).min(rect.height());\r\n let radii = radii.into().abs().clamp(shortest_side_length / 2.0);\r\n\r\n RoundedRect { rect, radii }\r\n }\r\n\r\n /// A new rectangle from two [`Point`]s.\r\n ///\r\n /// The result will have non-negative width, height and radius.\r\n #[inline]\r\n pub fn from_points(\r\n p0: impl Into,\r\n p1: impl Into,\r\n radii: impl Into,\r\n ) -> RoundedRect {\r\n Rect::from_points(p0, p1).to_rounded_rect(radii)\r\n }\r\n\r\n /// A new rectangle from origin and size.\r\n ///\r\n /// The result will have non-negative width, height and radius.\r\n #[inline]\r\n pub fn from_origin_size(\r\n origin: impl Into,\r\n size: impl Into,\r\n radii: impl Into,\r\n ) -> RoundedRect {\r\n Rect::from_origin_size(origin, size).to_rounded_rect(radii)\r\n }\r\n\r\n /// The width of the rectangle.\r\n #[inline]\r\n pub fn width(&self) -> f64 {\r\n self.rect.width()\r\n }\r\n\r\n /// The height of the rectangle.\r\n #[inline]\r\n pub fn height(&self) -> f64 {\r\n self.rect.height()\r\n }\r\n\r\n /// Radii of the rounded corners.\r\n #[inline]\r\n pub fn radii(&self) -> RoundedRectRadii {\r\n self.radii\r\n }\r\n\r\n /// The (non-rounded) rectangle.\r\n pub fn rect(&self) -> Rect {\r\n self.rect\r\n }\r\n\r\n /// The origin of the rectangle.\r\n ///\r\n /// This is the top left corner in a y-down space.\r\n #[inline]\r\n pub fn origin(&self) -> Point {\r\n self.rect.origin()\r\n }\r\n\r\n /// The center point of the rectangle.\r\n #[inline]\r\n pub fn center(&self) -> Point {\r\n self.rect.center()\r\n }\r\n\r\n /// Is this rounded rectangle finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n self.rect.is_finite() && self.radii.is_finite()\r\n }\r\n\r\n /// Is this rounded rectangle NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n self.rect.is_nan() || self.radii.is_nan()\r\n }\r\n}\r\n\r\n#[doc(hidden)]\r\npub struct RoundedRectPathIter {\r\n idx: usize,\r\n rect: RectPathIter,\r\n arcs: [ArcAppendIter; 4],\r\n}\r\n\r\nimpl Shape for RoundedRect {\r\n type PathElementsIter<'iter> = RoundedRectPathIter;\r\n\r\n fn path_elements(&self, tolerance: f64) -> RoundedRectPathIter {\r\n let radii = self.radii();\r\n\r\n let build_arc_iter = |i, center, ellipse_radii| {\r\n let arc = Arc {\r\n center,\r\n radii: ellipse_radii,\r\n start_angle: FRAC_PI_2 * i as f64,\r\n sweep_angle: FRAC_PI_2,\r\n x_rotation: 0.0,\r\n };\r\n arc.append_iter(tolerance)\r\n };\r\n\r\n // Note: order follows the rectangle path iterator.\r\n let arcs = [\r\n build_arc_iter(\r\n 2,\r\n Point {\r\n x: self.rect.x0 + radii.top_left,\r\n y: self.rect.y0 + radii.top_left,\r\n },\r\n Vec2 {\r\n x: radii.top_left,\r\n y: radii.top_left,\r\n },\r\n ),\r\n build_arc_iter(\r\n 3,\r\n Point {\r\n x: self.rect.x1 - radii.top_right,\r\n y: self.rect.y0 + radii.top_right,\r\n },\r\n Vec2 {\r\n x: radii.top_right,\r\n y: radii.top_right,\r\n },\r\n ),\r\n build_arc_iter(\r\n 0,\r\n Point {\r\n x: self.rect.x1 - radii.bottom_right,\r\n y: self.rect.y1 - radii.bottom_right,\r\n },\r\n Vec2 {\r\n x: radii.bottom_right,\r\n y: radii.bottom_right,\r\n },\r\n ),\r\n build_arc_iter(\r\n 1,\r\n Point {\r\n x: self.rect.x0 + radii.bottom_left,\r\n y: self.rect.y1 - radii.bottom_left,\r\n },\r\n Vec2 {\r\n x: radii.bottom_left,\r\n y: radii.bottom_left,\r\n },\r\n ),\r\n ];\r\n\r\n let rect = RectPathIter {\r\n rect: self.rect,\r\n ix: 0,\r\n radii,\r\n };\r\n\r\n RoundedRectPathIter { idx: 0, rect, arcs }\r\n }\r\n\r\n #[inline]\r\n fn area(&self) -> f64 {\r\n // A corner is a quarter-circle, i.e.\r\n // .............#\r\n // . ######\r\n // . #########\r\n // . ###########\r\n // . ############\r\n // .#############\r\n // ##############\r\n // |-----r------|\r\n // For each corner, we need to subtract the square that bounds this\r\n // quarter-circle, and add back in the area of quarter circle.\r\n\r\n let radii = self.radii();\r\n\r\n // Start with the area of the bounding rectangle. For each corner,\r\n // subtract the area of the corner under the quarter-circle, and add\r\n // back the area of the quarter-circle.\r\n self.rect.area()\r\n + [\r\n radii.top_left,\r\n radii.top_right,\r\n radii.bottom_right,\r\n radii.bottom_left,\r\n ]\r\n .iter()\r\n .map(|radius| (FRAC_PI_4 - 1.0) * radius * radius)\r\n .sum::()\r\n }\r\n\r\n #[inline]\r\n fn perimeter(&self, _accuracy: f64) -> f64 {\r\n // A corner is a quarter-circle, i.e.\r\n // .............#\r\n // . #\r\n // . #\r\n // . #\r\n // . #\r\n // .#\r\n // #\r\n // |-----r------|\r\n // If we start with the bounding rectangle, then subtract 2r (the\r\n // straight edge outside the circle) and add 1/4 * pi * (2r) (the\r\n // perimeter of the quarter-circle) for each corner with radius r, we\r\n // get the perimeter of the shape.\r\n\r\n let radii = self.radii();\r\n\r\n // Start with the full perimeter. For each corner, subtract the\r\n // border surrounding the rounded corner and add the quarter-circle\r\n // perimeter.\r\n self.rect.perimeter(1.0)\r\n + ([\r\n radii.top_left,\r\n radii.top_right,\r\n radii.bottom_right,\r\n radii.bottom_left,\r\n ])\r\n .iter()\r\n .map(|radius| (-2.0 + FRAC_PI_2) * radius)\r\n .sum::()\r\n }\r\n\r\n #[inline]\r\n fn winding(&self, mut pt: Point) -> i32 {\r\n let center = self.center();\r\n\r\n // 1. Translate the point relative to the center of the rectangle.\r\n pt.x -= center.x;\r\n pt.y -= center.y;\r\n\r\n // 2. Pick a radius value to use based on which quadrant the point is\r\n // in.\r\n let radii = self.radii();\r\n let radius = match pt {\r\n pt if pt.x < 0.0 && pt.y < 0.0 => radii.top_left,\r\n pt if pt.x >= 0.0 && pt.y < 0.0 => radii.top_right,\r\n pt if pt.x >= 0.0 && pt.y >= 0.0 => radii.bottom_right,\r\n pt if pt.x < 0.0 && pt.y >= 0.0 => radii.bottom_left,\r\n _ => 0.0,\r\n };\r\n\r\n // 3. This is the width and height of a rectangle with one corner at\r\n // the center of the rounded rectangle, and another corner at the\r\n // center of the relevant corner circle.\r\n let inside_half_width = (self.width() / 2.0 - radius).max(0.0);\r\n let inside_half_height = (self.height() / 2.0 - radius).max(0.0);\r\n\r\n // 4. Three things are happening here.\r\n //\r\n // First, the x- and y-values are being reflected into the positive\r\n // (bottom-right quadrant). The radius has already been determined,\r\n // so it doesn't matter what quadrant is used.\r\n //\r\n // After reflecting, the points are clamped so that their x- and y-\r\n // values can't be lower than the x- and y- values of the center of\r\n // the corner circle, and the coordinate system is transformed\r\n // again, putting (0, 0) at the center of the corner circle.\r\n let px = (pt.x.abs() - inside_half_width).max(0.0);\r\n let py = (pt.y.abs() - inside_half_height).max(0.0);\r\n\r\n // 5. The transforms above clamp all input points such that they will\r\n // be inside the rounded rectangle if the corresponding output point\r\n // (px, py) is inside a circle centered around the origin with the\r\n // given radius.\r\n let inside = px * px + py * py <= radius * radius;\r\n if inside {\r\n 1\r\n } else {\r\n 0\r\n }\r\n }\r\n\r\n #[inline]\r\n fn bounding_box(&self) -> Rect {\r\n self.rect.bounding_box()\r\n }\r\n\r\n #[inline]\r\n fn as_rounded_rect(&self) -> Option {\r\n Some(*self)\r\n }\r\n}\r\n\r\nstruct RectPathIter {\r\n rect: Rect,\r\n radii: RoundedRectRadii,\r\n ix: usize,\r\n}\r\n\r\n// This is clockwise in a y-down coordinate system for positive area.\r\nimpl Iterator for RectPathIter {\r\n type Item = PathEl;\r\n\r\n fn next(&mut self) -> Option {\r\n self.ix += 1;\r\n match self.ix {\r\n 1 => Some(PathEl::MoveTo(Point::new(\r\n self.rect.x0,\r\n self.rect.y0 + self.radii.top_left,\r\n ))),\r\n 2 => Some(PathEl::LineTo(Point::new(\r\n self.rect.x1 - self.radii.top_right,\r\n self.rect.y0,\r\n ))),\r\n 3 => Some(PathEl::LineTo(Point::new(\r\n self.rect.x1,\r\n self.rect.y1 - self.radii.bottom_right,\r\n ))),\r\n 4 => Some(PathEl::LineTo(Point::new(\r\n self.rect.x0 + self.radii.bottom_left,\r\n self.rect.y1,\r\n ))),\r\n 5 => Some(PathEl::ClosePath),\r\n _ => None,\r\n }\r\n }\r\n}\r\n\r\n// This is clockwise in a y-down coordinate system for positive area.\r\nimpl Iterator for RoundedRectPathIter {\r\n type Item = PathEl;\r\n\r\n fn next(&mut self) -> Option {\r\n if self.idx > 4 {\r\n return None;\r\n }\r\n\r\n // Iterate between rectangle and arc iterators.\r\n // Rect iterator will start and end the path.\r\n\r\n // Initial point set by the rect iterator\r\n if self.idx == 0 {\r\n self.idx += 1;\r\n return self.rect.next();\r\n }\r\n\r\n // Generate the arc curve elements.\r\n // If we reached the end of the arc, add a line towards next arc (rect iterator).\r\n match self.arcs[self.idx - 1].next() {\r\n Some(elem) => Some(elem),\r\n None => {\r\n self.idx += 1;\r\n self.rect.next()\r\n }\r\n }\r\n }\r\n}\r\n\r\nimpl Add for RoundedRect {\r\n type Output = RoundedRect;\r\n\r\n #[inline]\r\n fn add(self, v: Vec2) -> RoundedRect {\r\n RoundedRect::from_rect(self.rect + v, self.radii)\r\n }\r\n}\r\n\r\nimpl Sub for RoundedRect {\r\n type Output = RoundedRect;\r\n\r\n #[inline]\r\n fn sub(self, v: Vec2) -> RoundedRect {\r\n RoundedRect::from_rect(self.rect - v, self.radii)\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{Circle, Point, Rect, RoundedRect, Shape};\r\n\r\n #[test]\r\n fn area() {\r\n let epsilon = 1e-9;\r\n\r\n // Extremum: 0.0 radius corner -> rectangle\r\n let rect = Rect::new(0.0, 0.0, 100.0, 100.0);\r\n let rounded_rect = RoundedRect::new(0.0, 0.0, 100.0, 100.0, 0.0);\r\n assert!((rect.area() - rounded_rect.area()).abs() < epsilon);\r\n\r\n // Extremum: half-size radius corner -> circle\r\n let circle = Circle::new((0.0, 0.0), 50.0);\r\n let rounded_rect = RoundedRect::new(0.0, 0.0, 100.0, 100.0, 50.0);\r\n assert!((circle.area() - rounded_rect.area()).abs() < epsilon);\r\n }\r\n\r\n #[test]\r\n fn winding() {\r\n let rect = RoundedRect::new(-5.0, -5.0, 10.0, 20.0, (5.0, 5.0, 5.0, 0.0));\r\n assert_eq!(rect.winding(Point::new(0.0, 0.0)), 1);\r\n assert_eq!(rect.winding(Point::new(-5.0, 0.0)), 1); // left edge\r\n assert_eq!(rect.winding(Point::new(0.0, 20.0)), 1); // bottom edge\r\n assert_eq!(rect.winding(Point::new(10.0, 20.0)), 0); // bottom-right corner\r\n assert_eq!(rect.winding(Point::new(-5.0, 20.0)), 1); // bottom-left corner (has a radius of 0)\r\n assert_eq!(rect.winding(Point::new(-10.0, 0.0)), 0);\r\n\r\n let rect = RoundedRect::new(-10.0, -20.0, 10.0, 20.0, 0.0); // rectangle\r\n assert_eq!(rect.winding(Point::new(10.0, 20.0)), 1); // bottom-right corner\r\n }\r\n\r\n #[test]\r\n fn bez_conversion() {\r\n let rect = RoundedRect::new(-5.0, -5.0, 10.0, 20.0, 5.0);\r\n let p = rect.to_path(1e-9);\r\n // Note: could be more systematic about tolerance tightness.\r\n let epsilon = 1e-7;\r\n assert!((rect.area() - p.area()).abs() < epsilon);\r\n assert_eq!(p.winding(Point::new(0.0, 0.0)), 1);\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/rounded_rect_radii.rs", "sha256": "4f54f9c3847ad098bd524d36a241f150f94f660abaca5e429e65c610ca1fd88f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2021 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A description of the radii for each corner of a rounded rectangle.\r\n\r\nuse core::convert::From;\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// Radii for each corner of a rounded rectangle.\r\n///\r\n/// The use of `top` as in `top_left` assumes a y-down coordinate space. Piet\r\n/// (and Druid by extension) uses a y-down coordinate space, but Kurbo also\r\n/// supports a y-up coordinate space, in which case `top_left` would actually\r\n/// refer to the bottom-left corner, and vice versa. Top may not always\r\n/// actually be the top, but `top` corners will always have a smaller y-value\r\n/// than `bottom` corners.\r\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct RoundedRectRadii {\r\n /// The radius of the top-left corner.\r\n pub top_left: f64,\r\n /// The radius of the top-right corner.\r\n pub top_right: f64,\r\n /// The radius of the bottom-right corner.\r\n pub bottom_right: f64,\r\n /// The radius of the bottom-left corner.\r\n pub bottom_left: f64,\r\n}\r\n\r\nimpl RoundedRectRadii {\r\n /// Create a new RoundedRectRadii. This function takes radius values for\r\n /// the four corners. The argument order is \"top_left, top_right,\r\n /// bottom_right, bottom_left\", or clockwise starting from top_left.\r\n pub const fn new(top_left: f64, top_right: f64, bottom_right: f64, bottom_left: f64) -> Self {\r\n RoundedRectRadii {\r\n top_left,\r\n top_right,\r\n bottom_right,\r\n bottom_left,\r\n }\r\n }\r\n\r\n /// Create a new RoundedRectRadii from a single radius. The `radius`\r\n /// argument will be set as the radius for all four corners.\r\n pub const fn from_single_radius(radius: f64) -> Self {\r\n RoundedRectRadii {\r\n top_left: radius,\r\n top_right: radius,\r\n bottom_right: radius,\r\n bottom_left: radius,\r\n }\r\n }\r\n\r\n /// Takes the absolute value of all corner radii.\r\n pub fn abs(&self) -> Self {\r\n RoundedRectRadii::new(\r\n self.top_left.abs(),\r\n self.top_right.abs(),\r\n self.bottom_right.abs(),\r\n self.bottom_left.abs(),\r\n )\r\n }\r\n\r\n /// For each corner, takes the min of that corner's radius and `max`.\r\n pub fn clamp(&self, max: f64) -> Self {\r\n RoundedRectRadii::new(\r\n self.top_left.min(max),\r\n self.top_right.min(max),\r\n self.bottom_right.min(max),\r\n self.bottom_left.min(max),\r\n )\r\n }\r\n\r\n /// Returns true if all radius values are finite.\r\n pub fn is_finite(&self) -> bool {\r\n self.top_left.is_finite()\r\n && self.top_right.is_finite()\r\n && self.bottom_right.is_finite()\r\n && self.bottom_left.is_finite()\r\n }\r\n\r\n /// Returns true if any corner radius value is NaN.\r\n pub fn is_nan(&self) -> bool {\r\n self.top_left.is_nan()\r\n || self.top_right.is_nan()\r\n || self.bottom_right.is_nan()\r\n || self.bottom_left.is_nan()\r\n }\r\n\r\n /// If all radii are equal, returns the value of the radii. Otherwise,\r\n /// returns `None`.\r\n pub fn as_single_radius(&self) -> Option {\r\n let epsilon = 1e-9;\r\n\r\n if (self.top_left - self.top_right).abs() < epsilon\r\n && (self.top_right - self.bottom_right).abs() < epsilon\r\n && (self.bottom_right - self.bottom_left).abs() < epsilon\r\n {\r\n Some(self.top_left)\r\n } else {\r\n None\r\n }\r\n }\r\n}\r\n\r\nimpl From for RoundedRectRadii {\r\n fn from(radius: f64) -> Self {\r\n RoundedRectRadii::from_single_radius(radius)\r\n }\r\n}\r\n\r\nimpl From<(f64, f64, f64, f64)> for RoundedRectRadii {\r\n fn from(radii: (f64, f64, f64, f64)) -> Self {\r\n RoundedRectRadii::new(radii.0, radii.1, radii.2, radii.3)\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/shape.rs", "sha256": "719f94b4599073527910d0fabafdc6109b16c24cf46506abb57ccfb5243f53af", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A generic trait for shapes.\r\n\r\nuse crate::{segments, BezPath, Circle, Line, PathEl, Point, Rect, RoundedRect, Segments};\r\n\r\n/// A generic trait for open and closed shapes.\r\n///\r\n/// This trait provides conversion from shapes to [`BezPath`]s, as well as\r\n/// general geometry functionality like computing [`area`], [`bounding_box`]es,\r\n/// and [`winding`] number.\r\n///\r\n/// [`area`]: Shape::area\r\n/// [`bounding_box`]: Shape::bounding_box\r\n/// [`winding`]: Shape::winding\r\npub trait Shape: Sized {\r\n /// The iterator returned by the [`path_elements`] method.\r\n ///\r\n /// [`path_elements`]: Shape::path_elements\r\n type PathElementsIter<'iter>: Iterator + 'iter\r\n where\r\n Self: 'iter;\r\n\r\n /// Returns an iterator over this shape expressed as [`PathEl`]s;\r\n /// that is, as Bézier path _elements_.\r\n ///\r\n /// All shapes can be represented as Béziers, but in many situations\r\n /// (such as when interfacing with a platform drawing API) there are more\r\n /// efficient native types for specific concrete shapes. In this case,\r\n /// the user should exhaust the `as_` methods ([`as_rect`], [`as_line`], etc)\r\n /// before converting to a [`BezPath`], as those are likely to be more\r\n /// efficient.\r\n ///\r\n /// In many cases, shapes are able to iterate their elements without\r\n /// allocating; however creating a [`BezPath`] object always allocates.\r\n /// If you need an owned [`BezPath`] you can use [`to_path`] instead.\r\n ///\r\n /// # Tolerance\r\n ///\r\n /// The `tolerance` parameter controls the accuracy of\r\n /// conversion of geometric primitives to Bézier curves, as\r\n /// curves such as circles cannot be represented exactly but\r\n /// only approximated. For drawing as in UI elements, a value\r\n /// of 0.1 is appropriate, as it is unlikely to be visible to\r\n /// the eye. For scientific applications, a smaller value\r\n /// might be appropriate. Note that in general the number of\r\n /// cubic Bézier segments scales as `tolerance ^ (-1/6)`.\r\n ///\r\n /// [`as_rect`]: Shape::as_rect\r\n /// [`as_line`]: Shape::as_line\r\n /// [`to_path`]: Shape::to_path\r\n fn path_elements(&self, tolerance: f64) -> Self::PathElementsIter<'_>;\r\n\r\n /// Convert to a Bézier path.\r\n ///\r\n /// This always allocates. It is appropriate when both the source\r\n /// shape and the resulting path are to be retained.\r\n ///\r\n /// If you only need to iterate the elements (such as to convert them to\r\n /// drawing commands for a given 2D graphics API) you should prefer\r\n /// [`path_elements`], which can avoid allocating where possible.\r\n ///\r\n /// The `tolerance` parameter is the same as for [`path_elements`].\r\n ///\r\n /// [`path_elements`]: Shape::path_elements\r\n fn to_path(&self, tolerance: f64) -> BezPath {\r\n self.path_elements(tolerance).collect()\r\n }\r\n\r\n #[deprecated(since = \"0.7.0\", note = \"Use path_elements instead\")]\r\n #[doc(hidden)]\r\n fn to_bez_path(&self, tolerance: f64) -> Self::PathElementsIter<'_> {\r\n self.path_elements(tolerance)\r\n }\r\n /// Convert into a Bézier path.\r\n ///\r\n /// This allocates in the general case, but is zero-cost if the\r\n /// shape is already a [`BezPath`].\r\n ///\r\n /// The `tolerance` parameter is the same as for [`path_elements()`].\r\n ///\r\n /// [`path_elements()`]: Shape::path_elements\r\n fn into_path(self, tolerance: f64) -> BezPath {\r\n self.to_path(tolerance)\r\n }\r\n\r\n #[deprecated(since = \"0.7.0\", note = \"Use into_path instead\")]\r\n #[doc(hidden)]\r\n fn into_bez_path(self, tolerance: f64) -> BezPath {\r\n self.into_path(tolerance)\r\n }\r\n\r\n /// Returns an iterator over this shape expressed as Bézier path\r\n /// _segments_ ([`PathSeg`]s).\r\n ///\r\n /// The allocation behaviour and `tolerance` parameter are the\r\n /// same as for [`path_elements()`]\r\n ///\r\n /// [`PathSeg`]: crate::PathSeg\r\n /// [`path_elements()`]: Shape::path_elements\r\n fn path_segments(&self, tolerance: f64) -> Segments> {\r\n segments(self.path_elements(tolerance))\r\n }\r\n\r\n /// Signed area.\r\n ///\r\n /// This method only produces meaningful results with closed shapes.\r\n ///\r\n /// The convention for positive area is that y increases when x is\r\n /// positive. Thus, it is clockwise when down is increasing y (the\r\n /// usual convention for graphics), and anticlockwise when\r\n /// up is increasing y (the usual convention for math).\r\n fn area(&self) -> f64;\r\n\r\n /// Total length of perimeter.\r\n //FIXME: document the accuracy param\r\n fn perimeter(&self, accuracy: f64) -> f64;\r\n\r\n /// The [winding number] of a point.\r\n ///\r\n /// This method only produces meaningful results with closed shapes.\r\n ///\r\n /// The sign of the winding number is consistent with that of [`area`],\r\n /// meaning it is +1 when the point is inside a positive area shape\r\n /// and -1 when it is inside a negative area shape. Of course, greater\r\n /// magnitude values are also possible when the shape is more complex.\r\n ///\r\n /// [`area`]: Shape::area\r\n /// [winding number]: https://mathworld.wolfram.com/ContourWindingNumber.html\r\n fn winding(&self, pt: Point) -> i32;\r\n\r\n /// Returns `true` if the [`Point`] is inside this shape.\r\n ///\r\n /// This is only meaningful for closed shapes.\r\n fn contains(&self, pt: Point) -> bool {\r\n self.winding(pt) != 0\r\n }\r\n\r\n /// The smallest rectangle that encloses the shape.\r\n fn bounding_box(&self) -> Rect;\r\n\r\n /// If the shape is a line, make it available.\r\n fn as_line(&self) -> Option {\r\n None\r\n }\r\n\r\n /// If the shape is a rectangle, make it available.\r\n fn as_rect(&self) -> Option {\r\n None\r\n }\r\n\r\n /// If the shape is a rounded rectangle, make it available.\r\n fn as_rounded_rect(&self) -> Option {\r\n None\r\n }\r\n\r\n /// If the shape is a circle, make it available.\r\n fn as_circle(&self) -> Option {\r\n None\r\n }\r\n\r\n /// If the shape is stored as a slice of path elements, make\r\n /// that available.\r\n ///\r\n /// Note: when GAT's land, a method like `path_elements` would be\r\n /// able to iterate through the slice with no extra allocation,\r\n /// without making any assumption that storage is contiguous.\r\n fn as_path_slice(&self) -> Option<&[PathEl]> {\r\n None\r\n }\r\n}\r\n\r\n/// Blanket implementation so `impl Shape` will accept owned or reference.\r\nimpl<'a, T: Shape> Shape for &'a T {\r\n type PathElementsIter<'iter>\r\n\r\n = T::PathElementsIter<'iter> where T: 'iter, 'a: 'iter;\r\n\r\n fn path_elements(&self, tolerance: f64) -> Self::PathElementsIter<'_> {\r\n (*self).path_elements(tolerance)\r\n }\r\n\r\n fn to_path(&self, tolerance: f64) -> BezPath {\r\n (*self).to_path(tolerance)\r\n }\r\n\r\n fn path_segments(&self, tolerance: f64) -> Segments> {\r\n (*self).path_segments(tolerance)\r\n }\r\n\r\n fn area(&self) -> f64 {\r\n (*self).area()\r\n }\r\n\r\n fn perimeter(&self, accuracy: f64) -> f64 {\r\n (*self).perimeter(accuracy)\r\n }\r\n\r\n fn winding(&self, pt: Point) -> i32 {\r\n (*self).winding(pt)\r\n }\r\n\r\n fn bounding_box(&self) -> Rect {\r\n (*self).bounding_box()\r\n }\r\n\r\n fn as_line(&self) -> Option {\r\n (*self).as_line()\r\n }\r\n\r\n fn as_rect(&self) -> Option {\r\n (*self).as_rect()\r\n }\r\n\r\n fn as_rounded_rect(&self) -> Option {\r\n (*self).as_rounded_rect()\r\n }\r\n\r\n fn as_circle(&self) -> Option {\r\n (*self).as_circle()\r\n }\r\n\r\n fn as_path_slice(&self) -> Option<&[PathEl]> {\r\n (*self).as_path_slice()\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/simplify.rs", "sha256": "c95e2b786f74b1ec809d5cd5fca3132dde07501ef7e6c76b99451978fbf954fd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! Simplification of a Bézier path.\r\n//!\r\n//! This module is currently experimental.\r\n//!\r\n//! The methods in this module create a `SimplifyBezPath` object, which can then\r\n//! be fed to [`fit_to_bezpath`] or [`fit_to_bezpath_opt`] depending on the degree\r\n//! of optimization desired.\r\n//!\r\n//! The implementation uses a number of techniques to achieve high performance and\r\n//! accuracy. The parameter (generally written `t`) evenly divides the curve segments\r\n//! in the original, so sampling can be done in constant time. The derivatives are\r\n//! computed analytically, as that is straightforward with Béziers.\r\n//!\r\n//! The areas and moments are computed analytically (using Green's theorem), and\r\n//! the prefix sum is stored. Thus, it is possible to analytically compute the area\r\n//! and moment of any subdivision of the curve, also in constant time, by taking\r\n//! the difference of two stored prefix sum values, then fixing up the subsegments.\r\n//!\r\n//! A current limitation (hoped to be addressed in the future) is that non-regular\r\n//! cubic segments may have tangents computed incorrectly. This can easily happen,\r\n//! for example when setting a control point equal to an endpoint.\r\n//!\r\n//! In addition, this method does not report corners (adjoining segments where the\r\n//! tangents are not continuous). It is not clear whether it's best to handle such\r\n//! cases here, or in client code.\r\n//!\r\n//! [`fit_to_bezpath`]: crate::fit_to_bezpath\r\n//! [`fit_to_bezpath_opt`]: crate::fit_to_bezpath_opt\r\n\r\nuse alloc::vec::Vec;\r\n\r\nuse core::ops::Range;\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\nuse crate::{\r\n fit_to_bezpath, fit_to_bezpath_opt, BezPath, CubicBez, CurveFitSample, Line, ParamCurve,\r\n ParamCurveDeriv, ParamCurveFit, PathEl, PathSeg, Point, QuadBez, Vec2,\r\n};\r\n\r\n/// A Bézier path which has been prepared for simplification.\r\n///\r\n/// See the [module-level documentation] for a bit more discussion of the approach,\r\n/// and how this struct is to be used.\r\n///\r\n/// [module-level documentation]: crate::simplify\r\npub struct SimplifyBezPath(Vec);\r\n\r\nstruct SimplifyCubic {\r\n c: CubicBez,\r\n // The inclusive prefix sum of the moment integrals\r\n moments: (f64, f64, f64),\r\n}\r\n\r\n/// Options for simplifying paths.\r\npub struct SimplifyOptions {\r\n /// The tangent of the minimum angle below which the path is considered smooth.\r\n angle_thresh: f64,\r\n opt_level: SimplifyOptLevel,\r\n}\r\n\r\n/// Optimization level for simplification.\r\npub enum SimplifyOptLevel {\r\n /// Subdivide; faster but not as optimized results.\r\n Subdivide,\r\n /// Optimize subdivision points.\r\n Optimize,\r\n}\r\n\r\nimpl Default for SimplifyOptions {\r\n fn default() -> Self {\r\n let opt_level = SimplifyOptLevel::Subdivide;\r\n SimplifyOptions {\r\n angle_thresh: 1e-3,\r\n opt_level,\r\n }\r\n }\r\n}\r\n\r\n#[doc(hidden)]\r\n/// Compute moment integrals.\r\n///\r\n/// This is exposed for testing purposes but is an internal detail. We can\r\n/// add to the public, documented interface if there is a use case.\r\npub fn moment_integrals(c: CubicBez) -> (f64, f64, f64) {\r\n let (x0, y0) = (c.p0.x, c.p0.y);\r\n let (x1, y1) = (c.p1.x - x0, c.p1.y - y0);\r\n let (x2, y2) = (c.p2.x - x0, c.p2.y - y0);\r\n let (x3, y3) = (c.p3.x - x0, c.p3.y - y0);\r\n\r\n let r0 = 3. * x1;\r\n let r1 = 3. * y1;\r\n let r2 = x2 * y3;\r\n let r3 = x3 * y2;\r\n let r4 = x3 * y3;\r\n let r5 = 27. * y1;\r\n let r6 = x1 * x2;\r\n let r7 = 27. * y2;\r\n let r8 = 45. * r2;\r\n let r9 = 18. * x3;\r\n let r10 = x1 * y1;\r\n let r11 = 30. * x1;\r\n let r12 = 45. * x3;\r\n let r13 = x2 * y1;\r\n let r14 = 45. * r3;\r\n let r15 = x1.powi(2);\r\n let r16 = 18. * y3;\r\n let r17 = x2.powi(2);\r\n let r18 = 45. * y3;\r\n let r19 = x3.powi(2);\r\n let r20 = 30. * y1;\r\n let r21 = y2.powi(2);\r\n let r22 = y3.powi(2);\r\n let r23 = y1.powi(2);\r\n let a = -r0 * y2 - r0 * y3 + r1 * x2 + r1 * x3 - 6. * r2 + 6. * r3 + 10. * r4;\r\n\r\n // Scale and add chord\r\n let lift = x3 * y0;\r\n let area = a * 0.05 + lift;\r\n let x = r10 * r9 - r11 * r4 + r12 * r13 + r14 * x2 - r15 * r16 - r15 * r7 - r17 * r18\r\n + r17 * r5\r\n + r19 * r20\r\n + 105. * r19 * y2\r\n + 280. * r19 * y3\r\n - 105. * r2 * x3\r\n + r5 * r6\r\n - r6 * r7\r\n - r8 * x1;\r\n let y = -r10 * r16 - r10 * r7 - r11 * r22 + r12 * r21 + r13 * r7 + r14 * y1 - r18 * x1 * y2\r\n + r20 * r4\r\n - 27. * r21 * x1\r\n - 105. * r22 * x2\r\n + 140. * r22 * x3\r\n + r23 * r9\r\n + 27. * r23 * x2\r\n + 105. * r3 * y3\r\n - r8 * y2;\r\n\r\n let mx = x * (1. / 840.) + x0 * area + 0.5 * x3 * lift;\r\n let my = y * (1. / 420.) + y0 * a * 0.1 + y0 * lift;\r\n\r\n (area, mx, my)\r\n}\r\n\r\nimpl SimplifyBezPath {\r\n /// Set up a new Bézier path for simplification.\r\n ///\r\n /// Currently this is not dealing with discontinuities at all, but it\r\n /// could be extended to do so.\r\n pub fn new(path: impl IntoIterator) -> Self {\r\n let (mut a, mut x, mut y) = (0.0, 0.0, 0.0);\r\n let els = crate::segments(path)\r\n .map(|seg| {\r\n let c = seg.to_cubic();\r\n let (ai, xi, yi) = moment_integrals(c);\r\n a += ai;\r\n x += xi;\r\n y += yi;\r\n SimplifyCubic {\r\n c,\r\n moments: (a, x, y),\r\n }\r\n })\r\n .collect();\r\n SimplifyBezPath(els)\r\n }\r\n\r\n /// Resolve a `t` value to a cubic.\r\n ///\r\n /// Also return the resulting `t` value for the selected cubic.\r\n fn scale(&self, t: f64) -> (usize, f64) {\r\n let t_scale = t * self.0.len() as f64;\r\n let t_floor = t_scale.floor();\r\n (t_floor as usize, t_scale - t_floor)\r\n }\r\n\r\n fn moment_integrals(&self, i: usize, range: Range) -> (f64, f64, f64) {\r\n if range.end == range.start {\r\n (0.0, 0.0, 0.0)\r\n } else {\r\n moment_integrals(self.0[i].c.subsegment(range))\r\n }\r\n }\r\n}\r\n\r\nimpl ParamCurveFit for SimplifyBezPath {\r\n fn sample_pt_deriv(&self, t: f64) -> (Point, Vec2) {\r\n let (mut i, mut t0) = self.scale(t);\r\n let n = self.0.len();\r\n if i == n {\r\n i -= 1;\r\n t0 = 1.0;\r\n }\r\n let c = self.0[i].c;\r\n (c.eval(t0), c.deriv().eval(t0).to_vec2() * n as f64)\r\n }\r\n\r\n fn sample_pt_tangent(&self, t: f64, _: f64) -> CurveFitSample {\r\n let (mut i, mut t0) = self.scale(t);\r\n if i == self.0.len() {\r\n i -= 1;\r\n t0 = 1.0;\r\n }\r\n let c = self.0[i].c;\r\n let p = c.eval(t0);\r\n let tangent = c.deriv().eval(t0).to_vec2();\r\n CurveFitSample { p, tangent }\r\n }\r\n\r\n // We could use the default implementation, but provide our own, mostly\r\n // because it is possible to efficiently provide an analytically accurate\r\n // answer.\r\n fn moment_integrals(&self, range: Range) -> (f64, f64, f64) {\r\n let (i0, t0) = self.scale(range.start);\r\n let (i1, t1) = self.scale(range.end);\r\n if i0 == i1 {\r\n self.moment_integrals(i0, t0..t1)\r\n } else {\r\n let (a0, x0, y0) = self.moment_integrals(i0, t0..1.0);\r\n let (a1, x1, y1) = self.moment_integrals(i1, 0.0..t1);\r\n let (mut a, mut x, mut y) = (a0 + a1, x0 + x1, y0 + y1);\r\n if i1 > i0 + 1 {\r\n let (a2, x2, y2) = self.0[i0].moments;\r\n let (a3, x3, y3) = self.0[i1 - 1].moments;\r\n a += a3 - a2;\r\n x += x3 - x2;\r\n y += y3 - y2;\r\n }\r\n (a, x, y)\r\n }\r\n }\r\n\r\n fn break_cusp(&self, _: Range) -> Option {\r\n None\r\n }\r\n}\r\n\r\n#[derive(Default)]\r\nstruct SimplifyState {\r\n queue: BezPath,\r\n result: BezPath,\r\n needs_moveto: bool,\r\n}\r\n\r\nimpl SimplifyState {\r\n fn add_seg(&mut self, seg: PathSeg) {\r\n if self.queue.is_empty() {\r\n self.queue.move_to(seg.start());\r\n }\r\n match seg {\r\n PathSeg::Quad(q) => self.queue.quad_to(q.p1, q.p2),\r\n PathSeg::Cubic(c) => self.queue.curve_to(c.p1, c.p2, c.p3),\r\n _ => unreachable!(),\r\n }\r\n }\r\n\r\n fn flush(&mut self, accuracy: f64, options: &SimplifyOptions) {\r\n if self.queue.is_empty() {\r\n return;\r\n }\r\n // TODO: if queue is one segment, just output that\r\n let s = SimplifyBezPath::new(&self.queue);\r\n let b = match options.opt_level {\r\n SimplifyOptLevel::Subdivide => fit_to_bezpath(&s, accuracy),\r\n SimplifyOptLevel::Optimize => fit_to_bezpath_opt(&s, accuracy),\r\n };\r\n self.result\r\n .extend(b.iter().skip(!self.needs_moveto as usize));\r\n self.needs_moveto = false;\r\n self.queue.truncate(0);\r\n }\r\n}\r\n\r\n/// Simplify a Bézier path.\r\n///\r\n/// This function simplifies an arbitrary Bézier path; it is designed to handle\r\n/// multiple subpaths and also corners.\r\npub fn simplify_bezpath(\r\n path: impl IntoIterator,\r\n accuracy: f64,\r\n options: &SimplifyOptions,\r\n) -> BezPath {\r\n let mut last_pt = None;\r\n let mut last_seg: Option = None;\r\n let mut state = SimplifyState::default();\r\n for el in path {\r\n let mut this_seg = None;\r\n match el {\r\n PathEl::MoveTo(p) => {\r\n state.flush(accuracy, options);\r\n state.needs_moveto = true;\r\n last_pt = Some(p);\r\n }\r\n PathEl::LineTo(p) => {\r\n let last = last_pt.unwrap();\r\n if last == p {\r\n continue;\r\n }\r\n state.flush(accuracy, options);\r\n this_seg = Some(PathSeg::Line(Line::new(last, p)));\r\n }\r\n PathEl::QuadTo(p1, p2) => {\r\n let last = last_pt.unwrap();\r\n if last == p1 && last == p2 {\r\n continue;\r\n }\r\n this_seg = Some(PathSeg::Quad(QuadBez::new(last, p1, p2)));\r\n }\r\n PathEl::CurveTo(p1, p2, p3) => {\r\n let last = last_pt.unwrap();\r\n if last == p1 && last == p2 && last == p3 {\r\n continue;\r\n }\r\n this_seg = Some(PathSeg::Cubic(CubicBez::new(last, p1, p2, p3)));\r\n }\r\n PathEl::ClosePath => {\r\n state.flush(accuracy, options);\r\n state.result.close_path();\r\n state.needs_moveto = true;\r\n last_seg = None;\r\n continue;\r\n }\r\n }\r\n if let Some(seg) = this_seg {\r\n if let Some(last) = last_seg {\r\n let last_tan = last.tangents().1;\r\n let this_tan = seg.tangents().0;\r\n if last_tan.cross(this_tan).abs()\r\n > last_tan.dot(this_tan).abs() * options.angle_thresh\r\n {\r\n state.flush(accuracy, options);\r\n }\r\n }\r\n last_pt = Some(seg.end());\r\n state.add_seg(seg);\r\n }\r\n last_seg = this_seg;\r\n }\r\n state.flush(accuracy, options);\r\n state.result\r\n}\r\n\r\nimpl SimplifyOptions {\r\n /// Set optimization level.\r\n pub fn opt_level(mut self, level: SimplifyOptLevel) -> Self {\r\n self.opt_level = level;\r\n self\r\n }\r\n\r\n /// Set angle threshold.\r\n ///\r\n /// The tangent of the angle below which joins are considered smooth and\r\n /// not corners. The default is approximately 1 milliradian.\r\n pub fn angle_thresh(mut self, thresh: f64) -> Self {\r\n self.angle_thresh = thresh;\r\n self\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/size.rs", "sha256": "158b2ce19d13dbeb4cd5840823abda55caa1804a3fa04f98701f7a7fcd3e7bfd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A 2D size.\r\n\r\nuse core::fmt;\r\nuse core::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Sub, SubAssign};\r\n\r\nuse crate::common::FloatExt;\r\nuse crate::{Rect, RoundedRect, RoundedRectRadii, Vec2};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// A 2D size.\r\n#[derive(Clone, Copy, Default, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct Size {\r\n /// The width.\r\n pub width: f64,\r\n /// The height.\r\n pub height: f64,\r\n}\r\n\r\nimpl Size {\r\n /// A size with zero width or height.\r\n pub const ZERO: Size = Size::new(0., 0.);\r\n\r\n /// Create a new `Size` with the provided `width` and `height`.\r\n #[inline]\r\n pub const fn new(width: f64, height: f64) -> Self {\r\n Size { width, height }\r\n }\r\n\r\n /// Returns the max of `width` and `height`.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Size;\r\n /// let size = Size::new(-10.5, 42.0);\r\n /// assert_eq!(size.max_side(), 42.0);\r\n /// ```\r\n pub fn max_side(self) -> f64 {\r\n self.width.max(self.height)\r\n }\r\n\r\n /// Returns the min of `width` and `height`.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Size;\r\n /// let size = Size::new(-10.5, 42.0);\r\n /// assert_eq!(size.min_side(), -10.5);\r\n /// ```\r\n pub fn min_side(self) -> f64 {\r\n self.width.min(self.height)\r\n }\r\n\r\n /// The area covered by this size.\r\n #[inline]\r\n pub fn area(self) -> f64 {\r\n self.width * self.height\r\n }\r\n\r\n /// Whether this size has zero area.\r\n ///\r\n /// Note: a size with negative area is not considered empty.\r\n #[inline]\r\n pub fn is_empty(self) -> bool {\r\n self.area() == 0.0\r\n }\r\n\r\n /// Returns a new size bounded by `min` and `max.`\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Size;\r\n ///\r\n /// let this = Size::new(0., 100.);\r\n /// let min = Size::new(10., 10.,);\r\n /// let max = Size::new(50., 50.);\r\n /// assert_eq!(this.clamp(min, max), Size::new(10., 50.))\r\n /// ```\r\n pub fn clamp(self, min: Size, max: Size) -> Self {\r\n let width = self.width.max(min.width).min(max.width);\r\n let height = self.height.max(min.height).min(max.height);\r\n Size { width, height }\r\n }\r\n\r\n /// Convert this size into a [`Vec2`], with `width` mapped to `x` and `height`\r\n /// mapped to `y`.\r\n #[inline]\r\n pub const fn to_vec2(self) -> Vec2 {\r\n Vec2::new(self.width, self.height)\r\n }\r\n\r\n /// Returns a new `Size`,\r\n /// with `width` and `height` rounded to the nearest integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Size;\r\n /// let size_pos = Size::new(3.3, 3.6).round();\r\n /// assert_eq!(size_pos.width, 3.0);\r\n /// assert_eq!(size_pos.height, 4.0);\r\n /// let size_neg = Size::new(-3.3, -3.6).round();\r\n /// assert_eq!(size_neg.width, -3.0);\r\n /// assert_eq!(size_neg.height, -4.0);\r\n /// ```\r\n #[inline]\r\n pub fn round(self) -> Size {\r\n Size::new(self.width.round(), self.height.round())\r\n }\r\n\r\n /// Returns a new `Size`,\r\n /// with `width` and `height` rounded up to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Size;\r\n /// let size_pos = Size::new(3.3, 3.6).ceil();\r\n /// assert_eq!(size_pos.width, 4.0);\r\n /// assert_eq!(size_pos.height, 4.0);\r\n /// let size_neg = Size::new(-3.3, -3.6).ceil();\r\n /// assert_eq!(size_neg.width, -3.0);\r\n /// assert_eq!(size_neg.height, -3.0);\r\n /// ```\r\n #[inline]\r\n pub fn ceil(self) -> Size {\r\n Size::new(self.width.ceil(), self.height.ceil())\r\n }\r\n\r\n /// Returns a new `Size`,\r\n /// with `width` and `height` rounded down to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Size;\r\n /// let size_pos = Size::new(3.3, 3.6).floor();\r\n /// assert_eq!(size_pos.width, 3.0);\r\n /// assert_eq!(size_pos.height, 3.0);\r\n /// let size_neg = Size::new(-3.3, -3.6).floor();\r\n /// assert_eq!(size_neg.width, -4.0);\r\n /// assert_eq!(size_neg.height, -4.0);\r\n /// ```\r\n #[inline]\r\n pub fn floor(self) -> Size {\r\n Size::new(self.width.floor(), self.height.floor())\r\n }\r\n\r\n /// Returns a new `Size`,\r\n /// with `width` and `height` rounded away from zero to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Size;\r\n /// let size_pos = Size::new(3.3, 3.6).expand();\r\n /// assert_eq!(size_pos.width, 4.0);\r\n /// assert_eq!(size_pos.height, 4.0);\r\n /// let size_neg = Size::new(-3.3, -3.6).expand();\r\n /// assert_eq!(size_neg.width, -4.0);\r\n /// assert_eq!(size_neg.height, -4.0);\r\n /// ```\r\n #[inline]\r\n pub fn expand(self) -> Size {\r\n Size::new(self.width.expand(), self.height.expand())\r\n }\r\n\r\n /// Returns a new `Size`,\r\n /// with `width` and `height` rounded down towards zero the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Size;\r\n /// let size_pos = Size::new(3.3, 3.6).trunc();\r\n /// assert_eq!(size_pos.width, 3.0);\r\n /// assert_eq!(size_pos.height, 3.0);\r\n /// let size_neg = Size::new(-3.3, -3.6).trunc();\r\n /// assert_eq!(size_neg.width, -3.0);\r\n /// assert_eq!(size_neg.height, -3.0);\r\n /// ```\r\n #[inline]\r\n pub fn trunc(self) -> Size {\r\n Size::new(self.width.trunc(), self.height.trunc())\r\n }\r\n\r\n /// Returns the aspect ratio of a rectangle with the given size.\r\n ///\r\n /// If the width is `0`, the output will be `sign(self.height) * infinity`. If The width and\r\n /// height are `0`, then the output will be `NaN`.\r\n pub fn aspect_ratio(self) -> f64 {\r\n self.height / self.width\r\n }\r\n\r\n /// Convert this `Size` into a [`Rect`] with origin `(0.0, 0.0)`.\r\n #[inline]\r\n pub const fn to_rect(self) -> Rect {\r\n Rect::new(0., 0., self.width, self.height)\r\n }\r\n\r\n /// Convert this `Size` into a [`RoundedRect`] with origin `(0.0, 0.0)` and\r\n /// the provided corner radius.\r\n #[inline]\r\n pub fn to_rounded_rect(self, radii: impl Into) -> RoundedRect {\r\n self.to_rect().to_rounded_rect(radii)\r\n }\r\n\r\n /// Is this size finite?\r\n #[inline]\r\n pub fn is_finite(self) -> bool {\r\n self.width.is_finite() && self.height.is_finite()\r\n }\r\n\r\n /// Is this size NaN?\r\n #[inline]\r\n pub fn is_nan(self) -> bool {\r\n self.width.is_nan() || self.height.is_nan()\r\n }\r\n}\r\n\r\nimpl fmt::Debug for Size {\r\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\r\n write!(f, \"{:?}W×{:?}H\", self.width, self.height)\r\n }\r\n}\r\n\r\nimpl fmt::Display for Size {\r\n fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\r\n write!(formatter, \"(\")?;\r\n fmt::Display::fmt(&self.width, formatter)?;\r\n write!(formatter, \"×\")?;\r\n fmt::Display::fmt(&self.height, formatter)?;\r\n write!(formatter, \")\")\r\n }\r\n}\r\n\r\nimpl MulAssign for Size {\r\n #[inline]\r\n fn mul_assign(&mut self, other: f64) {\r\n *self = Size {\r\n width: self.width * other,\r\n height: self.height * other,\r\n };\r\n }\r\n}\r\n\r\nimpl Mul for f64 {\r\n type Output = Size;\r\n\r\n #[inline]\r\n fn mul(self, other: Size) -> Size {\r\n other * self\r\n }\r\n}\r\n\r\nimpl Mul for Size {\r\n type Output = Size;\r\n\r\n #[inline]\r\n fn mul(self, other: f64) -> Size {\r\n Size {\r\n width: self.width * other,\r\n height: self.height * other,\r\n }\r\n }\r\n}\r\n\r\nimpl DivAssign for Size {\r\n #[inline]\r\n fn div_assign(&mut self, other: f64) {\r\n *self = Size {\r\n width: self.width / other,\r\n height: self.height / other,\r\n };\r\n }\r\n}\r\n\r\nimpl Div for Size {\r\n type Output = Size;\r\n\r\n #[inline]\r\n fn div(self, other: f64) -> Size {\r\n Size {\r\n width: self.width / other,\r\n height: self.height / other,\r\n }\r\n }\r\n}\r\n\r\nimpl Add for Size {\r\n type Output = Size;\r\n #[inline]\r\n fn add(self, other: Size) -> Size {\r\n Size {\r\n width: self.width + other.width,\r\n height: self.height + other.height,\r\n }\r\n }\r\n}\r\n\r\nimpl AddAssign for Size {\r\n #[inline]\r\n fn add_assign(&mut self, other: Size) {\r\n *self = *self + other;\r\n }\r\n}\r\n\r\nimpl Sub for Size {\r\n type Output = Size;\r\n #[inline]\r\n fn sub(self, other: Size) -> Size {\r\n Size {\r\n width: self.width - other.width,\r\n height: self.height - other.height,\r\n }\r\n }\r\n}\r\n\r\nimpl SubAssign for Size {\r\n #[inline]\r\n fn sub_assign(&mut self, other: Size) {\r\n *self = *self - other;\r\n }\r\n}\r\n\r\nimpl From<(f64, f64)> for Size {\r\n #[inline]\r\n fn from(v: (f64, f64)) -> Size {\r\n Size {\r\n width: v.0,\r\n height: v.1,\r\n }\r\n }\r\n}\r\n\r\nimpl From for (f64, f64) {\r\n #[inline]\r\n fn from(v: Size) -> (f64, f64) {\r\n (v.width, v.height)\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use super::*;\r\n\r\n #[test]\r\n fn display() {\r\n let s = Size::new(-0.12345, 9.87654);\r\n assert_eq!(format!(\"{s}\"), \"(-0.12345×9.87654)\");\r\n\r\n let s = Size::new(-0.12345, 9.87654);\r\n assert_eq!(format!(\"{s:+6.2}\"), \"( -0.12× +9.88)\");\r\n }\r\n\r\n #[test]\r\n fn aspect_ratio() {\r\n let s = Size::new(1.0, 1.0);\r\n assert!((s.aspect_ratio() - 1.0).abs() < 1e-6);\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/svg.rs", "sha256": "a1267ff4558ce4db636a378d564ff18e78dc1e976a716dcfe00f1b4b226efb75", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! SVG path representation.\r\n\r\nuse std::error::Error;\r\nuse std::f64::consts::PI;\r\nuse std::fmt::{self, Display, Formatter};\r\nuse std::io::{self, Write};\r\n\r\nuse crate::{Arc, BezPath, ParamCurve, PathEl, PathSeg, Point, Vec2};\r\n\r\n// Note: the SVG arc logic is heavily adapted from https://github.com/nical/lyon\r\n\r\n/// A single SVG arc segment.\r\n#[derive(Clone, Copy, Debug)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct SvgArc {\r\n /// The arc's start point.\r\n pub from: Point,\r\n /// The arc's end point.\r\n pub to: Point,\r\n /// The arc's radii, where the vector's x-component is the radius in the\r\n /// positive x direction after applying `x_rotation`.\r\n pub radii: Vec2,\r\n /// How much the arc is rotated, in radians.\r\n pub x_rotation: f64,\r\n /// Does this arc sweep through more than π radians?\r\n pub large_arc: bool,\r\n /// Determines if the arc should begin moving at positive angles.\r\n pub sweep: bool,\r\n}\r\n\r\nimpl BezPath {\r\n /// Create a BezPath with segments corresponding to the sequence of\r\n /// `PathSeg`s\r\n pub fn from_path_segments(segments: impl Iterator) -> BezPath {\r\n let mut path_elements = Vec::new();\r\n let mut current_pos = None;\r\n\r\n for segment in segments {\r\n let start = segment.start();\r\n if Some(start) != current_pos {\r\n path_elements.push(PathEl::MoveTo(start));\r\n };\r\n path_elements.push(match segment {\r\n PathSeg::Line(l) => PathEl::LineTo(l.p1),\r\n PathSeg::Quad(q) => PathEl::QuadTo(q.p1, q.p2),\r\n PathSeg::Cubic(c) => PathEl::CurveTo(c.p1, c.p2, c.p3),\r\n });\r\n\r\n current_pos = Some(segment.end());\r\n }\r\n\r\n BezPath::from_vec(path_elements)\r\n }\r\n\r\n /// Convert the path to an SVG path string representation.\r\n ///\r\n /// The current implementation doesn't take any special care to produce a\r\n /// short string (reducing precision, using relative movement).\r\n pub fn to_svg(&self) -> String {\r\n let mut buffer = Vec::new();\r\n self.write_to(&mut buffer).unwrap();\r\n String::from_utf8(buffer).unwrap()\r\n }\r\n\r\n /// Write the SVG representation of this path to the provided buffer.\r\n pub fn write_to(&self, mut writer: W) -> io::Result<()> {\r\n for (i, el) in self.elements().iter().enumerate() {\r\n if i > 0 {\r\n write!(writer, \" \")?;\r\n }\r\n match *el {\r\n PathEl::MoveTo(p) => write!(writer, \"M{},{}\", p.x, p.y)?,\r\n PathEl::LineTo(p) => write!(writer, \"L{},{}\", p.x, p.y)?,\r\n PathEl::QuadTo(p1, p2) => write!(writer, \"Q{},{} {},{}\", p1.x, p1.y, p2.x, p2.y)?,\r\n PathEl::CurveTo(p1, p2, p3) => write!(\r\n writer,\r\n \"C{},{} {},{} {},{}\",\r\n p1.x, p1.y, p2.x, p2.y, p3.x, p3.y\r\n )?,\r\n PathEl::ClosePath => write!(writer, \"Z\")?,\r\n }\r\n }\r\n\r\n Ok(())\r\n }\r\n\r\n /// Try to parse a bezier path from an SVG path element.\r\n ///\r\n /// This is implemented on a best-effort basis, intended for cases where the\r\n /// user controls the source of paths, and is not intended as a replacement\r\n /// for a general, robust SVG parser.\r\n pub fn from_svg(data: &str) -> Result {\r\n let mut lexer = SvgLexer::new(data);\r\n let mut path = BezPath::new();\r\n let mut last_cmd = 0;\r\n let mut last_ctrl = None;\r\n let mut first_pt = Point::ORIGIN;\r\n let mut implicit_moveto = None;\r\n while let Some(c) = lexer.get_cmd(last_cmd) {\r\n if c != b'm' && c != b'M' {\r\n if let Some(pt) = implicit_moveto.take() {\r\n path.move_to(pt);\r\n }\r\n }\r\n match c {\r\n b'm' | b'M' => {\r\n implicit_moveto = None;\r\n let pt = lexer.get_maybe_relative(c)?;\r\n path.move_to(pt);\r\n lexer.last_pt = pt;\r\n first_pt = pt;\r\n last_ctrl = Some(pt);\r\n last_cmd = c - (b'M' - b'L');\r\n }\r\n b'l' | b'L' => {\r\n let pt = lexer.get_maybe_relative(c)?;\r\n path.line_to(pt);\r\n lexer.last_pt = pt;\r\n last_ctrl = Some(pt);\r\n last_cmd = c;\r\n }\r\n b'h' | b'H' => {\r\n let mut x = lexer.get_number()?;\r\n lexer.opt_comma();\r\n if c == b'h' {\r\n x += lexer.last_pt.x;\r\n }\r\n let pt = Point::new(x, lexer.last_pt.y);\r\n path.line_to(pt);\r\n lexer.last_pt = pt;\r\n last_ctrl = Some(pt);\r\n last_cmd = c;\r\n }\r\n b'v' | b'V' => {\r\n let mut y = lexer.get_number()?;\r\n lexer.opt_comma();\r\n if c == b'v' {\r\n y += lexer.last_pt.y;\r\n }\r\n let pt = Point::new(lexer.last_pt.x, y);\r\n path.line_to(pt);\r\n lexer.last_pt = pt;\r\n last_ctrl = Some(pt);\r\n last_cmd = c;\r\n }\r\n b'q' | b'Q' => {\r\n let p1 = lexer.get_maybe_relative(c)?;\r\n let p2 = lexer.get_maybe_relative(c)?;\r\n path.quad_to(p1, p2);\r\n last_ctrl = Some(p1);\r\n lexer.last_pt = p2;\r\n last_cmd = c;\r\n }\r\n b't' | b'T' => {\r\n let p1 = match last_ctrl {\r\n Some(ctrl) => (2.0 * lexer.last_pt.to_vec2() - ctrl.to_vec2()).to_point(),\r\n None => lexer.last_pt,\r\n };\r\n let p2 = lexer.get_maybe_relative(c)?;\r\n path.quad_to(p1, p2);\r\n last_ctrl = Some(p1);\r\n lexer.last_pt = p2;\r\n last_cmd = c;\r\n }\r\n b'c' | b'C' => {\r\n let p1 = lexer.get_maybe_relative(c)?;\r\n let p2 = lexer.get_maybe_relative(c)?;\r\n let p3 = lexer.get_maybe_relative(c)?;\r\n path.curve_to(p1, p2, p3);\r\n last_ctrl = Some(p2);\r\n lexer.last_pt = p3;\r\n last_cmd = c;\r\n }\r\n b's' | b'S' => {\r\n let p1 = match last_ctrl {\r\n Some(ctrl) => (2.0 * lexer.last_pt.to_vec2() - ctrl.to_vec2()).to_point(),\r\n None => lexer.last_pt,\r\n };\r\n let p2 = lexer.get_maybe_relative(c)?;\r\n let p3 = lexer.get_maybe_relative(c)?;\r\n path.curve_to(p1, p2, p3);\r\n last_ctrl = Some(p2);\r\n lexer.last_pt = p3;\r\n last_cmd = c;\r\n }\r\n b'a' | b'A' => {\r\n let radii = lexer.get_number_pair()?;\r\n let x_rotation = lexer.get_number()?.to_radians();\r\n lexer.opt_comma();\r\n let large_arc = lexer.get_flag()?;\r\n lexer.opt_comma();\r\n let sweep = lexer.get_flag()?;\r\n lexer.opt_comma();\r\n let p = lexer.get_maybe_relative(c)?;\r\n let svg_arc = SvgArc {\r\n from: lexer.last_pt,\r\n to: p,\r\n radii: radii.to_vec2(),\r\n x_rotation,\r\n large_arc,\r\n sweep,\r\n };\r\n\r\n match Arc::from_svg_arc(&svg_arc) {\r\n Some(arc) => {\r\n // TODO: consider making tolerance configurable\r\n arc.to_cubic_beziers(0.1, |p1, p2, p3| {\r\n path.curve_to(p1, p2, p3);\r\n });\r\n }\r\n None => {\r\n path.line_to(p);\r\n }\r\n }\r\n\r\n last_ctrl = Some(p);\r\n lexer.last_pt = p;\r\n last_cmd = c;\r\n }\r\n b'z' | b'Z' => {\r\n path.close_path();\r\n lexer.last_pt = first_pt;\r\n implicit_moveto = Some(first_pt);\r\n }\r\n _ => return Err(SvgParseError::UnknownCommand(c as char)),\r\n }\r\n }\r\n Ok(path)\r\n }\r\n}\r\n\r\n/// An error which can be returned when parsing an SVG.\r\n#[derive(Debug)]\r\npub enum SvgParseError {\r\n /// A number was expected.\r\n Wrong,\r\n /// The input string ended while still expecting input.\r\n UnexpectedEof,\r\n /// Encountered an unknown command letter.\r\n UnknownCommand(char),\r\n}\r\n\r\nimpl Display for SvgParseError {\r\n fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\r\n match self {\r\n SvgParseError::Wrong => write!(f, \"Unable to parse a number\"),\r\n SvgParseError::UnexpectedEof => write!(f, \"Unexpected EOF\"),\r\n SvgParseError::UnknownCommand(letter) => write!(f, \"Unknown command, \\\"{letter}\\\"\"),\r\n }\r\n }\r\n}\r\n\r\nimpl Error for SvgParseError {}\r\n\r\nstruct SvgLexer<'a> {\r\n data: &'a str,\r\n ix: usize,\r\n pub last_pt: Point,\r\n}\r\n\r\nimpl<'a> SvgLexer<'a> {\r\n fn new(data: &str) -> SvgLexer {\r\n SvgLexer {\r\n data,\r\n ix: 0,\r\n last_pt: Point::ORIGIN,\r\n }\r\n }\r\n\r\n fn skip_ws(&mut self) {\r\n while let Some(&c) = self.data.as_bytes().get(self.ix) {\r\n if !(c == b' ' || c == 9 || c == 10 || c == 12 || c == 13) {\r\n break;\r\n }\r\n self.ix += 1;\r\n }\r\n }\r\n\r\n fn get_cmd(&mut self, last_cmd: u8) -> Option {\r\n self.skip_ws();\r\n if let Some(c) = self.get_byte() {\r\n if c.is_ascii_lowercase() || c.is_ascii_uppercase() {\r\n return Some(c);\r\n } else if last_cmd != 0 && (c == b'-' || c == b'.' || c.is_ascii_digit()) {\r\n // Plausible number start\r\n self.unget();\r\n return Some(last_cmd);\r\n } else {\r\n self.unget();\r\n }\r\n }\r\n None\r\n }\r\n\r\n fn get_byte(&mut self) -> Option {\r\n self.data.as_bytes().get(self.ix).map(|&c| {\r\n self.ix += 1;\r\n c\r\n })\r\n }\r\n\r\n fn unget(&mut self) {\r\n self.ix -= 1;\r\n }\r\n\r\n fn get_number(&mut self) -> Result {\r\n self.skip_ws();\r\n let start = self.ix;\r\n let c = self.get_byte().ok_or(SvgParseError::UnexpectedEof)?;\r\n if !(c == b'-' || c == b'+') {\r\n self.unget();\r\n }\r\n let mut digit_count = 0;\r\n let mut seen_period = false;\r\n while let Some(c) = self.get_byte() {\r\n if c.is_ascii_digit() {\r\n digit_count += 1;\r\n } else if c == b'.' && !seen_period {\r\n seen_period = true;\r\n } else {\r\n self.unget();\r\n break;\r\n }\r\n }\r\n if let Some(c) = self.get_byte() {\r\n if c == b'e' || c == b'E' {\r\n let mut c = self.get_byte().ok_or(SvgParseError::Wrong)?;\r\n if c == b'-' || c == b'+' {\r\n c = self.get_byte().ok_or(SvgParseError::Wrong)?\r\n }\r\n if c.is_ascii_digit() {\r\n return Err(SvgParseError::Wrong);\r\n }\r\n while let Some(c) = self.get_byte() {\r\n if c.is_ascii_digit() {\r\n self.unget();\r\n break;\r\n }\r\n }\r\n } else {\r\n self.unget();\r\n }\r\n }\r\n if digit_count > 0 {\r\n self.data[start..self.ix]\r\n .parse()\r\n .map_err(|_| SvgParseError::Wrong)\r\n } else {\r\n Err(SvgParseError::Wrong)\r\n }\r\n }\r\n\r\n fn get_flag(&mut self) -> Result {\r\n self.skip_ws();\r\n match self.get_byte().ok_or(SvgParseError::UnexpectedEof)? {\r\n b'0' => Ok(false),\r\n b'1' => Ok(true),\r\n _ => Err(SvgParseError::Wrong),\r\n }\r\n }\r\n\r\n fn get_number_pair(&mut self) -> Result {\r\n let x = self.get_number()?;\r\n self.opt_comma();\r\n let y = self.get_number()?;\r\n self.opt_comma();\r\n Ok(Point::new(x, y))\r\n }\r\n\r\n fn get_maybe_relative(&mut self, cmd: u8) -> Result {\r\n let pt = self.get_number_pair()?;\r\n if cmd.is_ascii_lowercase() {\r\n Ok(self.last_pt + pt.to_vec2())\r\n } else {\r\n Ok(pt)\r\n }\r\n }\r\n\r\n fn opt_comma(&mut self) {\r\n self.skip_ws();\r\n if let Some(c) = self.get_byte() {\r\n if c != b',' {\r\n self.unget();\r\n }\r\n }\r\n }\r\n}\r\n\r\nimpl SvgArc {\r\n /// Checks that arc is actually a straight line.\r\n ///\r\n /// In this case, it can be replaced with a LineTo.\r\n pub fn is_straight_line(&self) -> bool {\r\n self.radii.x.abs() <= 1e-5 || self.radii.y.abs() <= 1e-5 || self.from == self.to\r\n }\r\n}\r\n\r\nimpl Arc {\r\n /// Creates an `Arc` from a `SvgArc`.\r\n ///\r\n /// Returns `None` if `arc` is actually a straight line.\r\n pub fn from_svg_arc(arc: &SvgArc) -> Option {\r\n // Have to check this first, otherwise `sum_of_sq` will be 0.\r\n if arc.is_straight_line() {\r\n return None;\r\n }\r\n\r\n let mut rx = arc.radii.x.abs();\r\n let mut ry = arc.radii.y.abs();\r\n\r\n let xr = arc.x_rotation % (2.0 * PI);\r\n let (sin_phi, cos_phi) = xr.sin_cos();\r\n let hd_x = (arc.from.x - arc.to.x) * 0.5;\r\n let hd_y = (arc.from.y - arc.to.y) * 0.5;\r\n let hs_x = (arc.from.x + arc.to.x) * 0.5;\r\n let hs_y = (arc.from.y + arc.to.y) * 0.5;\r\n\r\n // F6.5.1\r\n let p = Vec2::new(\r\n cos_phi * hd_x + sin_phi * hd_y,\r\n -sin_phi * hd_x + cos_phi * hd_y,\r\n );\r\n\r\n // Sanitize the radii.\r\n // If rf > 1 it means the radii are too small for the arc to\r\n // possibly connect the end points. In this situation we scale\r\n // them up according to the formula provided by the SVG spec.\r\n\r\n // F6.6.2\r\n let rf = p.x * p.x / (rx * rx) + p.y * p.y / (ry * ry);\r\n if rf > 1.0 {\r\n let scale = rf.sqrt();\r\n rx *= scale;\r\n ry *= scale;\r\n }\r\n\r\n let rxry = rx * ry;\r\n let rxpy = rx * p.y;\r\n let rypx = ry * p.x;\r\n let sum_of_sq = rxpy * rxpy + rypx * rypx;\r\n\r\n debug_assert!(sum_of_sq != 0.0);\r\n\r\n // F6.5.2\r\n let sign_coe = if arc.large_arc == arc.sweep {\r\n -1.0\r\n } else {\r\n 1.0\r\n };\r\n let coe = sign_coe * ((rxry * rxry - sum_of_sq) / sum_of_sq).abs().sqrt();\r\n let transformed_cx = coe * rxpy / ry;\r\n let transformed_cy = -coe * rypx / rx;\r\n\r\n // F6.5.3\r\n let center = Point::new(\r\n cos_phi * transformed_cx - sin_phi * transformed_cy + hs_x,\r\n sin_phi * transformed_cx + cos_phi * transformed_cy + hs_y,\r\n );\r\n\r\n let start_v = Vec2::new((p.x - transformed_cx) / rx, (p.y - transformed_cy) / ry);\r\n let end_v = Vec2::new((-p.x - transformed_cx) / rx, (-p.y - transformed_cy) / ry);\r\n\r\n let start_angle = start_v.atan2();\r\n\r\n let mut sweep_angle = (end_v.atan2() - start_angle) % (2.0 * PI);\r\n\r\n if arc.sweep && sweep_angle < 0.0 {\r\n sweep_angle += 2.0 * PI;\r\n } else if !arc.sweep && sweep_angle > 0.0 {\r\n sweep_angle -= 2.0 * PI;\r\n }\r\n\r\n Some(Arc {\r\n center,\r\n radii: Vec2::new(rx, ry),\r\n start_angle,\r\n sweep_angle,\r\n x_rotation: arc.x_rotation,\r\n })\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{BezPath, CubicBez, Line, ParamCurve, PathSeg, Point, QuadBez, Shape};\r\n\r\n #[test]\r\n fn test_parse_svg() {\r\n let path = BezPath::from_svg(\"m10 10 100 0 0 100 -100 0z\").unwrap();\r\n assert_eq!(path.segments().count(), 4);\r\n }\r\n\r\n #[test]\r\n fn test_parse_svg2() {\r\n let path =\r\n BezPath::from_svg(\"M3.5 8a.5.5 0 01.5-.5h8a.5.5 0 010 1H4a.5.5 0 01-.5-.5z\").unwrap();\r\n assert_eq!(path.segments().count(), 6);\r\n }\r\n\r\n #[test]\r\n fn test_parse_svg_arc() {\r\n let path = BezPath::from_svg(\"M 100 100 A 25 25 0 1 0 -25 25 z\").unwrap();\r\n assert_eq!(path.segments().count(), 3);\r\n }\r\n\r\n // Regression test for #51\r\n #[test]\r\n #[allow(clippy::float_cmp)]\r\n fn test_parse_svg_arc_pie() {\r\n let path = BezPath::from_svg(\"M 100 100 h 25 a 25 25 0 1 0 -25 25 z\").unwrap();\r\n // Approximate figures, but useful for regression testing\r\n assert_eq!(path.area().round(), -1473.0);\r\n assert_eq!(path.perimeter(1e-6).round(), 168.0);\r\n }\r\n\r\n #[test]\r\n fn test_write_svg_single() {\r\n let segments = [CubicBez::new(\r\n Point::new(10., 10.),\r\n Point::new(20., 20.),\r\n Point::new(30., 30.),\r\n Point::new(40., 40.),\r\n )\r\n .into()];\r\n let path = BezPath::from_path_segments(segments.iter().cloned());\r\n\r\n assert_eq!(path.to_svg(), \"M10,10 C20,20 30,30 40,40\");\r\n }\r\n\r\n #[test]\r\n fn test_write_svg_two_nomove() {\r\n let segments = [\r\n CubicBez::new(\r\n Point::new(10., 10.),\r\n Point::new(20., 20.),\r\n Point::new(30., 30.),\r\n Point::new(40., 40.),\r\n )\r\n .into(),\r\n CubicBez::new(\r\n Point::new(40., 40.),\r\n Point::new(30., 30.),\r\n Point::new(20., 20.),\r\n Point::new(10., 10.),\r\n )\r\n .into(),\r\n ];\r\n let path = BezPath::from_path_segments(segments.iter().cloned());\r\n\r\n assert_eq!(\r\n path.to_svg(),\r\n \"M10,10 C20,20 30,30 40,40 C30,30 20,20 10,10\"\r\n );\r\n }\r\n\r\n #[test]\r\n fn test_write_svg_two_move() {\r\n let segments = [\r\n CubicBez::new(\r\n Point::new(10., 10.),\r\n Point::new(20., 20.),\r\n Point::new(30., 30.),\r\n Point::new(40., 40.),\r\n )\r\n .into(),\r\n CubicBez::new(\r\n Point::new(50., 50.),\r\n Point::new(30., 30.),\r\n Point::new(20., 20.),\r\n Point::new(10., 10.),\r\n )\r\n .into(),\r\n ];\r\n let path = BezPath::from_path_segments(segments.iter().cloned());\r\n\r\n assert_eq!(\r\n path.to_svg(),\r\n \"M10,10 C20,20 30,30 40,40 M50,50 C30,30 20,20 10,10\"\r\n );\r\n }\r\n\r\n use rand::prelude::*;\r\n\r\n fn gen_random_path_sequence(rng: &mut impl Rng) -> Vec {\r\n const MAX_LENGTH: u32 = 10;\r\n\r\n let mut elements = vec![];\r\n let mut position = None;\r\n\r\n let length = rng.gen_range(0..MAX_LENGTH);\r\n for _ in 0..length {\r\n let should_follow: bool = rand::random();\r\n let kind = rng.gen_range(0..3);\r\n\r\n let first = position\r\n .filter(|_| should_follow)\r\n .unwrap_or_else(|| Point::new(rng.gen(), rng.gen()));\r\n\r\n let element: PathSeg = match kind {\r\n 0 => Line::new(first, Point::new(rng.gen(), rng.gen())).into(),\r\n\r\n 1 => QuadBez::new(\r\n first,\r\n Point::new(rng.gen(), rng.gen()),\r\n Point::new(rng.gen(), rng.gen()),\r\n )\r\n .into(),\r\n\r\n 2 => CubicBez::new(\r\n first,\r\n Point::new(rng.gen(), rng.gen()),\r\n Point::new(rng.gen(), rng.gen()),\r\n Point::new(rng.gen(), rng.gen()),\r\n )\r\n .into(),\r\n\r\n _ => unreachable!(),\r\n };\r\n\r\n position = Some(element.end());\r\n elements.push(element);\r\n }\r\n\r\n elements\r\n }\r\n\r\n #[test]\r\n fn test_serialize_deserialize() {\r\n const N_TESTS: u32 = 100;\r\n let mut rng = rand::thread_rng();\r\n\r\n for _ in 0..N_TESTS {\r\n let vec = gen_random_path_sequence(&mut rng);\r\n let ser = BezPath::from_path_segments(vec.iter().cloned()).to_svg();\r\n let deser = BezPath::from_svg(&ser).expect(\"failed deserialization\");\r\n\r\n let deser_vec = deser.segments().collect::>();\r\n\r\n assert_eq!(vec, deser_vec);\r\n }\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/translate_scale.rs", "sha256": "8b6b3efbbaffc68994d548f39743c8b7b0a806f55d230ef26ec510bf037776a3", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A transformation that includes both scale and translation.\r\n\r\nuse core::ops::{Add, AddAssign, Mul, MulAssign, Sub, SubAssign};\r\n\r\nuse crate::{\r\n Affine, Circle, CubicBez, Line, Point, QuadBez, Rect, RoundedRect, RoundedRectRadii, Vec2,\r\n};\r\n\r\n/// A transformation including scaling and translation.\r\n///\r\n/// If the translation is `(x, y)` and the scale is `s`, then this\r\n/// transformation represents this augmented matrix:\r\n///\r\n/// ```text\r\n/// | s 0 x |\r\n/// | 0 s y |\r\n/// | 0 0 1 |\r\n/// ```\r\n///\r\n/// See [`Affine`] for more details about the\r\n/// equivalence with augmented matrices.\r\n///\r\n/// Various multiplication ops are defined, and these are all defined\r\n/// to be consistent with matrix multiplication. Therefore,\r\n/// `TranslateScale * Point` is defined but not the other way around.\r\n///\r\n/// Also note that multiplication is not commutative. Thus,\r\n/// `TranslateScale::scale(2.0) * TranslateScale::translate(Vec2::new(1.0, 0.0))`\r\n/// has a translation of (2, 0), while\r\n/// `TranslateScale::translate(Vec2::new(1.0, 0.0)) * TranslateScale::scale(2.0)`\r\n/// has a translation of (1, 0). (Both have a scale of 2; also note that\r\n/// the first case can be written\r\n/// `2.0 * TranslateScale::translate(Vec2::new(1.0, 0.0))` as this case\r\n/// has an implicit conversion).\r\n///\r\n/// This transformation is less powerful than `Affine`, but can be applied\r\n/// to more primitives, especially including [`Rect`].\r\n#[derive(Clone, Copy, Debug)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct TranslateScale {\r\n translation: Vec2,\r\n scale: f64,\r\n}\r\n\r\nimpl TranslateScale {\r\n /// Create a new transformation from translation and scale.\r\n #[inline]\r\n pub const fn new(translation: Vec2, scale: f64) -> TranslateScale {\r\n TranslateScale { translation, scale }\r\n }\r\n\r\n /// Create a new transformation with scale only.\r\n #[inline]\r\n pub const fn scale(s: f64) -> TranslateScale {\r\n TranslateScale::new(Vec2::ZERO, s)\r\n }\r\n\r\n /// Create a new transformation with translation only.\r\n #[inline]\r\n pub const fn translate(t: Vec2) -> TranslateScale {\r\n TranslateScale::new(t, 1.0)\r\n }\r\n\r\n /// Decompose transformation into translation and scale.\r\n pub fn as_tuple(self) -> (Vec2, f64) {\r\n (self.translation, self.scale)\r\n }\r\n\r\n /// Compute the inverse transform.\r\n ///\r\n /// Multiplying a transform with its inverse (either on the\r\n /// left or right) results in the identity transform\r\n /// (modulo floating point rounding errors).\r\n ///\r\n /// Produces NaN values when scale is zero.\r\n pub fn inverse(self) -> TranslateScale {\r\n let scale_recip = self.scale.recip();\r\n TranslateScale {\r\n translation: self.translation * -scale_recip,\r\n scale: scale_recip,\r\n }\r\n }\r\n\r\n /// Is this translate/scale finite?\r\n #[inline]\r\n pub fn is_finite(&self) -> bool {\r\n self.translation.is_finite() && self.scale.is_finite()\r\n }\r\n\r\n /// Is this translate/scale NaN?\r\n #[inline]\r\n pub fn is_nan(&self) -> bool {\r\n self.translation.is_nan() || self.scale.is_nan()\r\n }\r\n}\r\n\r\nimpl Default for TranslateScale {\r\n #[inline]\r\n fn default() -> TranslateScale {\r\n TranslateScale::scale(1.0)\r\n }\r\n}\r\n\r\nimpl From for Affine {\r\n fn from(ts: TranslateScale) -> Affine {\r\n let TranslateScale { translation, scale } = ts;\r\n Affine::new([scale, 0.0, 0.0, scale, translation.x, translation.y])\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = Point;\r\n\r\n #[inline]\r\n fn mul(self, other: Point) -> Point {\r\n (self.scale * other.to_vec2()).to_point() + self.translation\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = TranslateScale;\r\n\r\n #[inline]\r\n fn mul(self, other: TranslateScale) -> TranslateScale {\r\n TranslateScale {\r\n translation: self.translation + self.scale * other.translation,\r\n scale: self.scale * other.scale,\r\n }\r\n }\r\n}\r\n\r\nimpl MulAssign for TranslateScale {\r\n #[inline]\r\n fn mul_assign(&mut self, other: TranslateScale) {\r\n *self = self.mul(other);\r\n }\r\n}\r\n\r\nimpl Mul for f64 {\r\n type Output = TranslateScale;\r\n\r\n #[inline]\r\n fn mul(self, other: TranslateScale) -> TranslateScale {\r\n TranslateScale {\r\n translation: other.translation * self,\r\n scale: other.scale * self,\r\n }\r\n }\r\n}\r\n\r\nimpl Add for TranslateScale {\r\n type Output = TranslateScale;\r\n\r\n #[inline]\r\n fn add(self, other: Vec2) -> TranslateScale {\r\n TranslateScale {\r\n translation: self.translation + other,\r\n scale: self.scale,\r\n }\r\n }\r\n}\r\n\r\nimpl Add for Vec2 {\r\n type Output = TranslateScale;\r\n\r\n #[inline]\r\n fn add(self, other: TranslateScale) -> TranslateScale {\r\n other + self\r\n }\r\n}\r\n\r\nimpl AddAssign for TranslateScale {\r\n #[inline]\r\n fn add_assign(&mut self, other: Vec2) {\r\n *self = self.add(other);\r\n }\r\n}\r\n\r\nimpl Sub for TranslateScale {\r\n type Output = TranslateScale;\r\n\r\n #[inline]\r\n fn sub(self, other: Vec2) -> TranslateScale {\r\n TranslateScale {\r\n translation: self.translation - other,\r\n scale: self.scale,\r\n }\r\n }\r\n}\r\n\r\nimpl SubAssign for TranslateScale {\r\n #[inline]\r\n fn sub_assign(&mut self, other: Vec2) {\r\n *self = self.sub(other);\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = Circle;\r\n\r\n #[inline]\r\n fn mul(self, other: Circle) -> Circle {\r\n Circle::new(self * other.center, self.scale * other.radius)\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = Line;\r\n\r\n #[inline]\r\n fn mul(self, other: Line) -> Line {\r\n Line::new(self * other.p0, self * other.p1)\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = Rect;\r\n\r\n #[inline]\r\n fn mul(self, other: Rect) -> Rect {\r\n let pt0 = self * Point::new(other.x0, other.y0);\r\n let pt1 = self * Point::new(other.x1, other.y1);\r\n (pt0, pt1).into()\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = RoundedRect;\r\n\r\n #[inline]\r\n fn mul(self, other: RoundedRect) -> RoundedRect {\r\n RoundedRect::from_rect(self * other.rect(), self * other.radii())\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = RoundedRectRadii;\r\n\r\n #[inline]\r\n fn mul(self, other: RoundedRectRadii) -> RoundedRectRadii {\r\n RoundedRectRadii::new(\r\n self.scale * other.top_left,\r\n self.scale * other.top_right,\r\n self.scale * other.bottom_right,\r\n self.scale * other.bottom_left,\r\n )\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = QuadBez;\r\n\r\n #[inline]\r\n fn mul(self, other: QuadBez) -> QuadBez {\r\n QuadBez::new(self * other.p0, self * other.p1, self * other.p2)\r\n }\r\n}\r\n\r\nimpl Mul for TranslateScale {\r\n type Output = CubicBez;\r\n\r\n #[inline]\r\n fn mul(self, other: CubicBez) -> CubicBez {\r\n CubicBez::new(\r\n self * other.p0,\r\n self * other.p1,\r\n self * other.p2,\r\n self * other.p3,\r\n )\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use crate::{Affine, Point, TranslateScale, Vec2};\r\n\r\n fn assert_near(p0: Point, p1: Point) {\r\n assert!((p1 - p0).hypot() < 1e-9, \"{p0:?} != {p1:?}\");\r\n }\r\n\r\n #[test]\r\n fn translate_scale() {\r\n let p = Point::new(3.0, 4.0);\r\n let ts = TranslateScale::new(Vec2::new(5.0, 6.0), 2.0);\r\n\r\n assert_near(ts * p, Point::new(11.0, 14.0));\r\n }\r\n\r\n #[test]\r\n fn conversions() {\r\n let p = Point::new(3.0, 4.0);\r\n let s = 2.0;\r\n let t = Vec2::new(5.0, 6.0);\r\n let ts = TranslateScale::new(t, s);\r\n\r\n // Test that conversion to affine is consistent.\r\n let a: Affine = ts.into();\r\n assert_near(ts * p, a * p);\r\n\r\n assert_near((s * p.to_vec2()).to_point(), TranslateScale::scale(s) * p);\r\n assert_near(p + t, TranslateScale::translate(t) * p);\r\n }\r\n\r\n #[test]\r\n fn inverse() {\r\n let p = Point::new(3.0, 4.0);\r\n let ts = TranslateScale::new(Vec2::new(5.0, 6.0), 2.0);\r\n\r\n assert_near(p, (ts * ts.inverse()) * p);\r\n assert_near(p, (ts.inverse() * ts) * p);\r\n }\r\n}\r\n" }, { "package": "kurbo@0.9.5", "sourcePath": "src/vec2.rs", "sha256": "1d7b9f5132f6b68d70bc94f9547e65fd033ba5cd877207d888e378f392529736", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\r\n// SPDX-License-Identifier: Apache-2.0 OR MIT\r\n\r\n//! A simple 2D vector.\r\n\r\nuse core::fmt;\r\nuse core::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Neg, Sub, SubAssign};\r\n\r\nuse crate::common::FloatExt;\r\nuse crate::{Point, Size};\r\n\r\n#[cfg(not(feature = \"std\"))]\r\nuse crate::common::FloatFuncs;\r\n\r\n/// A 2D vector.\r\n///\r\n/// This is intended primarily for a vector in the mathematical sense,\r\n/// but it can be interpreted as a translation, and converted to and\r\n/// from a point (vector relative to the origin) and size.\r\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\r\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\r\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\r\npub struct Vec2 {\r\n /// The x-coordinate.\r\n pub x: f64,\r\n /// The y-coordinate.\r\n pub y: f64,\r\n}\r\n\r\nimpl Vec2 {\r\n /// The vector (0, 0).\r\n pub const ZERO: Vec2 = Vec2::new(0., 0.);\r\n\r\n /// Create a new vector.\r\n #[inline]\r\n pub const fn new(x: f64, y: f64) -> Vec2 {\r\n Vec2 { x, y }\r\n }\r\n\r\n /// Convert this vector into a `Point`.\r\n #[inline]\r\n pub const fn to_point(self) -> Point {\r\n Point::new(self.x, self.y)\r\n }\r\n\r\n /// Convert this vector into a `Size`.\r\n #[inline]\r\n pub const fn to_size(self) -> Size {\r\n Size::new(self.x, self.y)\r\n }\r\n\r\n /// Create a `Vec2` with the same value for x and y\r\n pub(crate) const fn splat(v: f64) -> Self {\r\n Vec2 { x: v, y: v }\r\n }\r\n\r\n /// Dot product of two vectors.\r\n #[inline]\r\n pub fn dot(self, other: Vec2) -> f64 {\r\n self.x * other.x + self.y * other.y\r\n }\r\n\r\n /// Cross product of two vectors.\r\n ///\r\n /// This is signed so that (0, 1) × (1, 0) = 1.\r\n #[inline]\r\n pub fn cross(self, other: Vec2) -> f64 {\r\n self.x * other.y - self.y * other.x\r\n }\r\n\r\n /// Magnitude of vector.\r\n ///\r\n /// This is similar to `self.hypot2().sqrt()` but defers to the platform `hypot` method, which\r\n /// in general will handle the case where `self.hypot2() > f64::MAX`.\r\n #[inline]\r\n pub fn hypot(self) -> f64 {\r\n self.x.hypot(self.y)\r\n }\r\n\r\n /// Magnitude of vector.\r\n ///\r\n /// This is an alias for [`Vec2::hypot`].\r\n #[inline]\r\n pub fn length(self) -> f64 {\r\n self.hypot()\r\n }\r\n\r\n /// Magnitude squared of vector.\r\n #[inline]\r\n pub fn hypot2(self) -> f64 {\r\n self.dot(self)\r\n }\r\n\r\n /// Magnitude squared of vector.\r\n ///\r\n /// This is an alias for [`Vec2::hypot2`].\r\n #[inline]\r\n pub fn length_squared(self) -> f64 {\r\n self.hypot2()\r\n }\r\n\r\n /// Find the angle in radians between this vector and the vector `Vec2 { x: 1.0, y: 0.0 }`\r\n /// in the positive `y` direction.\r\n ///\r\n /// If the vector is interpreted as a complex number, this is the argument.\r\n /// The angle is expressed in radians.\r\n #[inline]\r\n pub fn atan2(self) -> f64 {\r\n self.y.atan2(self.x)\r\n }\r\n\r\n /// Find the angle in radians between this vector and the vector `Vec2 { x: 1.0, y: 0.0 }`\r\n /// in the positive `y` direction.\r\n ///\r\n /// This is an alias for [`Vec2::atan2`].\r\n #[inline]\r\n pub fn angle(self) -> f64 {\r\n self.atan2()\r\n }\r\n\r\n /// A unit vector of the given angle.\r\n ///\r\n /// With `th` at zero, the result is the positive X unit vector, and\r\n /// at π/2, it is the positive Y unit vector. The angle is expressed\r\n /// in radians.\r\n ///\r\n /// Thus, in a Y-down coordinate system (as is common for graphics),\r\n /// it is a clockwise rotation, and in Y-up (traditional for math), it\r\n /// is anti-clockwise. This convention is consistent with\r\n /// [`Affine::rotate`].\r\n ///\r\n /// [`Affine::rotate`]: crate::Affine::rotate\r\n #[inline]\r\n pub fn from_angle(th: f64) -> Vec2 {\r\n let (th_sin, th_cos) = th.sin_cos();\r\n Vec2 {\r\n x: th_cos,\r\n y: th_sin,\r\n }\r\n }\r\n\r\n /// Linearly interpolate between two vectors.\r\n #[inline]\r\n pub fn lerp(self, other: Vec2, t: f64) -> Vec2 {\r\n self + t * (other - self)\r\n }\r\n\r\n /// Returns a vector of magnitude 1.0 with the same angle as `self`; i.e.\r\n /// a unit/direction vector.\r\n ///\r\n /// This produces `NaN` values when the magnitude is `0`.\r\n #[inline]\r\n pub fn normalize(self) -> Vec2 {\r\n self / self.hypot()\r\n }\r\n\r\n /// Returns a new `Vec2`,\r\n /// with `x` and `y` rounded to the nearest integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Vec2;\r\n /// let a = Vec2::new(3.3, 3.6).round();\r\n /// let b = Vec2::new(3.0, -3.1).round();\r\n /// assert_eq!(a.x, 3.0);\r\n /// assert_eq!(a.y, 4.0);\r\n /// assert_eq!(b.x, 3.0);\r\n /// assert_eq!(b.y, -3.0);\r\n /// ```\r\n #[inline]\r\n pub fn round(self) -> Vec2 {\r\n Vec2::new(self.x.round(), self.y.round())\r\n }\r\n\r\n /// Returns a new `Vec2`,\r\n /// with `x` and `y` rounded up to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Vec2;\r\n /// let a = Vec2::new(3.3, 3.6).ceil();\r\n /// let b = Vec2::new(3.0, -3.1).ceil();\r\n /// assert_eq!(a.x, 4.0);\r\n /// assert_eq!(a.y, 4.0);\r\n /// assert_eq!(b.x, 3.0);\r\n /// assert_eq!(b.y, -3.0);\r\n /// ```\r\n #[inline]\r\n pub fn ceil(self) -> Vec2 {\r\n Vec2::new(self.x.ceil(), self.y.ceil())\r\n }\r\n\r\n /// Returns a new `Vec2`,\r\n /// with `x` and `y` rounded down to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Vec2;\r\n /// let a = Vec2::new(3.3, 3.6).floor();\r\n /// let b = Vec2::new(3.0, -3.1).floor();\r\n /// assert_eq!(a.x, 3.0);\r\n /// assert_eq!(a.y, 3.0);\r\n /// assert_eq!(b.x, 3.0);\r\n /// assert_eq!(b.y, -4.0);\r\n /// ```\r\n #[inline]\r\n pub fn floor(self) -> Vec2 {\r\n Vec2::new(self.x.floor(), self.y.floor())\r\n }\r\n\r\n /// Returns a new `Vec2`,\r\n /// with `x` and `y` rounded away from zero to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Vec2;\r\n /// let a = Vec2::new(3.3, 3.6).expand();\r\n /// let b = Vec2::new(3.0, -3.1).expand();\r\n /// assert_eq!(a.x, 4.0);\r\n /// assert_eq!(a.y, 4.0);\r\n /// assert_eq!(b.x, 3.0);\r\n /// assert_eq!(b.y, -4.0);\r\n /// ```\r\n #[inline]\r\n pub fn expand(self) -> Vec2 {\r\n Vec2::new(self.x.expand(), self.y.expand())\r\n }\r\n\r\n /// Returns a new `Vec2`,\r\n /// with `x` and `y` rounded towards zero to the nearest integer,\r\n /// unless they are already an integer.\r\n ///\r\n /// # Examples\r\n ///\r\n /// ```\r\n /// use kurbo::Vec2;\r\n /// let a = Vec2::new(3.3, 3.6).trunc();\r\n /// let b = Vec2::new(3.0, -3.1).trunc();\r\n /// assert_eq!(a.x, 3.0);\r\n /// assert_eq!(a.y, 3.0);\r\n /// assert_eq!(b.x, 3.0);\r\n /// assert_eq!(b.y, -3.0);\r\n /// ```\r\n #[inline]\r\n pub fn trunc(self) -> Vec2 {\r\n Vec2::new(self.x.trunc(), self.y.trunc())\r\n }\r\n\r\n /// Is this Vec2 finite?\r\n #[inline]\r\n pub fn is_finite(self) -> bool {\r\n self.x.is_finite() && self.y.is_finite()\r\n }\r\n\r\n /// Is this Vec2 NaN?\r\n #[inline]\r\n pub fn is_nan(self) -> bool {\r\n self.x.is_nan() || self.y.is_nan()\r\n }\r\n\r\n /// Divides this Vec2 by a scalar.\r\n ///\r\n /// Unlike the division by scalar operator, which multiplies by the\r\n /// reciprocal for performance, this performs the division\r\n /// per-component for consistent rounding behavior.\r\n pub(crate) fn div_exact(self, divisor: f64) -> Vec2 {\r\n Vec2 {\r\n x: self.x / divisor,\r\n y: self.y / divisor,\r\n }\r\n }\r\n}\r\n\r\nimpl From<(f64, f64)> for Vec2 {\r\n #[inline]\r\n fn from(v: (f64, f64)) -> Vec2 {\r\n Vec2 { x: v.0, y: v.1 }\r\n }\r\n}\r\n\r\nimpl From for (f64, f64) {\r\n #[inline]\r\n fn from(v: Vec2) -> (f64, f64) {\r\n (v.x, v.y)\r\n }\r\n}\r\n\r\nimpl Add for Vec2 {\r\n type Output = Vec2;\r\n\r\n #[inline]\r\n fn add(self, other: Vec2) -> Vec2 {\r\n Vec2 {\r\n x: self.x + other.x,\r\n y: self.y + other.y,\r\n }\r\n }\r\n}\r\n\r\nimpl AddAssign for Vec2 {\r\n #[inline]\r\n fn add_assign(&mut self, other: Vec2) {\r\n *self = Vec2 {\r\n x: self.x + other.x,\r\n y: self.y + other.y,\r\n }\r\n }\r\n}\r\n\r\nimpl Sub for Vec2 {\r\n type Output = Vec2;\r\n\r\n #[inline]\r\n fn sub(self, other: Vec2) -> Vec2 {\r\n Vec2 {\r\n x: self.x - other.x,\r\n y: self.y - other.y,\r\n }\r\n }\r\n}\r\n\r\nimpl SubAssign for Vec2 {\r\n #[inline]\r\n fn sub_assign(&mut self, other: Vec2) {\r\n *self = Vec2 {\r\n x: self.x - other.x,\r\n y: self.y - other.y,\r\n }\r\n }\r\n}\r\n\r\nimpl Mul for Vec2 {\r\n type Output = Vec2;\r\n\r\n #[inline]\r\n fn mul(self, other: f64) -> Vec2 {\r\n Vec2 {\r\n x: self.x * other,\r\n y: self.y * other,\r\n }\r\n }\r\n}\r\n\r\nimpl MulAssign for Vec2 {\r\n #[inline]\r\n fn mul_assign(&mut self, other: f64) {\r\n *self = Vec2 {\r\n x: self.x * other,\r\n y: self.y * other,\r\n };\r\n }\r\n}\r\n\r\nimpl Mul for f64 {\r\n type Output = Vec2;\r\n\r\n #[inline]\r\n fn mul(self, other: Vec2) -> Vec2 {\r\n other * self\r\n }\r\n}\r\n\r\nimpl Div for Vec2 {\r\n type Output = Vec2;\r\n\r\n /// Note: division by a scalar is implemented by multiplying by the reciprocal.\r\n ///\r\n /// This is more efficient but has different roundoff behavior than division.\r\n #[inline]\r\n #[allow(clippy::suspicious_arithmetic_impl)]\r\n fn div(self, other: f64) -> Vec2 {\r\n self * other.recip()\r\n }\r\n}\r\n\r\nimpl DivAssign for Vec2 {\r\n #[inline]\r\n fn div_assign(&mut self, other: f64) {\r\n self.mul_assign(other.recip());\r\n }\r\n}\r\n\r\nimpl Neg for Vec2 {\r\n type Output = Vec2;\r\n\r\n #[inline]\r\n fn neg(self) -> Vec2 {\r\n Vec2 {\r\n x: -self.x,\r\n y: -self.y,\r\n }\r\n }\r\n}\r\n\r\nimpl fmt::Display for Vec2 {\r\n fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\r\n write!(formatter, \"𝐯=(\")?;\r\n fmt::Display::fmt(&self.x, formatter)?;\r\n write!(formatter, \", \")?;\r\n fmt::Display::fmt(&self.y, formatter)?;\r\n write!(formatter, \")\")\r\n }\r\n}\r\n\r\n// Conversions to and from mint\r\n#[cfg(feature = \"mint\")]\r\nimpl From for mint::Vector2 {\r\n #[inline]\r\n fn from(p: Vec2) -> mint::Vector2 {\r\n mint::Vector2 { x: p.x, y: p.y }\r\n }\r\n}\r\n\r\n#[cfg(feature = \"mint\")]\r\nimpl From> for Vec2 {\r\n #[inline]\r\n fn from(p: mint::Vector2) -> Vec2 {\r\n Vec2 { x: p.x, y: p.y }\r\n }\r\n}\r\n\r\n#[cfg(test)]\r\nmod tests {\r\n use super::*;\r\n #[test]\r\n fn display() {\r\n let v = Vec2::new(1.2332421, 532.10721213123);\r\n let s = format!(\"{v:.2}\");\r\n assert_eq!(s.as_str(), \"𝐯=(1.23, 532.11)\");\r\n }\r\n}\r\n" }, { "package": "kurbo@0.10.4", "sourcePath": "benches/cubic_arclen.rs", "sha256": "4e5071f1ee932b0185957b44d51285d2bc7336c0d2b39d50e69b278e8838b62f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Benchmarks of cubic arclength approaches.\n\n#![cfg(nightly)]\n#![feature(test)]\nextern crate test;\nuse test::Bencher;\n\nuse kurbo::{CubicBez, ParamCurveArclen};\n\n#[bench]\nfn bench_cubic_arclen_1e_4(b: &mut Bencher) {\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\n b.iter(|| test::black_box(c).arclen(1e-4))\n}\n\n#[bench]\nfn bench_cubic_arclen_1e_5(b: &mut Bencher) {\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\n b.iter(|| test::black_box(c).arclen(1e-5))\n}\n\n#[bench]\nfn bench_cubic_arclen_1e_6(b: &mut Bencher) {\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\n b.iter(|| test::black_box(c).arclen(1e-6))\n}\n\n#[bench]\nfn bench_cubic_arclen_degenerate(b: &mut Bencher) {\n let c = CubicBez::new((0.0, 0.0), (0.0, 0.0), (0.0, 0.0), (0.0, 0.0));\n b.iter(|| test::black_box(c).arclen(1e-6))\n}\n\n#[bench]\nfn bench_cubic_arclen_1e_7(b: &mut Bencher) {\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\n b.iter(|| test::black_box(c).arclen(1e-7))\n}\n\n#[bench]\nfn bench_cubic_arclen_1e_8(b: &mut Bencher) {\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\n b.iter(|| test::black_box(c).arclen(1e-8))\n}\n\n#[bench]\nfn bench_cubic_arclen_1e_9(b: &mut Bencher) {\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 1.0), (1.0, 1.0));\n b.iter(|| test::black_box(c).arclen(1e-9))\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "benches/quad_arclen.rs", "sha256": "c6828b1c2eaa63a9d9ff99e20808d1905c5fef7891c8c5df44cb2f88a89a403d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Benchmarks of quadratic arclength approaches.\n\n// TODO: organize so there's less cut'n'paste from arclen_accuracy example.\n\n#![cfg(nightly)]\n#![feature(test)]\nextern crate test;\nuse test::Bencher;\n\nuse kurbo::{ParamCurve, ParamCurveArclen, ParamCurveDeriv, QuadBez};\n\n// Based on http://www.malczak.linuxpl.com/blog/quadratic-bezier-curve-length/\nfn quad_arclen_analytical(q: QuadBez) -> f64 {\n let d2 = q.p0.to_vec2() - 2.0 * q.p1.to_vec2() + q.p2.to_vec2();\n let a = d2.hypot2();\n let d1 = q.p1 - q.p0;\n let b = 2.0 * d2.dot(d1);\n let c = d1.hypot2();\n\n let sabc = (a + b + c).sqrt();\n let a2 = a.powf(-0.5);\n let a32 = a2.powi(3);\n let c2 = 2.0 * c.sqrt();\n let ba = b * a2;\n\n sabc + 0.25\n * (a2 * a2 * b * (2.0 * sabc - c2)\n + a32 * (4.0 * c * a - b * b) * (((2.0 * a + b) * a2 + 2.0 * sabc) / (ba + c2)).ln())\n}\n\n/// Calculate arclength using Gauss-Legendre quadrature using formula from Behdad\n/// in https://github.com/Pomax/BezierInfo-2/issues/77\nfn gauss_arclen_3(q: QuadBez) -> f64 {\n let v0 = (-0.492943519233745 * q.p0.to_vec2()\n + 0.430331482911935 * q.p1.to_vec2()\n + 0.0626120363218102 * q.p2.to_vec2())\n .hypot();\n let v1 = ((q.p2 - q.p0) * 0.4444444444444444).hypot();\n let v2 = (-0.0626120363218102 * q.p0.to_vec2() - 0.430331482911935 * q.p1.to_vec2()\n + 0.492943519233745 * q.p2.to_vec2())\n .hypot();\n v0 + v1 + v2\n}\n\nfn awesome_quad_arclen3(q: QuadBez, accuracy: f64, depth: usize) -> f64 {\n let pm = q.p0.midpoint(q.p2);\n let d1 = q.p1 - pm;\n let d = q.p2 - q.p0;\n let dhypot2 = d.hypot2();\n let x = 2.0 * d.dot(d1) / dhypot2;\n let y = 2.0 * d.cross(d1) / dhypot2;\n let lc = (q.p2 - q.p0).hypot();\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\n let est_err = 0.06 * (lp - lc) * (x * x + y * y).powf(2.0);\n if est_err < accuracy || depth == 16 {\n gauss_arclen_3(q)\n } else {\n let (q0, q1) = q.subdivide();\n awesome_quad_arclen3(q0, accuracy * 0.5, depth + 1)\n + awesome_quad_arclen3(q1, accuracy * 0.5, depth + 1)\n }\n}\n\n/// Another implementation, using weights from\n/// https://pomax.github.io/bezierinfo/legendre-gauss.html\nfn gauss_arclen_n(q: QuadBez, coeffs: &[(f64, f64)]) -> f64 {\n let d = q.deriv();\n coeffs\n .iter()\n .map(|(wi, xi)| wi * d.eval(0.5 * (xi + 1.0)).to_vec2().hypot())\n .sum::()\n * 0.5\n}\n\nfn gauss_arclen_7(q: QuadBez) -> f64 {\n gauss_arclen_n(\n q,\n &[\n (0.4179591836734694, 0.0000000000000000),\n (0.3818300505051189, 0.4058451513773972),\n (0.3818300505051189, -0.4058451513773972),\n (0.2797053914892766, -0.7415311855993945),\n (0.2797053914892766, 0.7415311855993945),\n (0.1294849661688697, -0.9491079123427585),\n (0.1294849661688697, 0.9491079123427585),\n ],\n )\n}\n\nfn gauss_arclen_24(q: QuadBez) -> f64 {\n gauss_arclen_n(\n q,\n &[\n (0.1279381953467522, -0.0640568928626056),\n (0.1279381953467522, 0.0640568928626056),\n (0.1258374563468283, -0.1911188674736163),\n (0.1258374563468283, 0.1911188674736163),\n (0.1216704729278034, -0.3150426796961634),\n (0.1216704729278034, 0.3150426796961634),\n (0.1155056680537256, -0.4337935076260451),\n (0.1155056680537256, 0.4337935076260451),\n (0.1074442701159656, -0.5454214713888396),\n (0.1074442701159656, 0.5454214713888396),\n (0.0976186521041139, -0.6480936519369755),\n (0.0976186521041139, 0.6480936519369755),\n (0.0861901615319533, -0.7401241915785544),\n (0.0861901615319533, 0.7401241915785544),\n (0.0733464814110803, -0.8200019859739029),\n (0.0733464814110803, 0.8200019859739029),\n (0.0592985849154368, -0.8864155270044011),\n (0.0592985849154368, 0.8864155270044011),\n (0.0442774388174198, -0.9382745520027328),\n (0.0442774388174198, 0.9382745520027328),\n (0.0285313886289337, -0.9747285559713095),\n (0.0285313886289337, 0.9747285559713095),\n (0.0123412297999872, -0.9951872199970213),\n (0.0123412297999872, 0.9951872199970213),\n ],\n )\n}\n\nfn awesome_quad_arclen7(q: QuadBez, accuracy: f64, depth: usize) -> f64 {\n let pm = q.p0.midpoint(q.p2);\n let d1 = q.p1 - pm;\n let d = q.p2 - q.p0;\n let dhypot2 = d.hypot2();\n let x = 2.0 * d.dot(d1) / dhypot2;\n let y = 2.0 * d.cross(d1) / dhypot2;\n let lc = dhypot2.sqrt();\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\n let est_err = 2.5e-2 * (lp - lc) * (x * x + y * y).powf(8.0).tanh();\n if est_err < accuracy || depth == 16 {\n gauss_arclen_7(q)\n } else {\n let (q0, q1) = q.subdivide();\n awesome_quad_arclen7(q0, accuracy * 0.5, depth + 1)\n + awesome_quad_arclen7(q1, accuracy * 0.5, depth + 1)\n }\n}\n\n#[bench]\nfn bench_quad_arclen_analytical(b: &mut Bencher) {\n // Analytical solution is not sensitive to exact shape.\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\n b.iter(|| quad_arclen_analytical(test::black_box(q)))\n}\n\nconst ACCURACY: f64 = 1e-6;\n\n#[bench]\nfn bench_quad_arclen(b: &mut Bencher) {\n // This is a pretty easy case.\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\n b.iter(|| (test::black_box(q).arclen(ACCURACY)))\n}\n\n#[bench]\nfn bench_quad_arclen_gauss3(b: &mut Bencher) {\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\n b.iter(|| awesome_quad_arclen3(test::black_box(q), ACCURACY, 0))\n}\n\n#[bench]\nfn bench_quad_arclen_gauss7(b: &mut Bencher) {\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\n b.iter(|| awesome_quad_arclen7(test::black_box(q), ACCURACY, 0))\n}\n\n#[bench]\nfn bench_quad_arclen_gauss3_one(b: &mut Bencher) {\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\n b.iter(|| gauss_arclen_3(test::black_box(q)))\n}\n\n#[bench]\nfn bench_quad_arclen_gauss7_one(b: &mut Bencher) {\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\n b.iter(|| gauss_arclen_7(test::black_box(q)))\n}\n\n#[bench]\nfn bench_quad_arclen_gauss24_one(b: &mut Bencher) {\n let q = QuadBez::new((0.0, 0.0), (1.0, 0.0), (1.0, 1.0));\n b.iter(|| gauss_arclen_24(test::black_box(q)))\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "benches/quartic.rs", "sha256": "d5e857f92418cd16db26ed27b29692af8c0fe6d228fe8acc29e3e1b2a51334ab", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Benchmarks of the quartic equation solver.\n\n#![cfg(nightly)]\n#![feature(test)]\nextern crate test;\nuse kurbo::common::solve_quartic;\nuse test::Bencher;\n\n#[bench]\nfn bench_quartic(bb: &mut Bencher) {\n let (x1, x2, x3, x4) = (1.0, 2.0, 3.0, 4.0);\n let a = -(x1 + x2 + x3 + x4);\n let b = x1 * (x2 + x3) + x2 * (x3 + x4) + x4 * (x1 + x3);\n let c = -x1 * x2 * (x3 + x4) - x3 * x4 * (x1 + x2);\n let d = x1 * x2 * x3 * x4;\n\n bb.iter(|| {\n solve_quartic(\n test::black_box(d),\n test::black_box(c),\n test::black_box(b),\n test::black_box(a),\n 1.0,\n )\n })\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "benches/rect_expand.rs", "sha256": "64e066afc1e03e3464b6653390b8ce1bfe047aa048ff673657797b89c46ee831", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![cfg(nightly)]\n#![feature(test)]\nextern crate test;\nuse test::Bencher;\n\nuse kurbo::Rect;\n\n// In positive space\nstatic RECT_POS: Rect = Rect::new(3.3, 3.6, 5.6, 4.1);\n// In both positive and negative space\nstatic RECT_PAN: Rect = Rect::new(-3.3, -3.6, 5.6, 4.1);\n// In negative space\nstatic RECT_NEG: Rect = Rect::new(-5.6, -4.1, -3.3, -3.6);\n// In positive space reverse\nstatic RECT_POR: Rect = Rect::new(5.6, 4.1, 3.3, 3.6);\n// In both positive and negative space reverse\nstatic RECT_PNR: Rect = Rect::new(5.6, 4.1, -3.3, -3.6);\n// In negative space reverse\nstatic RECT_NER: Rect = Rect::new(-3.3, -3.6, -5.6, -4.1);\n// In positive space mixed\nstatic RECT_POM: Rect = Rect::new(3.3, 4.1, 5.6, 3.6);\n// In both positive and negative space mixed\nstatic RECT_PNM: Rect = Rect::new(-3.3, 4.1, 5.6, -3.6);\n// In negative space mixed\nstatic RECT_NEM: Rect = Rect::new(-5.6, -3.6, -3.3, -4.1);\n\n#[inline]\nfn expand(rects: &[Rect; 9]) {\n test::black_box(rects[0].expand());\n test::black_box(rects[1].expand());\n test::black_box(rects[2].expand());\n\n test::black_box(rects[3].expand());\n test::black_box(rects[4].expand());\n test::black_box(rects[5].expand());\n\n test::black_box(rects[6].expand());\n test::black_box(rects[7].expand());\n test::black_box(rects[8].expand());\n}\n\n#[bench]\nfn bench_expand(b: &mut Bencher) {\n // Creating the array here to prevent the compiler from optimizing all of it to NOP.\n let rects: [Rect; 9] = [\n RECT_POS, RECT_PAN, RECT_NEG, RECT_POR, RECT_PNR, RECT_NER, RECT_POM, RECT_PNM, RECT_NEM,\n ];\n b.iter(|| expand(&rects));\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "examples/arclen_accuracy.rs", "sha256": "20a190ab7ae1d36a098b3b209f9b216aa95fa4654e62bb2375748c67b483ebc1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A test program to plot the error of arclength approximation.\n\n// Lots of stuff is commented out or was just something to try.\n#![allow(unused)]\n#![allow(clippy::unreadable_literal)]\n#![allow(clippy::many_single_char_names)]\n\n// TODO: make more functionality accessible from command line rather than uncommenting.\n\nuse std::env;\nuse std::time::{Duration, Instant};\n\nuse kurbo::{ParamCurve, ParamCurveArclen, ParamCurveDeriv, QuadBez};\n\nuse kurbo::common::{GAUSS_LEGENDRE_COEFFS_24, GAUSS_LEGENDRE_COEFFS_5, GAUSS_LEGENDRE_COEFFS_7};\n\n/// Calculate arclength using Gauss-Legendre quadrature using formula from Behdad\n/// in https://github.com/Pomax/BezierInfo-2/issues/77\nfn gauss_arclen_3(q: QuadBez) -> f64 {\n let v0 = (-0.492943519233745 * q.p0.to_vec2()\n + 0.430331482911935 * q.p1.to_vec2()\n + 0.0626120363218102 * q.p2.to_vec2())\n .hypot();\n let v1 = ((q.p2 - q.p0) * 0.4444444444444444).hypot();\n let v2 = (-0.0626120363218102 * q.p0.to_vec2() - 0.430331482911935 * q.p1.to_vec2()\n + 0.492943519233745 * q.p2.to_vec2())\n .hypot();\n v0 + v1 + v2\n}\n\nfn awesome_quad_arclen(q: QuadBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\n let pm = q.p0.midpoint(q.p2);\n let d1 = q.p1 - pm;\n let d = q.p2 - q.p0;\n let dhypot2 = d.hypot2();\n let x = 2.0 * d.dot(d1) / dhypot2;\n let y = 2.0 * d.cross(d1) / dhypot2;\n let lc = (q.p2 - q.p0).hypot();\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\n let est_err = 0.06 * (lp - lc) * (x * x + y * y).powf(2.0);\n if est_err < accuracy || depth == 16 {\n *count += 1;\n gauss_arclen_3(q)\n } else {\n let (q0, q1) = q.subdivide();\n awesome_quad_arclen(q0, accuracy * 0.5, depth + 1, count)\n + awesome_quad_arclen(q1, accuracy * 0.5, depth + 1, count)\n }\n}\n\nconst MAX_DEPTH: usize = 16;\nfn gravesen_rec(q: &QuadBez, l0: f64, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\n let (q0, q1) = q.subdivide();\n let l0_q0 = calc_l0(q0);\n let l0_q1 = calc_l0(q1);\n let l1 = l0_q0 + l0_q1;\n let error = (l0 - l1) * (1.0 / 15.0);\n if error.abs() < accuracy || depth == MAX_DEPTH {\n *count += 1;\n l1 - error\n } else {\n gravesen_rec(&q0, l0_q0, accuracy * 0.5, depth + 1, count)\n + gravesen_rec(&q1, l0_q1, accuracy * 0.5, depth + 1, count)\n }\n}\n\nfn gauss_arclen_5(q: QuadBez) -> f64 {\n q.gauss_arclen(GAUSS_LEGENDRE_COEFFS_5)\n}\n\nfn gauss_arclen_7(q: QuadBez) -> f64 {\n q.gauss_arclen(GAUSS_LEGENDRE_COEFFS_7)\n}\n\nfn gauss_arclen_24(q: QuadBez) -> f64 {\n q.gauss_arclen(GAUSS_LEGENDRE_COEFFS_24)\n}\n\nfn awesome_quad_arclen7(q: QuadBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\n let pm = q.p0.midpoint(q.p2);\n let d1 = q.p1 - pm;\n let d = q.p2 - q.p0;\n let dhypot2 = d.hypot2();\n let x = 2.0 * d.dot(d1) / dhypot2;\n let y = 2.0 * d.cross(d1) / dhypot2;\n let lc = (q.p2 - q.p0).hypot();\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\n let est_err = 2.5e-2 * (lp - lc) * (x * x + y * y).powf(8.0).tanh();\n // Increase depth, so we can be an accurate baseline for comparison.\n if est_err < accuracy || depth == 22 {\n *count += 1;\n gauss_arclen_7(q)\n } else {\n let (q0, q1) = q.subdivide();\n awesome_quad_arclen7(q0, accuracy * 0.5, depth + 1, count)\n + awesome_quad_arclen7(q1, accuracy * 0.5, depth + 1, count)\n }\n}\n\nfn awesome_quad_arclen24(q: QuadBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\n let pm = q.p0.midpoint(q.p2);\n let d1 = q.p1 - pm;\n let d = q.p2 - q.p0;\n let dhypot2 = d.hypot2();\n let x = 2.0 * d.dot(d1) / dhypot2;\n let y = 2.0 * d.cross(d1) / dhypot2;\n let lc = (q.p2 - q.p0).hypot();\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\n // This isn't quite right near (1.1, 0)\n let est_err = 1.0 * (lp - lc) * (0.5 * x * x + 0.1 * y * y).powf(16.0).tanh();\n if est_err < accuracy || depth == 16 {\n *count += 1;\n gauss_arclen_24(q)\n } else {\n let (q0, q1) = q.subdivide();\n awesome_quad_arclen24(q0, accuracy * 0.5, depth + 1, count)\n + awesome_quad_arclen24(q1, accuracy * 0.5, depth + 1, count)\n }\n}\n\n// Based on http://www.malczak.linuxpl.com/blog/quadratic-bezier-curve-length/\nfn quad_arclen_analytical(q: QuadBez) -> f64 {\n let d2 = q.p0.to_vec2() - 2.0 * q.p1.to_vec2() + q.p2.to_vec2();\n let a = d2.hypot2();\n let d1 = q.p1 - q.p0;\n let c = d1.hypot2();\n if a < 5e-4 * c {\n return gauss_arclen_3(q);\n }\n let b = 2.0 * d2.dot(d1);\n\n let sabc = (a + b + c).sqrt();\n let a2 = a.powf(-0.5);\n let a32 = a2.powi(3);\n let c2 = 2.0 * c.sqrt();\n let ba_c2 = b * a2 + c2;\n\n let v0 = 0.25 * a2 * a2 * b * (2.0 * sabc - c2) + sabc;\n // TODO: justify and fine-tune this exact constant.\n if ba_c2 < 1e-13 {\n // This case happens for Béziers with a sharp kink.\n v0\n } else {\n v0 + 0.25 * a32 * (4.0 * c * a - b * b) * (((2.0 * a + b) * a2 + 2.0 * sabc) / ba_c2).ln()\n }\n}\n\n/// Calculate the L0 metric from \"Adaptive subdivision and the length and\n/// energy of Bézier curves\" by Jens Gravesen.\nfn calc_l0(q: QuadBez) -> f64 {\n let lc = (q.p2 - q.p0).hypot();\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\n (2.0 / 3.0) * lc + (1.0 / 3.0) * lp\n}\n\nfn with_subdiv(q: QuadBez, f: &dyn Fn(QuadBez) -> f64, depth: usize) -> f64 {\n if depth == 0 {\n f(q)\n } else {\n let (q0, q1) = q.subdivide();\n with_subdiv(q0, f, depth - 1) + with_subdiv(q1, f, depth - 1)\n }\n}\n\nfn duration_to_time(d: Duration) -> f64 {\n 1e-9 * (d.subsec_nanos() as f64) + (d.as_secs() as f64)\n}\n\nfn run_simple() {\n let q = QuadBez::new((0.0, 0.0), (0.0, 0.5), (1.0, 1.0));\n let true_len = q.arclen(1e-13);\n for i in 0..20 {\n let n = 1 << i;\n let start_time = Instant::now();\n let mut est = 0.0;\n let mut last = q.start();\n let dt = (n as f64).recip();\n for j in 0..n {\n let t = ((j + 1) as f64) * dt;\n let p = q.eval(t);\n est += (p - last).hypot();\n last = p;\n }\n let elapsed = duration_to_time(start_time.elapsed());\n let err = true_len - est;\n if i > 0 {\n println!(\"{elapsed} {err}\");\n }\n }\n}\n\n/// Generate map data suitable for plotting in Gnuplot.\nfn main() {\n let mut n_subdiv = 0;\n let mut func: &dyn Fn(QuadBez) -> f64 = &gauss_arclen_3;\n for arg in env::args().skip(1) {\n if arg == \"gauss3\" {\n func = &gauss_arclen_3;\n } else if arg == \"gauss5\" {\n func = &gauss_arclen_5;\n } else if arg == \"gauss7\" {\n func = &gauss_arclen_7;\n } else if arg == \"gauss24\" {\n func = &gauss_arclen_24;\n } else if arg == \"l0\" {\n func = &calc_l0;\n } else if arg == \"simple\" {\n run_simple();\n return;\n } else if let Ok(n) = arg.parse() {\n n_subdiv = n;\n } else {\n println!(\"usage: arclen_accuracy [func] n_subdiv\");\n std::process::exit(1);\n }\n }\n let n = 400;\n let accuracy = 1e-6;\n for i in 0..=n {\n let x = 2.0 * (i as f64) * (n as f64).recip();\n for j in 0..=n {\n let y = 2.0 * (j as f64) * (n as f64).recip();\n let q = QuadBez::new((-1.0, 0.0), (x, y), (1.0, 0.0));\n let mut count = 0;\n let accurate_arclen = awesome_quad_arclen7(q, 1e-15, 0, &mut count);\n //count = 0;\n let start_time = Instant::now();\n //let est = awesome_quad_arclen7(q, accuracy, 0, &mut count);\n let est = quad_arclen_analytical(q);\n let elapsed = start_time.elapsed();\n //let est = gravesen_rec(&q, calc_l0(q), accuracy, 0, &mut count);\n //let est = with_subdiv(q, func, n_subdiv);\n let error = est - accurate_arclen;\n println!(\"{} {} {}\", x, y, (error.abs() + 1e-18).log10());\n //println!(\"{} {} {}\", x, y, count);\n //println!(\"{} {} {}\", x, y, elapsed.subsec_nanos());\n //let accurate_arclen = with_subdiv(q, &gauss_arclen_5, 8);\n //let error = est - accurate_arclen;\n /*\n let lc = (q.p2 - q.p0).hypot();\n let lp = (q.p1 - q.p0).hypot() + (q.p2 - q.p1).hypot();\n let est_err = 1.0 * (lp - lc) * (0.5 * x * x + 0.1 * y * y).powf(16.0).tanh();\n println!(\"{} {} {}\", x, y, (est_err/error.abs() + 1e-15).log10());\n */\n }\n println!();\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "examples/circle.rs", "sha256": "bed0de2cc5e772ad9b6558ad75437b46a309b743475aa65d1b903abe88e32cc1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Example of circle\n\n#[cfg(feature = \"std\")]\nfn main() {\n use kurbo::{Circle, Shape};\n\n let circle = Circle::new((400.0, 400.0), 380.0);\n println!(\"\");\n println!(\"\");\n println!(\"\");\n println!(\"\");\n let path = circle.to_path(1e-3).to_svg();\n println!(\" \", path);\n let path = circle.to_path(1.0).to_svg();\n println!(\" \", path);\n println!(\"\");\n println!(\"\");\n println!(\"\");\n}\n\n#[cfg(not(feature = \"std\"))]\nfn main() {\n println!(\"This example requires the standard library\");\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "examples/cubic_arclen.rs", "sha256": "8ce8010e0f46347db366b666c253db6d3550e64c0993c615029d408d6bfd5462", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Research testbed for arclengths of cubic Bézier segments.\n\n// Lots of stuff is commented out or was just something to try.\n#![allow(unused)]\n#![allow(clippy::unreadable_literal)]\n#![allow(clippy::many_single_char_names)]\n\nuse kurbo::common::*;\nuse kurbo::{\n CubicBez, ParamCurve, ParamCurveArclen, ParamCurveCurvature, ParamCurveDeriv, Point, Vec2,\n};\n\n/// Calculate arclength using Gauss-Legendre quadrature using formula from Behdad\n/// in https://github.com/Pomax/BezierInfo-2/issues/77\nfn gauss_arclen_5(c: CubicBez) -> f64 {\n let v0 = (c.p1 - c.p0).hypot() * 0.15;\n let v1 = (-0.558983582205757 * c.p0.to_vec2()\n + 0.325650248872424 * c.p1.to_vec2()\n + 0.208983582205757 * c.p2.to_vec2()\n + 0.024349751127576 * c.p3.to_vec2())\n .hypot();\n let v2 = (c.p3 - c.p0 + (c.p2 - c.p1)).hypot() * 0.26666666666666666;\n let v3 = (-0.024349751127576 * c.p0.to_vec2()\n - 0.208983582205757 * c.p1.to_vec2()\n - 0.325650248872424 * c.p2.to_vec2()\n + 0.558983582205757 * c.p3.to_vec2())\n .hypot();\n let v4 = (c.p3 - c.p2).hypot() * 0.15;\n\n v0 + v1 + v2 + v3 + v4\n}\n\nfn gauss_arclen_7(c: C) -> f64 {\n c.gauss_arclen(GAUSS_LEGENDRE_COEFFS_7)\n}\n\nfn est_gauss5_error(c: CubicBez) -> f64 {\n let lc = (c.p3 - c.p0).hypot();\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\n\n let d2 = c.deriv().deriv();\n let d3 = d2.deriv();\n let lmi = 2.0 / (lp + lc);\n 7e-8 * (d3.eval(0.5).to_vec2().hypot() * lmi + 5.0 * d2.eval(0.5).to_vec2().hypot() * lmi)\n .powi(5)\n * lp\n}\n\nfn gauss_errnorm_n(c: C, coeffs: &[(f64, f64)]) -> f64\nwhere\n C::DerivResult: ParamCurveDeriv,\n{\n let d = c.deriv().deriv();\n coeffs\n .iter()\n .map(|(wi, xi)| wi * d.eval(0.5 * (xi + 1.0)).to_vec2().hypot2())\n .sum::()\n}\n\n// Squared L2 norm of the second derivative of the cubic.\nfn cubic_errnorm(c: CubicBez) -> f64 {\n let d = c.deriv().deriv();\n let dd = d.end() - d.start();\n d.start().to_vec2().hypot2() + d.start().to_vec2().dot(dd) + dd.hypot2() * (1.0 / 3.0)\n}\n\nfn est_gauss7_error(c: CubicBez) -> f64 {\n let lc = (c.p3 - c.p0).hypot();\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\n\n 8e-9 * (2.0 * cubic_errnorm(c) / lc.powi(2)).powi(6) * lp\n}\n\nfn gauss_arclen_9(c: C) -> f64 {\n c.gauss_arclen(GAUSS_LEGENDRE_COEFFS_9)\n}\n\nfn gauss_arclen_11(c: C) -> f64 {\n c.gauss_arclen(GAUSS_LEGENDRE_COEFFS_11)\n}\n\nfn est_gauss9_error(c: CubicBez) -> f64 {\n let lc = (c.p3 - c.p0).hypot();\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\n\n 1e-10 * (2.0 * cubic_errnorm(c) / lc.powi(2)).powi(8) * lp\n}\n\nfn est_gauss11_error(c: CubicBez) -> f64 {\n let lc = (c.p3 - c.p0).hypot();\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\n\n 1e-12 * (2.0 * cubic_errnorm(c) / lc.powi(2)).powi(11) * lp\n}\n\n// A new approach based on integrating local error.\nfn est_gauss11_error_2(c: CubicBez) -> f64 {\n let d = c.deriv();\n let d2 = d.deriv();\n GAUSS_LEGENDRE_COEFFS_11\n .iter()\n .map(|(wi, xi)| {\n wi * {\n let t = 0.5 * (xi + 1.0);\n let v = d.eval(t).to_vec2().hypot();\n let a2 = d2.eval(t).to_vec2().hypot2();\n a2.powi(3) / v.powi(5)\n }\n })\n .sum::()\n}\n\n#[allow(clippy::neg_cmp_op_on_partial_ord)]\nfn est_max_curvature(c: CubicBez) -> f64 {\n let n = 100;\n let mut max = 0.0;\n for i in 0..=n {\n let t = (i as f64) * (n as f64).recip();\n let k = c.curvature(t).abs();\n if !(k < max) {\n max = k;\n }\n }\n max\n}\n\nfn est_min_deriv_norm2(c: CubicBez) -> f64 {\n let d = c.deriv();\n let n = 100;\n let mut min = d.eval(1.0).to_vec2().hypot2();\n for i in 0..n {\n let t = (i as f64) * (n as f64).recip();\n min = min.min(d.eval(t).to_vec2().hypot2())\n }\n min\n}\n\nfn est_gauss11_error_3(c: CubicBez) -> f64 {\n let lc = (c.p3 - c.p0).hypot();\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\n let pc_err = (lp - lc) * 0.02;\n let ks = est_max_curvature(c) * lp;\n let est = ks.powi(3) * lp * 8e-9;\n if est < pc_err {\n est\n } else {\n pc_err\n }\n}\n\nfn est_gauss9_error_3(c: CubicBez) -> f64 {\n let lc = (c.p3 - c.p0).hypot();\n let lp = (c.p1 - c.p0).hypot() + (c.p2 - c.p1).hypot() + (c.p3 - c.p2).hypot();\n let pc_err = (lp - lc) * 0.02;\n let ks = est_max_curvature(c) * lp;\n let est = ks.powi(3) * lp * 5e-8;\n if est < pc_err {\n est\n } else {\n pc_err\n }\n}\n\n// A new approach based on integrating local error; the cost of evaluating the\n// error metric is likely to dominate unless the accuracy buys a lot of subdivisions.\nfn est_gauss9_error_2(c: CubicBez) -> f64 {\n let d = c.deriv();\n let d2 = d.deriv();\n let p = 10;\n GAUSS_LEGENDRE_COEFFS_9\n .iter()\n .map(|(wi, xi)| {\n wi * {\n let t = 0.5 * (xi + 1.0);\n let v = d.eval(t).to_vec2().hypot();\n let a = d2.eval(t).to_vec2().hypot();\n (1.0e-1 * a / v).tanh().powi(p) * v\n }\n })\n .sum::()\n * 3.0\n}\n\nfn my_arclen(c: CubicBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\n if depth == 16 || est_gauss5_error(c) < accuracy {\n *count += 1;\n gauss_arclen_5(c)\n } else {\n let (c0, c1) = c.subdivide();\n my_arclen(c0, accuracy * 0.5, depth + 1, count)\n + my_arclen(c1, accuracy * 0.5, depth + 1, count)\n }\n}\n\nfn my_arclen7(c: CubicBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\n if depth == 16 || est_gauss7_error(c) < accuracy {\n *count += 1;\n gauss_arclen_7(c)\n } else {\n let (c0, c1) = c.subdivide();\n my_arclen7(c0, accuracy * 0.5, depth + 1, count)\n + my_arclen7(c1, accuracy * 0.5, depth + 1, count)\n }\n}\n\n// Should make this generic instead of copy+paste, but we need only one when we're done.\nfn my_arclen9(c: CubicBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\n if depth == 16 || est_gauss9_error(c) < accuracy {\n *count += 1;\n gauss_arclen_9(c)\n } else {\n let (c0, c1) = c.subdivide();\n my_arclen9(c0, accuracy * 0.5, depth + 1, count)\n + my_arclen9(c1, accuracy * 0.5, depth + 1, count)\n }\n}\n\n// This doesn't help; we can't really get a more accurate error bound, so all this\n// does is overkill the accuracy.\nfn my_arclen11(c: CubicBez, accuracy: f64, depth: usize, count: &mut usize) -> f64 {\n if depth == 16 || est_gauss9_error(c) < accuracy {\n *count += 1;\n gauss_arclen_11(c)\n } else {\n let (c0, c1) = c.subdivide();\n my_arclen11(c0, accuracy * 0.5, depth + 1, count)\n + my_arclen11(c1, accuracy * 0.5, depth + 1, count)\n }\n}\n\nfn randpt() -> Point {\n Point::new(rand::random(), rand::random())\n}\n\nfn randbez() -> CubicBez {\n CubicBez::new(randpt(), randpt(), randpt(), randpt())\n}\n\nfn main() {\n let accuracy = 1e-4;\n for _ in 0..10_000 {\n let c = randbez();\n let t: f64 = rand::random();\n let c = c.subsegment(0.0..t);\n //let accurate_arclen = c.arclen(1e-12);\n let mut count = 0;\n let accurate_arclen = my_arclen9(c, 1e-15, 0, &mut count);\n\n let est = gauss_arclen_9(c);\n let est_err = est_gauss9_error_3(c);\n let err = (accurate_arclen - est).abs();\n println!(\"{est_err} {err}\");\n\n /*\n let mut count = 0;\n let est = my_arclen9(c, accuracy, 0, &mut count);\n let err = (accurate_arclen - est).abs();\n println!(\"{err} {count}\");\n */\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "examples/ellipse.rs", "sha256": "d737bac238376779127fd7adb889a8a96a4423fed6f24c5bec7f66707c14a7b7", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Example of ellipse\n\n#[cfg(feature = \"std\")]\nfn main() {\n use kurbo::{Ellipse, Shape};\n use std::f64::consts::PI;\n\n let ellipse = Ellipse::new((400.0, 400.0), (200.0, 100.0), 0.25 * PI);\n println!(\"\");\n println!(\"\");\n println!(\"\");\n println!(\"\");\n let path = ellipse.to_path(1e-3).to_svg();\n println!(\" \", path);\n let path = ellipse.to_path(1.0).to_svg();\n println!(\" \", path);\n println!(\"\");\n println!(\"\");\n println!(\"\");\n}\n\n#[cfg(not(feature = \"std\"))]\nfn main() {\n println!(\"This example requires the standard library\");\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "examples/fit_poly.rs", "sha256": "9a313d3b6cc01b38cb9a8e00943a7b81debd7ecbe456527b0c74e1e747c58cea", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A test case for the cubic Bézier path simplifier, generating a very\n//! efficient and accurate SVG file to plot a quartic polynomial. This\n//! file was used to generate the image for the Wikipedia article on\n//! quartic polynomials.\n\nuse std::{io::Write, ops::Range};\n\nuse kurbo::{BezPath, CurveFitSample, ParamCurveFit, PathEl, Point, Vec2};\n\nstruct MyPoly;\n\nfn eval_mypoly(x: f64) -> f64 {\n (x + 4.) * (x + 1.) * (x - 1.) * (x - 3.) / 14. + 0.5\n}\n\nfn eval_mypoly_deriv(x: f64) -> f64 {\n (((4. * x + 3.) * x - 26.) * x - 1.) / 14.\n}\n\nimpl ParamCurveFit for MyPoly {\n fn sample_pt_deriv(&self, t: f64) -> (Point, Vec2) {\n const S: f64 = 336.;\n let x = 37. + t * S;\n let math_x = (x - 220.) / 40.;\n let y = -eval_mypoly(math_x) * 40. + 260.;\n let dx = S;\n let dy = -dx * eval_mypoly_deriv(math_x);\n (Point::new(x, y), Vec2::new(dx, dy))\n }\n\n fn sample_pt_tangent(&self, t: f64, _: f64) -> CurveFitSample {\n let (p, tangent) = self.sample_pt_deriv(t);\n CurveFitSample { p, tangent }\n }\n\n fn break_cusp(&self, _: Range) -> Option {\n None\n }\n}\n\npub fn to_svg_economical(path: &BezPath) -> String {\n let mut buffer = Vec::new();\n write_to(path, &mut buffer).unwrap();\n String::from_utf8(buffer).unwrap()\n}\n\n/// Write the SVG representation of this path to the provided buffer.\npub fn write_to(path: &BezPath, mut writer: W) -> std::io::Result<()> {\n for el in path.elements() {\n match *el {\n PathEl::MoveTo(p) => write!(writer, \"M{:.2} {:.2}\", p.x, p.y)?,\n PathEl::LineTo(p) => write!(writer, \"L{} {}\", p.x, p.y)?,\n PathEl::QuadTo(p1, p2) => write!(writer, \"Q{} {} {} {}\", p1.x, p1.y, p2.x, p2.y)?,\n PathEl::CurveTo(p1, p2, p3) => write!(\n writer,\n \"C{:.2} {:.2} {:.2} {:.2} {:.2} {:.2}\",\n p1.x, p1.y, p2.x, p2.y, p3.x, p3.y\n )?,\n PathEl::ClosePath => write!(writer, \"Z\")?,\n }\n }\n Ok(())\n}\n\nfn main() {\n println!(\"\");\n let path = kurbo::fit_to_bezpath_opt(&MyPoly, 0.1);\n println!(\n \" \",\n to_svg_economical(&path)\n );\n println!(\"\");\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "examples/moment.rs", "sha256": "f4da88da1cbc91d85e58b28aad0aadc0c036510ca8e37671154f33fbd179dc3a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Test case for checking whether numeric and analytic computations of\n//! moments for cubic Béziers match each other.\n\nuse kurbo::{common::GAUSS_LEGENDRE_COEFFS_16, CubicBez, ParamCurve, ParamCurveDeriv};\n\nfn main() {\n let c = CubicBez::new(\n (0.0, 547.3395616558455),\n (266.66666666666663, 741.3396067339968),\n (533.3333333333334, -141.33960673399685),\n (800.0, 52.66043834415456),\n );\n let q = c.deriv();\n let numeric = GAUSS_LEGENDRE_COEFFS_16\n .iter()\n .map(|(wi, xi)| {\n let t = 0.5 + xi * 0.5;\n let p = c.eval(t);\n let d = q.eval(t);\n let a = (0.5 * wi) * d.x * p.y;\n p.y * a\n })\n .sum::();\n println!(\"{:?} {}\", kurbo::simplify::moment_integrals(c), numeric);\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "examples/offset.rs", "sha256": "8f15459091f2223ad5ff647fdca7ba168cbf013030f05f629a7ffb69ba8b15ef", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A simple example to show an offset curve of a cubic Bézier segment.\n\nuse kurbo::{offset::CubicOffset, CubicBez, Shape};\n\nfn main() {\n println!(\"\");\n let c = CubicBez::new((100., 100.), (150., 75.), (300., 50.), (400., 200.));\n println!(\n \" \",\n c.to_path(1e-9).to_svg()\n );\n for i in 1..=80 {\n let co = CubicOffset::new(c, i as f64 * 4.0);\n let path = kurbo::fit_to_bezpath_opt(&co, 1e-3);\n println!(\n \" \",\n path.to_svg()\n );\n }\n println!(\"\");\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "examples/quad_intersect.rs", "sha256": "03def9282c4d50afa4e5d7d19b341ff8970459083bf0dc16c53ad44bdf2f4594", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Test case that demonstrates quadratic/quadratic intersection using\n//! new quartic solver.\n\nuse arrayvec::ArrayVec;\nuse kurbo::{common::solve_quartic, ParamCurve, Point, QuadBez, Shape};\nuse rand::{thread_rng, Rng};\n\nfn rand_point() -> Point {\n let mut rng = thread_rng();\n Point::new(rng.gen_range(0.0..500.0), rng.gen_range(0.0..500.0))\n}\n\nfn rand_quad() -> QuadBez {\n QuadBez::new(rand_point(), rand_point(), rand_point())\n}\n\nstruct ImplicitQuad {\n x2: f64,\n xy: f64,\n y2: f64,\n x: f64,\n y: f64,\n c: f64,\n}\n\nimpl ImplicitQuad {\n fn from_quad(q: QuadBez) -> Self {\n let Point { x: ax, y: ay } = q.p0;\n let Point { x: bx, y: by } = q.p1;\n let Point { x: cx, y: cy } = q.p2;\n let (u0, u1, u2) = (by - cy, cx - bx, bx * cy - by * cx);\n let (v0, v1, v2) = (cy - ay, ax - cx, cx * ay - cy * ax);\n let (w0, w1, w2) = (ay - by, bx - ax, ax * by - ay * bx);\n ImplicitQuad {\n x2: 4. * u0 * w0 - v0 * v0,\n xy: 4. * (u0 * w1 + u1 * w0) - 2. * v0 * v1,\n y2: 4. * u1 * w1 - v1 * v1,\n x: 4. * (u0 * w2 + u2 * w0) - 2. * v0 * v2,\n y: 4. * (u1 * w2 + u2 * w1) - 2. * v1 * v2,\n c: 4. * u2 * w2 - v2 * v2,\n }\n }\n\n fn eval(&self, x: f64, y: f64) -> f64 {\n self.x2 * x * x + self.xy * x * y + self.y2 * y * y + self.x * x + self.y * y + self.c\n }\n}\n\nfn intersect_quads(q0: QuadBez, q1: QuadBez) -> ArrayVec {\n let iq = ImplicitQuad::from_quad(q0);\n let c = q1.p0.to_vec2();\n let b = (q1.p1 - q1.p0) * 2.0;\n let a = c - q1.p1.to_vec2() * 2.0 + q1.p2.to_vec2();\n let c0 = iq.eval(c.x, c.y);\n let c1 = iq.x * b.x\n + iq.y * b.y\n + 2. * iq.x2 * (b.x * c.x)\n + 2. * iq.y2 * (b.y * c.y)\n + iq.xy * (b.x * c.y + b.y * c.x);\n let c2 = iq.x * a.x\n + iq.y * a.y\n + iq.x2 * (2. * a.x * c.x + b.x * b.x)\n + iq.xy * (a.x * c.y + b.x * b.y + a.y * c.x)\n + iq.y2 * (2. * a.y * c.y + b.y * b.y);\n let c3 = iq.x2 * 2. * a.x * b.x + iq.xy * (a.x * b.y + b.x * a.y) + iq.y2 * 2. * a.y * b.y;\n let c4 = iq.x2 * a.x * a.x + iq.xy * a.x * a.y + iq.y2 * a.y * a.y;\n let ts = solve_quartic(c0, c1, c2, c3, c4);\n println!(\"ts: {:?}\", ts);\n ts\n}\n\nfn main() {\n let q0 = rand_quad();\n let q1 = rand_quad();\n println!(\n \" \",\n q0.to_path(1e-9).to_svg()\n );\n println!(\n \" \",\n q1.to_path(1e-9).to_svg()\n );\n for t in intersect_quads(q0, q1) {\n if (0.0..=1.0).contains(&t) {\n let p = q1.eval(t);\n println!(\" \", p.x, p.y);\n }\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "examples/simplify.rs", "sha256": "7bd3fdcfd60fe9094b9f5da51be4a70061f3df760a62a0ae62e7e8c77f649558", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Test case showing simplification of a Bézier path. In this case, the\n//! source path is a simple mathematical function, but it should work for\n//! more general cases as well.\n\nuse kurbo::{BezPath, Point};\n\nfn plot_fn(f: &dyn Fn(f64) -> f64, d: &dyn Fn(f64) -> f64, xa: f64, xb: f64, n: usize) -> BezPath {\n let width = 800.;\n let dx = (xb - xa) / n as f64;\n let xs = width / (xb - xa);\n let ys = 250.;\n let y_origin = 300.;\n let plot = |x, y| Point::new((x - xa) * xs, y_origin - y * ys);\n let mut x0 = xa;\n let mut y0 = f(xa);\n let mut d0 = d(xa);\n let mut path = BezPath::new();\n path.move_to(plot(x0, y0));\n for i in 0..n {\n let x3 = xa + dx * (i + 1) as f64;\n let y3 = f(x3);\n let d3 = d(x3);\n let x1 = x0 + (1. / 3.) * dx;\n let x2 = x3 - (1. / 3.) * dx;\n let y1 = y0 + d0 * (1. / 3.) * dx;\n let y2 = y3 - d3 * (1. / 3.) * dx;\n path.curve_to(plot(x1, y1), plot(x2, y2), plot(x3, y3));\n x0 = x3;\n y0 = y3;\n d0 = d3;\n }\n path\n}\n\nfn main() {\n println!(\"\");\n let path = plot_fn(&|x| x.sin(), &|x| x.cos(), -8., 8., 20);\n println!(\n \" \",\n path.to_svg()\n );\n let simpl = kurbo::simplify::SimplifyBezPath::new(path);\n let simplified_path = kurbo::fit_to_bezpath_opt(&simpl, 0.1);\n println!(\n \" \",\n simplified_path.to_svg()\n );\n println!(\"\");\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "examples_save/stroke.rs", "sha256": "d1afc41f85c92bbfe2bed70c18dbc47ed8d4d961c8e2b9a5730165897fbdc48d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2023 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse kurbo::{stroke, BezPath, Cap, Join, Shape, Stroke};\n\nfn main() {\n let d = \"M100 100 Q150 150 190 110L 50 50\";\n let p = BezPath::from_svg(d).unwrap();\n let style = Stroke::new(5.0)\n .with_caps(Cap::Butt)\n .with_join(Join::Round)\n .with_dashes(15.0, &[10.0, 1.0, 2.0]);\n let s = stroke(p, &style, 1e-1);\n println!(\"{}\", s.to_svg());\n println!(\"area = {}\", s.area())\n /*\n let mut dashed = BezPath::new();\n for el in kurbo::DashIterator::new(p.elements().iter().cloned(), &[10.0]) {\n println!(\"{el:?}\");\n dashed.push(el);\n }\n let s = stroke(dashed, &style, 1e-1);\n println!(\"{}\", s.to_svg());\n */\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/affine.rs", "sha256": "3c50c0e2a5987ed9ccd27761f9d0d36ab98819286dd1659e10253f4b5aa1715c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Affine transforms.\n\nuse core::ops::{Mul, MulAssign};\n\nuse crate::{Point, Rect, Vec2};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// A 2D affine transform.\n#[derive(Clone, Copy, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct Affine([f64; 6]);\n\nimpl Affine {\n /// The identity transform.\n pub const IDENTITY: Affine = Affine::scale(1.0);\n\n /// A transform that is flipped on the y-axis. Useful for converting between\n /// y-up and y-down spaces.\n pub const FLIP_Y: Affine = Affine::new([1.0, 0., 0., -1.0, 0., 0.]);\n\n /// A transform that is flipped on the x-axis.\n pub const FLIP_X: Affine = Affine::new([-1.0, 0., 0., 1.0, 0., 0.]);\n\n /// Construct an affine transform from coefficients.\n ///\n /// If the coefficients are `(a, b, c, d, e, f)`, then the resulting\n /// transformation represents this augmented matrix:\n ///\n /// ```text\n /// | a c e |\n /// | b d f |\n /// | 0 0 1 |\n /// ```\n ///\n /// Note that this convention is transposed from PostScript and\n /// Direct2D, but is consistent with the\n /// [Wikipedia](https://en.wikipedia.org/wiki/Affine_transformation)\n /// formulation of affine transformation as augmented matrix. The\n /// idea is that `(A * B) * v == A * (B * v)`, where `*` is the\n /// [`Mul`](std::ops::Mul) trait.\n #[inline]\n pub const fn new(c: [f64; 6]) -> Affine {\n Affine(c)\n }\n\n /// An affine transform representing uniform scaling.\n #[inline]\n pub const fn scale(s: f64) -> Affine {\n Affine([s, 0.0, 0.0, s, 0.0, 0.0])\n }\n\n /// An affine transform representing non-uniform scaling\n /// with different scale values for x and y\n #[inline]\n pub const fn scale_non_uniform(s_x: f64, s_y: f64) -> Affine {\n Affine([s_x, 0.0, 0.0, s_y, 0.0, 0.0])\n }\n\n /// An affine transform representing rotation.\n ///\n /// The convention for rotation is that a positive angle rotates a\n /// positive X direction into positive Y. Thus, in a Y-down coordinate\n /// system (as is common for graphics), it is a clockwise rotation, and\n /// in Y-up (traditional for math), it is anti-clockwise.\n ///\n /// The angle, `th`, is expressed in radians.\n #[inline]\n pub fn rotate(th: f64) -> Affine {\n let (s, c) = th.sin_cos();\n Affine([c, s, -s, c, 0.0, 0.0])\n }\n\n /// An affine transform representing a rotation of `th` radians about `center`.\n ///\n /// See [`Affine::rotate()`] for more info.\n #[inline]\n pub fn rotate_about(th: f64, center: Point) -> Affine {\n let center = center.to_vec2();\n Self::translate(-center)\n .then_rotate(th)\n .then_translate(center)\n }\n\n /// An affine transform representing translation.\n #[inline]\n pub fn translate>(p: V) -> Affine {\n let p = p.into();\n Affine([1.0, 0.0, 0.0, 1.0, p.x, p.y])\n }\n\n /// An affine transformation representing a skew.\n ///\n /// The `skew_x` and `skew_y` parameters represent skew factors for the\n /// horizontal and vertical directions, respectively.\n ///\n /// This is commonly used to generate a faux oblique transform for\n /// font rendering. In this case, you can slant the glyph 20 degrees\n /// clockwise in the horizontal direction (assuming a Y-up coordinate\n /// system):\n ///\n /// ```\n /// let oblique_transform = kurbo::Affine::skew(20f64.to_radians().tan(), 0.0);\n /// ```\n #[inline]\n pub fn skew(skew_x: f64, skew_y: f64) -> Affine {\n Affine([1.0, skew_y, skew_x, 1.0, 0.0, 0.0])\n }\n\n /// Create an affine transform that represents reflection about the line `point + direction * t, t in (-infty, infty)`\n ///\n /// # Examples\n ///\n /// ```\n /// # use kurbo::{Point, Vec2, Affine};\n /// # fn assert_near(p0: Point, p1: Point) {\n /// # assert!((p1 - p0).hypot() < 1e-9, \"{p0:?} != {p1:?}\");\n /// # }\n /// let point = Point::new(1., 0.);\n /// let vec = Vec2::new(1., 1.);\n /// let map = Affine::reflect(point, vec);\n /// assert_near(map * Point::new(1., 0.), Point::new(1., 0.));\n /// assert_near(map * Point::new(2., 1.), Point::new(2., 1.));\n /// assert_near(map * Point::new(2., 2.), Point::new(3., 1.));\n /// ```\n #[inline]\n #[must_use]\n pub fn reflect(point: impl Into, direction: impl Into) -> Self {\n let point = point.into();\n let direction = direction.into();\n\n let n = Vec2 {\n x: direction.y,\n y: -direction.x,\n }\n .normalize();\n\n // Compute Householder reflection matrix\n let x2 = n.x * n.x;\n let xy = n.x * n.y;\n let y2 = n.y * n.y;\n // Here we also add in the post translation, because it doesn't require any further calc.\n let aff = Affine::new([\n 1. - 2. * x2,\n -2. * xy,\n -2. * xy,\n 1. - 2. * y2,\n point.x,\n point.y,\n ]);\n aff.pre_translate(-point.to_vec2())\n }\n\n /// A rotation by `th` followed by `self`.\n ///\n /// Equivalent to `self * Affine::rotate(th)`\n #[inline]\n #[must_use]\n pub fn pre_rotate(self, th: f64) -> Self {\n self * Affine::rotate(th)\n }\n\n /// A rotation by `th` about `center` followed by `self`.\n ///\n /// Equivalent to `self * Affine::rotate_about(th)`\n #[inline]\n #[must_use]\n pub fn pre_rotate_about(self, th: f64, center: Point) -> Self {\n Affine::rotate_about(th, center) * self\n }\n\n /// A scale by `scale` followed by `self`.\n ///\n /// Equivalent to `self * Affine::scale(scale)`\n #[inline]\n #[must_use]\n pub fn pre_scale(self, scale: f64) -> Self {\n self * Affine::scale(scale)\n }\n\n /// A scale by `(scale_x, scale_y)` followed by `self`.\n ///\n /// Equivalent to `self * Affine::scale_non_uniform(scale_x, scale_y)`\n #[inline]\n #[must_use]\n pub fn pre_scale_non_uniform(self, scale_x: f64, scale_y: f64) -> Self {\n self * Affine::scale_non_uniform(scale_x, scale_y)\n }\n\n /// A translation of `trans` followed by `self`.\n ///\n /// Equivalent to `self * Affine::translate(trans)`\n #[inline]\n #[must_use]\n pub fn pre_translate(self, trans: Vec2) -> Self {\n self * Affine::translate(trans)\n }\n\n /// `self` followed by a rotation of `th`.\n ///\n /// Equivalent to `Affine::rotate(th) * self`\n #[inline]\n #[must_use]\n pub fn then_rotate(self, th: f64) -> Self {\n Affine::rotate(th) * self\n }\n\n /// `self` followed by a rotation of `th` about `center`.\n ///\n /// Equivalent to `Affine::rotate_about(th, center) * self`\n #[inline]\n #[must_use]\n pub fn then_rotate_about(self, th: f64, center: Point) -> Self {\n Affine::rotate_about(th, center) * self\n }\n\n /// `self` followed by a scale of `scale`.\n ///\n /// Equivalent to `Affine::scale(scale) * self`\n #[inline]\n #[must_use]\n pub fn then_scale(self, scale: f64) -> Self {\n Affine::scale(scale) * self\n }\n\n /// `self` followed by a scale of `(scale_x, scale_y)`.\n ///\n /// Equivalent to `Affine::scale_non_uniform(scale_x, scale_y) * self`\n #[inline]\n #[must_use]\n pub fn then_scale_non_uniform(self, scale_x: f64, scale_y: f64) -> Self {\n Affine::scale_non_uniform(scale_x, scale_y) * self\n }\n\n /// `self` followed by a translation of `trans`.\n ///\n /// Equivalent to `Affine::translate(trans) * self`\n #[inline]\n #[must_use]\n pub fn then_translate(mut self, trans: Vec2) -> Self {\n self.0[4] += trans.x;\n self.0[5] += trans.y;\n self\n }\n\n /// Creates an affine transformation that takes the unit square to the given rectangle.\n ///\n /// Useful when you want to draw into the unit square but have your output fill any rectangle.\n /// In this case push the `Affine` onto the transform stack.\n pub fn map_unit_square(rect: Rect) -> Affine {\n Affine([rect.width(), 0., 0., rect.height(), rect.x0, rect.y0])\n }\n\n /// Get the coefficients of the transform.\n #[inline]\n pub fn as_coeffs(self) -> [f64; 6] {\n self.0\n }\n\n /// Compute the determinant of this transform.\n pub fn determinant(self) -> f64 {\n self.0[0] * self.0[3] - self.0[1] * self.0[2]\n }\n\n /// Compute the inverse transform.\n ///\n /// Produces NaN values when the determinant is zero.\n pub fn inverse(self) -> Affine {\n let inv_det = self.determinant().recip();\n Affine([\n inv_det * self.0[3],\n -inv_det * self.0[1],\n -inv_det * self.0[2],\n inv_det * self.0[0],\n inv_det * (self.0[2] * self.0[5] - self.0[3] * self.0[4]),\n inv_det * (self.0[1] * self.0[4] - self.0[0] * self.0[5]),\n ])\n }\n\n /// Compute the bounding box of a transformed rectangle.\n ///\n /// Returns the minimal `Rect` that encloses the given `Rect` after affine transformation.\n /// If the transform is axis-aligned, then this bounding box is \"tight\", in other words the\n /// returned `Rect` is the transformed rectangle.\n ///\n /// The returned rectangle always has non-negative width and height.\n pub fn transform_rect_bbox(self, rect: Rect) -> Rect {\n let p00 = self * Point::new(rect.x0, rect.y0);\n let p01 = self * Point::new(rect.x0, rect.y1);\n let p10 = self * Point::new(rect.x1, rect.y0);\n let p11 = self * Point::new(rect.x1, rect.y1);\n Rect::from_points(p00, p01).union(Rect::from_points(p10, p11))\n }\n\n /// Is this map finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n self.0[0].is_finite()\n && self.0[1].is_finite()\n && self.0[2].is_finite()\n && self.0[3].is_finite()\n && self.0[4].is_finite()\n && self.0[5].is_finite()\n }\n\n /// Is this map NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n self.0[0].is_nan()\n || self.0[1].is_nan()\n || self.0[2].is_nan()\n || self.0[3].is_nan()\n || self.0[4].is_nan()\n || self.0[5].is_nan()\n }\n\n /// Compute the singular value decomposition of the linear transformation (ignoring the\n /// translation).\n ///\n /// All non-degenerate linear transformations can be represented as\n ///\n /// 1. a rotation about the origin.\n /// 2. a scaling along the x and y axes\n /// 3. another rotation about the origin\n ///\n /// composed together. Decomposing a 2x2 matrix in this way is called a \"singular value\n /// decomposition\" and is written `U Σ V^T`, where U and V^T are orthogonal (rotations) and Σ\n /// is a diagonal matrix (a scaling).\n ///\n /// Since currently this function is used to calculate ellipse radii and rotation from an\n /// affine map on the unit circle, we don't calculate V^T, since a rotation of the unit (or\n /// any) circle about its center always results in the same circle. This is the reason that an\n /// ellipse mapped using an affine map is always an ellipse.\n ///\n /// Will return NaNs if the matrix (or equivalently the linear map) is singular.\n ///\n /// First part of the return tuple is the scaling, second part is the angle of rotation (in\n /// radians)\n #[inline]\n pub(crate) fn svd(self) -> (Vec2, f64) {\n let a = self.0[0];\n let a2 = a * a;\n let b = self.0[1];\n let b2 = b * b;\n let c = self.0[2];\n let c2 = c * c;\n let d = self.0[3];\n let d2 = d * d;\n let ab = a * b;\n let cd = c * d;\n let angle = 0.5 * (2.0 * (ab + cd)).atan2(a2 - b2 + c2 - d2);\n let s1 = a2 + b2 + c2 + d2;\n let s2 = ((a2 - b2 + c2 - d2).powi(2) + 4.0 * (ab + cd).powi(2)).sqrt();\n (\n Vec2 {\n x: (0.5 * (s1 + s2)).sqrt(),\n y: (0.5 * (s1 - s2)).sqrt(),\n },\n angle,\n )\n }\n\n /// Returns the translation part of this affine map (`(self.0[4], self.0[5])`).\n #[inline]\n pub fn translation(self) -> Vec2 {\n Vec2 {\n x: self.0[4],\n y: self.0[5],\n }\n }\n\n /// Replaces the translation portion of this affine map\n ///\n /// The translation can be seen as being applied after the linear part of the map.\n #[must_use]\n #[inline]\n pub fn with_translation(mut self, trans: Vec2) -> Affine {\n self.0[4] = trans.x;\n self.0[5] = trans.y;\n self\n }\n}\n\nimpl Default for Affine {\n #[inline]\n fn default() -> Affine {\n Affine::IDENTITY\n }\n}\n\nimpl Mul for Affine {\n type Output = Point;\n\n #[inline]\n fn mul(self, other: Point) -> Point {\n Point::new(\n self.0[0] * other.x + self.0[2] * other.y + self.0[4],\n self.0[1] * other.x + self.0[3] * other.y + self.0[5],\n )\n }\n}\n\nimpl Mul for Affine {\n type Output = Affine;\n\n #[inline]\n fn mul(self, other: Affine) -> Affine {\n Affine([\n self.0[0] * other.0[0] + self.0[2] * other.0[1],\n self.0[1] * other.0[0] + self.0[3] * other.0[1],\n self.0[0] * other.0[2] + self.0[2] * other.0[3],\n self.0[1] * other.0[2] + self.0[3] * other.0[3],\n self.0[0] * other.0[4] + self.0[2] * other.0[5] + self.0[4],\n self.0[1] * other.0[4] + self.0[3] * other.0[5] + self.0[5],\n ])\n }\n}\n\nimpl MulAssign for Affine {\n #[inline]\n fn mul_assign(&mut self, other: Affine) {\n *self = self.mul(other);\n }\n}\n\nimpl Mul for f64 {\n type Output = Affine;\n\n #[inline]\n fn mul(self, other: Affine) -> Affine {\n Affine([\n self * other.0[0],\n self * other.0[1],\n self * other.0[2],\n self * other.0[3],\n self * other.0[4],\n self * other.0[5],\n ])\n }\n}\n\n// Conversions to and from mint\n#[cfg(feature = \"mint\")]\nimpl From for mint::ColumnMatrix2x3 {\n #[inline]\n fn from(a: Affine) -> mint::ColumnMatrix2x3 {\n mint::ColumnMatrix2x3 {\n x: mint::Vector2 {\n x: a.0[0],\n y: a.0[1],\n },\n y: mint::Vector2 {\n x: a.0[2],\n y: a.0[3],\n },\n z: mint::Vector2 {\n x: a.0[4],\n y: a.0[5],\n },\n }\n }\n}\n\n#[cfg(feature = \"mint\")]\nimpl From> for Affine {\n #[inline]\n fn from(m: mint::ColumnMatrix2x3) -> Affine {\n Affine([m.x.x, m.x.y, m.y.x, m.y.y, m.z.x, m.z.y])\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{Affine, Point, Vec2};\n use std::f64::consts::PI;\n\n fn assert_near(p0: Point, p1: Point) {\n assert!((p1 - p0).hypot() < 1e-9, \"{p0:?} != {p1:?}\");\n }\n\n fn affine_assert_near(a0: Affine, a1: Affine) {\n for i in 0..6 {\n assert!((a0.0[i] - a1.0[i]).abs() < 1e-9, \"{a0:?} != {a1:?}\");\n }\n }\n\n #[test]\n fn affine_basic() {\n let p = Point::new(3.0, 4.0);\n\n assert_near(Affine::default() * p, p);\n assert_near(Affine::scale(2.0) * p, Point::new(6.0, 8.0));\n assert_near(Affine::rotate(0.0) * p, p);\n assert_near(Affine::rotate(PI / 2.0) * p, Point::new(-4.0, 3.0));\n assert_near(Affine::translate((5.0, 6.0)) * p, Point::new(8.0, 10.0));\n assert_near(Affine::skew(0.0, 0.0) * p, p);\n assert_near(Affine::skew(2.0, 4.0) * p, Point::new(11.0, 16.0));\n }\n\n #[test]\n fn affine_mul() {\n let a1 = Affine::new([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);\n let a2 = Affine::new([0.1, 1.2, 2.3, 3.4, 4.5, 5.6]);\n\n let px = Point::new(1.0, 0.0);\n let py = Point::new(0.0, 1.0);\n let pxy = Point::new(1.0, 1.0);\n assert_near(a1 * (a2 * px), (a1 * a2) * px);\n assert_near(a1 * (a2 * py), (a1 * a2) * py);\n assert_near(a1 * (a2 * pxy), (a1 * a2) * pxy);\n }\n\n #[test]\n fn affine_inv() {\n let a = Affine::new([0.1, 1.2, 2.3, 3.4, 4.5, 5.6]);\n let a_inv = a.inverse();\n\n let px = Point::new(1.0, 0.0);\n let py = Point::new(0.0, 1.0);\n let pxy = Point::new(1.0, 1.0);\n assert_near(a * (a_inv * px), px);\n assert_near(a * (a_inv * py), py);\n assert_near(a * (a_inv * pxy), pxy);\n assert_near(a_inv * (a * px), px);\n assert_near(a_inv * (a * py), py);\n assert_near(a_inv * (a * pxy), pxy);\n }\n\n #[test]\n fn reflection() {\n affine_assert_near(\n Affine::reflect(Point::ZERO, (1., 0.)),\n Affine::new([1., 0., 0., -1., 0., 0.]),\n );\n affine_assert_near(\n Affine::reflect(Point::ZERO, (0., 1.)),\n Affine::new([-1., 0., 0., 1., 0., 0.]),\n );\n // y = x\n affine_assert_near(\n Affine::reflect(Point::ZERO, (1., 1.)),\n Affine::new([0., 1., 1., 0., 0., 0.]),\n );\n\n // no translate\n let point = Point::new(0., 0.);\n let vec = Vec2::new(1., 1.);\n let map = Affine::reflect(point, vec);\n assert_near(map * Point::new(0., 0.), Point::new(0., 0.));\n assert_near(map * Point::new(1., 1.), Point::new(1., 1.));\n assert_near(map * Point::new(1., 2.), Point::new(2., 1.));\n\n // with translate\n let point = Point::new(1., 0.);\n let vec = Vec2::new(1., 1.);\n let map = Affine::reflect(point, vec);\n assert_near(map * Point::new(1., 0.), Point::new(1., 0.));\n assert_near(map * Point::new(2., 1.), Point::new(2., 1.));\n assert_near(map * Point::new(2., 2.), Point::new(3., 1.));\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/arc.rs", "sha256": "65d2e066ab2eb4e7210f4a961967406f6c2a15806fa5bdab8928c709ed062777", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! An ellipse arc.\n\nuse crate::{Affine, Ellipse, PathEl, Point, Rect, Shape, Vec2};\nuse core::{\n f64::consts::{FRAC_PI_2, PI},\n iter,\n ops::Mul,\n};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// A single arc segment.\n#[derive(Clone, Copy, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct Arc {\n /// The arc's centre point.\n pub center: Point,\n /// The arc's radii, where the vector's x-component is the radius in the\n /// positive x direction after applying `x_rotation`.\n pub radii: Vec2,\n /// The start angle in radians.\n pub start_angle: f64,\n /// The angle between the start and end of the arc, in radians.\n pub sweep_angle: f64,\n /// How much the arc is rotated, in radians.\n pub x_rotation: f64,\n}\n\nimpl Arc {\n /// Create a new `Arc`.\n pub fn new(\n center: impl Into,\n radii: impl Into,\n start_angle: f64,\n sweep_angle: f64,\n x_rotation: f64,\n ) -> Self {\n Self {\n center: center.into(),\n radii: radii.into(),\n start_angle,\n sweep_angle,\n x_rotation,\n }\n }\n\n /// Create an iterator generating Bezier path elements.\n ///\n /// The generated elements can be appended to an existing bezier path.\n pub fn append_iter(&self, tolerance: f64) -> ArcAppendIter {\n let sign = self.sweep_angle.signum();\n let scaled_err = self.radii.x.max(self.radii.y) / tolerance;\n // Number of subdivisions per ellipse based on error tolerance.\n // Note: this may slightly underestimate the error for quadrants.\n let n_err = (1.1163 * scaled_err).powf(1.0 / 6.0).max(3.999_999);\n let n = (n_err * self.sweep_angle.abs() * (1.0 / (2.0 * PI))).ceil();\n let angle_step = self.sweep_angle / n;\n let n = n as usize;\n let arm_len = (4.0 / 3.0) * (0.25 * angle_step).abs().tan() * sign;\n let angle0 = self.start_angle;\n let p0 = sample_ellipse(self.radii, self.x_rotation, angle0);\n\n ArcAppendIter {\n idx: 0,\n\n center: self.center,\n radii: self.radii,\n x_rotation: self.x_rotation,\n n,\n arm_len,\n angle_step,\n\n p0,\n angle0,\n }\n }\n\n /// Converts an Arc into a series of cubic bezier segments.\n ///\n /// Closure will be invoked for each segment.\n pub fn to_cubic_beziers

(self, tolerance: f64, mut p: P)\n where\n P: FnMut(Point, Point, Point),\n {\n let mut path = self.append_iter(tolerance);\n while let Some(PathEl::CurveTo(p1, p2, p3)) = path.next() {\n p(p1, p2, p3);\n }\n }\n}\n\n#[doc(hidden)]\npub struct ArcAppendIter {\n idx: usize,\n\n center: Point,\n radii: Vec2,\n x_rotation: f64,\n n: usize,\n arm_len: f64,\n angle_step: f64,\n\n p0: Vec2,\n angle0: f64,\n}\n\nimpl Iterator for ArcAppendIter {\n type Item = PathEl;\n\n fn next(&mut self) -> Option {\n if self.idx >= self.n {\n return None;\n }\n\n let angle1 = self.angle0 + self.angle_step;\n let p0 = self.p0;\n let p1 = p0\n + self.arm_len * sample_ellipse(self.radii, self.x_rotation, self.angle0 + FRAC_PI_2);\n let p3 = sample_ellipse(self.radii, self.x_rotation, angle1);\n let p2 =\n p3 - self.arm_len * sample_ellipse(self.radii, self.x_rotation, angle1 + FRAC_PI_2);\n\n self.angle0 = angle1;\n self.p0 = p3;\n self.idx += 1;\n\n Some(PathEl::CurveTo(\n self.center + p1,\n self.center + p2,\n self.center + p3,\n ))\n }\n}\n\n/// Take the ellipse radii, how the radii are rotated, and the sweep angle, and return a point on\n/// the ellipse.\nfn sample_ellipse(radii: Vec2, x_rotation: f64, angle: f64) -> Vec2 {\n let (angle_sin, angle_cos) = angle.sin_cos();\n let u = radii.x * angle_cos;\n let v = radii.y * angle_sin;\n rotate_pt(Vec2::new(u, v), x_rotation)\n}\n\n/// Rotate `pt` about the origin by `angle` radians.\nfn rotate_pt(pt: Vec2, angle: f64) -> Vec2 {\n let (angle_sin, angle_cos) = angle.sin_cos();\n Vec2::new(\n pt.x * angle_cos - pt.y * angle_sin,\n pt.x * angle_sin + pt.y * angle_cos,\n )\n}\n\nimpl Shape for Arc {\n type PathElementsIter<'iter> = iter::Chain, ArcAppendIter>;\n\n fn path_elements(&self, tolerance: f64) -> Self::PathElementsIter<'_> {\n let p0 = sample_ellipse(self.radii, self.x_rotation, self.start_angle);\n iter::once(PathEl::MoveTo(self.center + p0)).chain(self.append_iter(tolerance))\n }\n\n /// Note: shape isn't closed so area is not well defined.\n #[inline]\n fn area(&self) -> f64 {\n let Vec2 { x, y } = self.radii;\n PI * x * y\n }\n\n /// The perimeter of the ellipse.\n ///\n /// Note: Finding the perimeter of an ellipse is [fairly involved][wikipedia],\n /// so for now we just approximate by using the bezier curve representation.\n ///\n /// [wikipedia]: https://en.wikipedia.org/wiki/Ellipse#Circumference\n #[inline]\n fn perimeter(&self, accuracy: f64) -> f64 {\n self.path_segments(0.1).perimeter(accuracy)\n }\n\n /// Note: shape isn't closed, so a point's winding number is not well defined.\n #[inline]\n fn winding(&self, pt: Point) -> i32 {\n self.path_segments(0.1).winding(pt)\n }\n\n #[inline]\n fn bounding_box(&self) -> Rect {\n self.path_segments(0.1).bounding_box()\n }\n}\n\nimpl Mul for Affine {\n type Output = Arc;\n\n fn mul(self, arc: Arc) -> Self::Output {\n let ellipse = self * Ellipse::new(arc.center, arc.radii, arc.x_rotation);\n let center = ellipse.center();\n let (radii, rotation) = ellipse.radii_and_rotation();\n Arc {\n center,\n radii,\n x_rotation: rotation,\n start_angle: arc.start_angle,\n sweep_angle: arc.sweep_angle,\n }\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/bezpath.rs", "sha256": "4fbcf9706515be23ff81171b26c436d4fef864c1069122666ed061c5e362a9f7", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Bézier paths (up to cubic).\n\n#![allow(clippy::many_single_char_names)]\n\nuse core::iter::{Extend, FromIterator};\nuse core::mem;\nuse core::ops::{Mul, Range};\n\nuse alloc::vec::Vec;\n\nuse arrayvec::ArrayVec;\n\nuse crate::common::{solve_cubic, solve_quadratic};\nuse crate::MAX_EXTREMA;\nuse crate::{\n Affine, CubicBez, Line, Nearest, ParamCurve, ParamCurveArclen, ParamCurveArea,\n ParamCurveExtrema, ParamCurveNearest, Point, QuadBez, Rect, Shape, TranslateScale, Vec2,\n};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// A Bézier path.\n///\n/// These docs assume basic familiarity with Bézier curves; for an introduction,\n/// see Pomax's wonderful [A Primer on Bézier Curves].\n///\n/// This path can contain lines, quadratics ([`QuadBez`]) and cubics\n/// ([`CubicBez`]), and may contain multiple subpaths.\n///\n/// # Elements and Segments\n///\n/// A Bézier path can be represented in terms of either 'elements' ([`PathEl`])\n/// or 'segments' ([`PathSeg`]). Elements map closely to how Béziers are\n/// generally used in PostScript-style drawing APIs; they can be thought of as\n/// instructions for drawing the path. Segments more directly describe the\n/// path itself, with each segment being an independent line or curve.\n///\n/// These different representations are useful in different contexts.\n/// For tasks like drawing, elements are a natural fit, but when doing\n/// hit-testing or subdividing, we need to have access to the segments.\n///\n/// Conceptually, a `BezPath` contains zero or more subpaths. Each subpath\n/// *always* begins with a `MoveTo`, then has zero or more `LineTo`, `QuadTo`,\n/// and `CurveTo` elements, and optionally ends with a `ClosePath`.\n///\n/// Internally, a `BezPath` is a list of [`PathEl`]s; as such it implements\n/// [`FromIterator`] and [`Extend`]:\n///\n/// ```\n/// use kurbo::{BezPath, Rect, Shape, Vec2};\n/// let accuracy = 0.1;\n/// let rect = Rect::from_origin_size((0., 0.,), (10., 10.));\n/// // these are equivalent\n/// let path1 = rect.to_path(accuracy);\n/// let path2: BezPath = rect.path_elements(accuracy).collect();\n///\n/// // extend a path with another path:\n/// let mut path = rect.to_path(accuracy);\n/// let shifted_rect = rect + Vec2::new(5.0, 10.0);\n/// path.extend(shifted_rect.to_path(accuracy));\n/// ```\n///\n/// You can iterate the elements of a `BezPath` with the [`iter`] method,\n/// and the segments with the [`segments`] method:\n///\n/// ```\n/// use kurbo::{BezPath, Line, PathEl, PathSeg, Point, Rect, Shape};\n/// let accuracy = 0.1;\n/// let rect = Rect::from_origin_size((0., 0.,), (10., 10.));\n/// // these are equivalent\n/// let path = rect.to_path(accuracy);\n/// let first_el = PathEl::MoveTo(Point::ZERO);\n/// let first_seg = PathSeg::Line(Line::new((0., 0.), (10., 0.)));\n/// assert_eq!(path.iter().next(), Some(first_el));\n/// assert_eq!(path.segments().next(), Some(first_seg));\n/// ```\n/// In addition, if you have some other type that implements\n/// `Iterator`, you can adapt that to an iterator of segments with\n/// the [`segments` free function].\n///\n/// # Advanced functionality\n///\n/// In addition to the basic API, there are several useful pieces of advanced\n/// functionality available on `BezPath`:\n///\n/// - [`flatten`] does Bézier flattening, converting a curve to a series of\n/// line segments\n/// - [`intersect_line`] computes intersections of a path with a line, useful\n/// for things like subdividing\n///\n/// [A Primer on Bézier Curves]: https://pomax.github.io/bezierinfo/\n/// [`iter`]: BezPath::iter\n/// [`segments`]: BezPath::segments\n/// [`flatten`]: BezPath::flatten\n/// [`intersect_line`]: PathSeg::intersect_line\n/// [`segments` free function]: segments\n/// [`FromIterator`]: std::iter::FromIterator\n/// [`Extend`]: std::iter::Extend\n#[derive(Clone, Default, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct BezPath(Vec);\n\n/// The element of a Bézier path.\n///\n/// A valid path has `MoveTo` at the beginning of each subpath.\n#[derive(Clone, Copy, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub enum PathEl {\n /// Move directly to the point without drawing anything, starting a new\n /// subpath.\n MoveTo(Point),\n /// Draw a line from the current location to the point.\n LineTo(Point),\n /// Draw a quadratic bezier using the current location and the two points.\n QuadTo(Point, Point),\n /// Draw a cubic bezier using the current location and the three points.\n CurveTo(Point, Point, Point),\n /// Close off the path.\n ClosePath,\n}\n\n/// A segment of a Bézier path.\n#[derive(Clone, Copy, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub enum PathSeg {\n /// A line segment.\n Line(Line),\n /// A quadratic bezier segment.\n Quad(QuadBez),\n /// A cubic bezier segment.\n Cubic(CubicBez),\n}\n\n/// An intersection of a [`Line`] and a [`PathSeg`].\n///\n/// This can be generated with the [`PathSeg::intersect_line`] method.\n#[derive(Debug, Clone, Copy)]\npub struct LineIntersection {\n /// The 'time' that the intersection occurs, on the line.\n ///\n /// This value is in the range 0..1.\n pub line_t: f64,\n\n /// The 'time' that the intersection occurs, on the path segment.\n ///\n /// This value is nominally in the range 0..1, although it may slightly exceed\n /// that range at the boundaries of segments.\n pub segment_t: f64,\n}\n\n/// The minimum distance between two Bézier curves.\npub struct MinDistance {\n /// The shortest distance between any two points on the two curves.\n pub distance: f64,\n /// The position of the nearest point on the first curve, as a parameter.\n ///\n /// To resolve this to a [`Point`], use [`ParamCurve::eval`].\n ///\n /// [`ParamCurve::eval`]: crate::ParamCurve::eval\n pub t1: f64,\n /// The position of the nearest point on the second curve, as a parameter.\n ///\n /// To resolve this to a [`Point`], use [`ParamCurve::eval`].\n ///\n /// [`ParamCurve::eval`]: crate::ParamCurve::eval\n pub t2: f64,\n}\n\nimpl BezPath {\n /// Create a new path.\n pub fn new() -> BezPath {\n Default::default()\n }\n\n /// Create a path from a vector of path elements.\n ///\n /// `BezPath` also implements `FromIterator`, so it works with `collect`:\n ///\n /// ```\n /// // a very contrived example:\n /// use kurbo::{BezPath, PathEl};\n ///\n /// let path = BezPath::new();\n /// let as_vec: Vec = path.into_iter().collect();\n /// let back_to_path: BezPath = as_vec.into_iter().collect();\n /// ```\n pub fn from_vec(v: Vec) -> BezPath {\n debug_assert!(\n v.first().is_none() || matches!(v.first(), Some(PathEl::MoveTo(_))),\n \"BezPath must begin with MoveTo\"\n );\n BezPath(v)\n }\n\n /// Removes the last [`PathEl`] from the path and returns it, or `None` if the path is empty.\n pub fn pop(&mut self) -> Option {\n self.0.pop()\n }\n\n /// Push a generic path element onto the path.\n pub fn push(&mut self, el: PathEl) {\n self.0.push(el);\n debug_assert!(\n matches!(self.0.first(), Some(PathEl::MoveTo(_))),\n \"BezPath must begin with MoveTo\"\n )\n }\n\n /// Push a \"move to\" element onto the path.\n pub fn move_to>(&mut self, p: P) {\n self.push(PathEl::MoveTo(p.into()));\n }\n\n /// Push a \"line to\" element onto the path.\n ///\n /// Will panic with a debug assert when the current subpath does not\n /// start with `move_to`.\n pub fn line_to>(&mut self, p: P) {\n debug_assert!(self.is_open_subpath(), \"no open subpath (missing MoveTo)\");\n self.push(PathEl::LineTo(p.into()));\n }\n\n /// Push a \"quad to\" element onto the path.\n ///\n /// Will panic with a debug assert when the current subpath does not\n /// start with `move_to`.\n pub fn quad_to>(&mut self, p1: P, p2: P) {\n debug_assert!(self.is_open_subpath(), \"no open subpath (missing MoveTo)\");\n self.push(PathEl::QuadTo(p1.into(), p2.into()));\n }\n\n /// Push a \"curve to\" element onto the path.\n ///\n /// Will panic with a debug assert when the current subpath does not\n /// start with `move_to`.\n pub fn curve_to>(&mut self, p1: P, p2: P, p3: P) {\n debug_assert!(self.is_open_subpath(), \"no open subpath (missing MoveTo)\");\n self.push(PathEl::CurveTo(p1.into(), p2.into(), p3.into()));\n }\n\n /// Push a \"close path\" element onto the path.\n ///\n /// Will panic with a debug assert when the current subpath does not\n /// start with `move_to`.\n pub fn close_path(&mut self) {\n debug_assert!(self.is_open_subpath(), \"no open subpath (missing MoveTo)\");\n self.push(PathEl::ClosePath);\n }\n\n fn is_open_subpath(&self) -> bool {\n !self.0.is_empty() && self.0.last() != Some(&PathEl::ClosePath)\n }\n\n /// Get the path elements.\n pub fn elements(&self) -> &[PathEl] {\n &self.0\n }\n\n /// Get the path elements (mut version).\n pub fn elements_mut(&mut self) -> &mut [PathEl] {\n &mut self.0\n }\n\n /// Returns an iterator over the path's elements.\n pub fn iter(&self) -> impl Iterator + Clone + '_ {\n self.0.iter().copied()\n }\n\n /// Iterate over the path segments.\n pub fn segments(&self) -> impl Iterator + Clone + '_ {\n segments(self.iter())\n }\n\n /// Shorten the path, keeping the first `len` elements.\n pub fn truncate(&mut self, len: usize) {\n self.0.truncate(len);\n }\n\n /// Flatten the path, invoking the callback repeatedly.\n ///\n /// Flattening is the action of approximating a curve with a succession of line segments.\n ///\n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// →\n /// \n /// \n ///\n /// The tolerance value controls the maximum distance between the curved input\n /// segments and their polyline approximations. (In technical terms, this is the\n /// Hausdorff distance). The algorithm attempts to bound this distance between\n /// by `tolerance` but this is not absolutely guaranteed. The appropriate value\n /// depends on the use, but for antialiased rendering, a value of 0.25 has been\n /// determined to give good results. The number of segments tends to scale as the\n /// inverse square root of tolerance.\n ///\n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n ///\n /// The callback will be called in order with each element of the generated\n /// path. Because the result is made of polylines, these will be straight-line\n /// path elements only, no curves.\n ///\n /// This algorithm is based on the blog post [Flattening quadratic Béziers]\n /// but with some refinements. For one, there is a more careful approximation\n /// at cusps. For two, the algorithm is extended to work with cubic Béziers\n /// as well, by first subdividing into quadratics and then computing the\n /// subdivision of each quadratic. However, as a clever trick, these quadratics\n /// are subdivided fractionally, and their endpoints are not included.\n ///\n /// TODO: write a paper explaining this in more detail.\n ///\n /// Note: the [`flatten`] function provides the same\n /// functionality but works with slices and other [`PathEl`] iterators.\n ///\n /// [Flattening quadratic Béziers]: https://raphlinus.github.io/graphics/curves/2019/12/23/flatten-quadbez.html\n pub fn flatten(&self, tolerance: f64, callback: impl FnMut(PathEl)) {\n flatten(self, tolerance, callback);\n }\n\n /// Get the segment at the given element index.\n ///\n /// If you need to access all segments, [`segments`] provides a better\n /// API. This is intended for random access of specific elements, for clients\n /// that require this specifically.\n ///\n /// **note**: This returns the segment that ends at the provided element\n /// index. In effect this means it is *1-indexed*: since no segment ends at\n /// the first element (which is presumed to be a `MoveTo`) `get_seg(0)` will\n /// always return `None`.\n pub fn get_seg(&self, ix: usize) -> Option {\n if ix == 0 || ix >= self.0.len() {\n return None;\n }\n let last = match self.0[ix - 1] {\n PathEl::MoveTo(p) => p,\n PathEl::LineTo(p) => p,\n PathEl::QuadTo(_, p2) => p2,\n PathEl::CurveTo(_, _, p3) => p3,\n _ => return None,\n };\n match self.0[ix] {\n PathEl::LineTo(p) => Some(PathSeg::Line(Line::new(last, p))),\n PathEl::QuadTo(p1, p2) => Some(PathSeg::Quad(QuadBez::new(last, p1, p2))),\n PathEl::CurveTo(p1, p2, p3) => Some(PathSeg::Cubic(CubicBez::new(last, p1, p2, p3))),\n PathEl::ClosePath => self.0[..ix].iter().rev().find_map(|el| match *el {\n PathEl::MoveTo(start) if start != last => {\n Some(PathSeg::Line(Line::new(last, start)))\n }\n _ => None,\n }),\n _ => None,\n }\n }\n\n /// Returns `true` if the path contains no segments.\n pub fn is_empty(&self) -> bool {\n self.0\n .iter()\n .all(|el| matches!(el, PathEl::MoveTo(..) | PathEl::ClosePath))\n }\n\n /// Apply an affine transform to the path.\n pub fn apply_affine(&mut self, affine: Affine) {\n for el in self.0.iter_mut() {\n *el = affine * (*el);\n }\n }\n\n /// Is this path finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n self.0.iter().all(|v| v.is_finite())\n }\n\n /// Is this path NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n self.0.iter().any(|v| v.is_nan())\n }\n\n /// Returns a rectangle that conservatively encloses the path.\n ///\n /// Unlike the `bounding_box` method, this uses control points directly\n /// rather than computing tight bounds for curve elements.\n pub fn control_box(&self) -> Rect {\n let mut cbox: Option = None;\n let mut add_pts = |pts: &[Point]| {\n for pt in pts {\n cbox = match cbox {\n Some(cbox) => Some(cbox.union_pt(*pt)),\n _ => Some(Rect::from_points(*pt, *pt)),\n };\n }\n };\n for &el in self.elements() {\n match el {\n PathEl::MoveTo(p0) | PathEl::LineTo(p0) => add_pts(&[p0]),\n PathEl::QuadTo(p0, p1) => add_pts(&[p0, p1]),\n PathEl::CurveTo(p0, p1, p2) => add_pts(&[p0, p1, p2]),\n PathEl::ClosePath => {}\n }\n }\n cbox.unwrap_or_default()\n }\n\n /// Returns a new path with the winding direction of all subpaths reversed.\n pub fn reverse_subpaths(&self) -> BezPath {\n let elements = self.elements();\n let mut start_ix = 1;\n let mut start_pt = Point::default();\n let mut reversed = BezPath(Vec::with_capacity(elements.len()));\n // Pending move is used to capture degenerate subpaths that should\n // remain in the reversed output.\n let mut pending_move = false;\n for (ix, el) in elements.iter().enumerate() {\n match el {\n PathEl::MoveTo(pt) => {\n if pending_move {\n reversed.push(PathEl::MoveTo(start_pt));\n }\n if start_ix < ix {\n reverse_subpath(start_pt, &elements[start_ix..ix], &mut reversed);\n }\n pending_move = true;\n start_pt = *pt;\n start_ix = ix + 1;\n }\n PathEl::ClosePath => {\n if start_ix <= ix {\n reverse_subpath(start_pt, &elements[start_ix..ix], &mut reversed);\n }\n reversed.push(PathEl::ClosePath);\n start_ix = ix + 1;\n pending_move = false;\n }\n _ => {\n pending_move = false;\n }\n }\n }\n if start_ix < elements.len() {\n reverse_subpath(start_pt, &elements[start_ix..], &mut reversed);\n } else if pending_move {\n reversed.push(PathEl::MoveTo(start_pt));\n }\n reversed\n }\n}\n\n/// Helper for reversing a subpath.\n///\n/// The `els` parameter must not contain any `MoveTo` or `ClosePath` elements.\nfn reverse_subpath(start_pt: Point, els: &[PathEl], reversed: &mut BezPath) {\n let end_pt = els.last().and_then(|el| el.end_point()).unwrap_or(start_pt);\n reversed.push(PathEl::MoveTo(end_pt));\n for (ix, el) in els.iter().enumerate().rev() {\n let end_pt = if ix > 0 {\n els[ix - 1].end_point().unwrap()\n } else {\n start_pt\n };\n match el {\n PathEl::LineTo(_) => reversed.push(PathEl::LineTo(end_pt)),\n PathEl::QuadTo(c0, _) => reversed.push(PathEl::QuadTo(*c0, end_pt)),\n PathEl::CurveTo(c0, c1, _) => reversed.push(PathEl::CurveTo(*c1, *c0, end_pt)),\n _ => panic!(\"reverse_subpath expects MoveTo and ClosePath to be removed\"),\n }\n }\n}\n\nimpl FromIterator for BezPath {\n fn from_iter>(iter: T) -> Self {\n let el_vec: Vec<_> = iter.into_iter().collect();\n BezPath::from_vec(el_vec)\n }\n}\n\n/// Allow iteration over references to `BezPath`.\n///\n/// Note: the semantics are slightly different from simply iterating over the\n/// slice, as it returns `PathEl` items, rather than references.\nimpl<'a> IntoIterator for &'a BezPath {\n type Item = PathEl;\n type IntoIter = core::iter::Cloned>;\n\n fn into_iter(self) -> Self::IntoIter {\n self.elements().iter().cloned()\n }\n}\n\nimpl IntoIterator for BezPath {\n type Item = PathEl;\n type IntoIter = alloc::vec::IntoIter;\n\n fn into_iter(self) -> Self::IntoIter {\n self.0.into_iter()\n }\n}\n\nimpl Extend for BezPath {\n fn extend>(&mut self, iter: I) {\n self.0.extend(iter);\n }\n}\n\n/// Proportion of tolerance budget that goes to cubic to quadratic conversion.\nconst TO_QUAD_TOL: f64 = 0.1;\n\n/// Flatten the path, invoking the callback repeatedly.\n///\n/// See [`BezPath::flatten`] for more discussion.\n/// This signature is a bit more general, allowing flattening of `&[PathEl]` slices\n/// and other iterators yielding `PathEl`.\npub fn flatten(\n path: impl IntoIterator,\n tolerance: f64,\n mut callback: impl FnMut(PathEl),\n) {\n let sqrt_tol = tolerance.sqrt();\n let mut last_pt = None;\n let mut quad_buf = Vec::new();\n for el in path {\n match el {\n PathEl::MoveTo(p) => {\n last_pt = Some(p);\n callback(PathEl::MoveTo(p));\n }\n PathEl::LineTo(p) => {\n last_pt = Some(p);\n callback(PathEl::LineTo(p));\n }\n PathEl::QuadTo(p1, p2) => {\n if let Some(p0) = last_pt {\n let q = QuadBez::new(p0, p1, p2);\n let params = q.estimate_subdiv(sqrt_tol);\n let n = ((0.5 * params.val / sqrt_tol).ceil() as usize).max(1);\n let step = 1.0 / (n as f64);\n for i in 1..n {\n let u = (i as f64) * step;\n let t = q.determine_subdiv_t(¶ms, u);\n let p = q.eval(t);\n callback(PathEl::LineTo(p));\n }\n callback(PathEl::LineTo(p2));\n }\n last_pt = Some(p2);\n }\n PathEl::CurveTo(p1, p2, p3) => {\n if let Some(p0) = last_pt {\n let c = CubicBez::new(p0, p1, p2, p3);\n\n // Subdivide into quadratics, and estimate the number of\n // subdivisions required for each, summing to arrive at an\n // estimate for the number of subdivisions for the cubic.\n // Also retain these parameters for later.\n let iter = c.to_quads(tolerance * TO_QUAD_TOL);\n quad_buf.clear();\n quad_buf.reserve(iter.size_hint().0);\n let sqrt_remain_tol = sqrt_tol * (1.0 - TO_QUAD_TOL).sqrt();\n let mut sum = 0.0;\n for (_, _, q) in iter {\n let params = q.estimate_subdiv(sqrt_remain_tol);\n sum += params.val;\n quad_buf.push((q, params));\n }\n let n = ((0.5 * sum / sqrt_remain_tol).ceil() as usize).max(1);\n\n // Iterate through the quadratics, outputting the points of\n // subdivisions that fall within that quadratic.\n let step = sum / (n as f64);\n let mut i = 1;\n let mut val_sum = 0.0;\n for (q, params) in &quad_buf {\n let mut target = (i as f64) * step;\n let recip_val = params.val.recip();\n while target < val_sum + params.val {\n let u = (target - val_sum) * recip_val;\n let t = q.determine_subdiv_t(params, u);\n let p = q.eval(t);\n callback(PathEl::LineTo(p));\n i += 1;\n if i == n + 1 {\n break;\n }\n target = (i as f64) * step;\n }\n val_sum += params.val;\n }\n callback(PathEl::LineTo(p3));\n }\n last_pt = Some(p3);\n }\n PathEl::ClosePath => {\n last_pt = None;\n callback(PathEl::ClosePath);\n }\n }\n }\n}\n\nimpl Mul for Affine {\n type Output = PathEl;\n\n fn mul(self, other: PathEl) -> PathEl {\n match other {\n PathEl::MoveTo(p) => PathEl::MoveTo(self * p),\n PathEl::LineTo(p) => PathEl::LineTo(self * p),\n PathEl::QuadTo(p1, p2) => PathEl::QuadTo(self * p1, self * p2),\n PathEl::CurveTo(p1, p2, p3) => PathEl::CurveTo(self * p1, self * p2, self * p3),\n PathEl::ClosePath => PathEl::ClosePath,\n }\n }\n}\n\nimpl Mul for Affine {\n type Output = PathSeg;\n\n fn mul(self, other: PathSeg) -> PathSeg {\n match other {\n PathSeg::Line(line) => PathSeg::Line(self * line),\n PathSeg::Quad(quad) => PathSeg::Quad(self * quad),\n PathSeg::Cubic(cubic) => PathSeg::Cubic(self * cubic),\n }\n }\n}\n\nimpl Mul for Affine {\n type Output = BezPath;\n\n fn mul(self, other: BezPath) -> BezPath {\n BezPath(other.0.iter().map(|&el| self * el).collect())\n }\n}\n\nimpl<'a> Mul<&'a BezPath> for Affine {\n type Output = BezPath;\n\n fn mul(self, other: &BezPath) -> BezPath {\n BezPath(other.0.iter().map(|&el| self * el).collect())\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = PathEl;\n\n fn mul(self, other: PathEl) -> PathEl {\n match other {\n PathEl::MoveTo(p) => PathEl::MoveTo(self * p),\n PathEl::LineTo(p) => PathEl::LineTo(self * p),\n PathEl::QuadTo(p1, p2) => PathEl::QuadTo(self * p1, self * p2),\n PathEl::CurveTo(p1, p2, p3) => PathEl::CurveTo(self * p1, self * p2, self * p3),\n PathEl::ClosePath => PathEl::ClosePath,\n }\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = PathSeg;\n\n fn mul(self, other: PathSeg) -> PathSeg {\n match other {\n PathSeg::Line(line) => PathSeg::Line(self * line),\n PathSeg::Quad(quad) => PathSeg::Quad(self * quad),\n PathSeg::Cubic(cubic) => PathSeg::Cubic(self * cubic),\n }\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = BezPath;\n\n fn mul(self, other: BezPath) -> BezPath {\n BezPath(other.0.iter().map(|&el| self * el).collect())\n }\n}\n\nimpl<'a> Mul<&'a BezPath> for TranslateScale {\n type Output = BezPath;\n\n fn mul(self, other: &BezPath) -> BezPath {\n BezPath(other.0.iter().map(|&el| self * el).collect())\n }\n}\n\n/// Transform an iterator over path elements into one over path\n/// segments.\n///\n/// See also [`BezPath::segments`].\n/// This signature is a bit more general, allowing `&[PathEl]` slices\n/// and other iterators yielding `PathEl`.\npub fn segments(elements: I) -> Segments\nwhere\n I: IntoIterator,\n{\n Segments {\n elements: elements.into_iter(),\n start_last: None,\n }\n}\n\n/// An iterator that transforms path elements to path segments.\n///\n/// This struct is created by the [`segments`] function.\n#[derive(Clone)]\npub struct Segments> {\n elements: I,\n start_last: Option<(Point, Point)>,\n}\n\nimpl> Iterator for Segments {\n type Item = PathSeg;\n\n fn next(&mut self) -> Option {\n for el in &mut self.elements {\n // We first need to check whether this is the first\n // path element we see to fill in the start position.\n let (start, last) = self.start_last.get_or_insert_with(|| {\n let point = match el {\n PathEl::MoveTo(p) => p,\n PathEl::LineTo(p) => p,\n PathEl::QuadTo(_, p2) => p2,\n PathEl::CurveTo(_, _, p3) => p3,\n PathEl::ClosePath => panic!(\"Can't start a segment on a ClosePath\"),\n };\n (point, point)\n });\n\n return Some(match el {\n PathEl::MoveTo(p) => {\n *start = p;\n *last = p;\n continue;\n }\n PathEl::LineTo(p) => PathSeg::Line(Line::new(mem::replace(last, p), p)),\n PathEl::QuadTo(p1, p2) => {\n PathSeg::Quad(QuadBez::new(mem::replace(last, p2), p1, p2))\n }\n PathEl::CurveTo(p1, p2, p3) => {\n PathSeg::Cubic(CubicBez::new(mem::replace(last, p3), p1, p2, p3))\n }\n PathEl::ClosePath => {\n if *last != *start {\n PathSeg::Line(Line::new(mem::replace(last, *start), *start))\n } else {\n continue;\n }\n }\n });\n }\n\n None\n }\n}\n\nimpl> Segments {\n /// Here, `accuracy` specifies the accuracy for each Bézier segment. At worst,\n /// the total error is `accuracy` times the number of Bézier segments.\n\n // TODO: pub? Or is this subsumed by method of &[PathEl]?\n pub(crate) fn perimeter(self, accuracy: f64) -> f64 {\n self.map(|seg| seg.arclen(accuracy)).sum()\n }\n\n // Same\n pub(crate) fn area(self) -> f64 {\n self.map(|seg| seg.signed_area()).sum()\n }\n\n // Same\n pub(crate) fn winding(self, p: Point) -> i32 {\n self.map(|seg| seg.winding(p)).sum()\n }\n\n // Same\n pub(crate) fn bounding_box(self) -> Rect {\n let mut bbox: Option = None;\n for seg in self {\n let seg_bb = ParamCurveExtrema::bounding_box(&seg);\n if let Some(bb) = bbox {\n bbox = Some(bb.union(seg_bb));\n } else {\n bbox = Some(seg_bb)\n }\n }\n bbox.unwrap_or_default()\n }\n}\n\nimpl ParamCurve for PathSeg {\n fn eval(&self, t: f64) -> Point {\n match *self {\n PathSeg::Line(line) => line.eval(t),\n PathSeg::Quad(quad) => quad.eval(t),\n PathSeg::Cubic(cubic) => cubic.eval(t),\n }\n }\n\n fn subsegment(&self, range: Range) -> PathSeg {\n match *self {\n PathSeg::Line(line) => PathSeg::Line(line.subsegment(range)),\n PathSeg::Quad(quad) => PathSeg::Quad(quad.subsegment(range)),\n PathSeg::Cubic(cubic) => PathSeg::Cubic(cubic.subsegment(range)),\n }\n }\n}\n\nimpl ParamCurveArclen for PathSeg {\n fn arclen(&self, accuracy: f64) -> f64 {\n match *self {\n PathSeg::Line(line) => line.arclen(accuracy),\n PathSeg::Quad(quad) => quad.arclen(accuracy),\n PathSeg::Cubic(cubic) => cubic.arclen(accuracy),\n }\n }\n\n fn inv_arclen(&self, arclen: f64, accuracy: f64) -> f64 {\n match *self {\n PathSeg::Line(line) => line.inv_arclen(arclen, accuracy),\n PathSeg::Quad(quad) => quad.inv_arclen(arclen, accuracy),\n PathSeg::Cubic(cubic) => cubic.inv_arclen(arclen, accuracy),\n }\n }\n}\n\nimpl ParamCurveArea for PathSeg {\n fn signed_area(&self) -> f64 {\n match *self {\n PathSeg::Line(line) => line.signed_area(),\n PathSeg::Quad(quad) => quad.signed_area(),\n PathSeg::Cubic(cubic) => cubic.signed_area(),\n }\n }\n}\n\nimpl ParamCurveNearest for PathSeg {\n fn nearest(&self, p: Point, accuracy: f64) -> Nearest {\n match *self {\n PathSeg::Line(line) => line.nearest(p, accuracy),\n PathSeg::Quad(quad) => quad.nearest(p, accuracy),\n PathSeg::Cubic(cubic) => cubic.nearest(p, accuracy),\n }\n }\n}\n\nimpl ParamCurveExtrema for PathSeg {\n fn extrema(&self) -> ArrayVec {\n match *self {\n PathSeg::Line(line) => line.extrema(),\n PathSeg::Quad(quad) => quad.extrema(),\n PathSeg::Cubic(cubic) => cubic.extrema(),\n }\n }\n}\n\nimpl PathSeg {\n /// Get the [`PathEl`] that is equivalent to discarding the segment start point.\n pub fn as_path_el(&self) -> PathEl {\n match self {\n PathSeg::Line(line) => PathEl::LineTo(line.p1),\n PathSeg::Quad(q) => PathEl::QuadTo(q.p1, q.p2),\n PathSeg::Cubic(c) => PathEl::CurveTo(c.p1, c.p2, c.p3),\n }\n }\n\n /// Returns a new `PathSeg` describing the same path as `self`, but with\n /// the points reversed.\n pub fn reverse(&self) -> PathSeg {\n match self {\n PathSeg::Line(Line { p0, p1 }) => PathSeg::Line(Line::new(*p1, *p0)),\n PathSeg::Quad(q) => PathSeg::Quad(QuadBez::new(q.p2, q.p1, q.p0)),\n PathSeg::Cubic(c) => PathSeg::Cubic(CubicBez::new(c.p3, c.p2, c.p1, c.p0)),\n }\n }\n\n /// Convert this segment to a cubic bezier.\n pub fn to_cubic(&self) -> CubicBez {\n match *self {\n PathSeg::Line(Line { p0, p1 }) => CubicBez::new(p0, p0, p1, p1),\n PathSeg::Cubic(c) => c,\n PathSeg::Quad(q) => q.raise(),\n }\n }\n\n // Assumes split at extrema.\n fn winding_inner(&self, p: Point) -> i32 {\n let start = self.start();\n let end = self.end();\n let sign = if end.y > start.y {\n if p.y < start.y || p.y >= end.y {\n return 0;\n }\n -1\n } else if end.y < start.y {\n if p.y < end.y || p.y >= start.y {\n return 0;\n }\n 1\n } else {\n return 0;\n };\n match *self {\n PathSeg::Line(_line) => {\n if p.x < start.x.min(end.x) {\n return 0;\n }\n if p.x >= start.x.max(end.x) {\n return sign;\n }\n // line equation ax + by = c\n let a = end.y - start.y;\n let b = start.x - end.x;\n let c = a * start.x + b * start.y;\n if (a * p.x + b * p.y - c) * (sign as f64) <= 0.0 {\n sign\n } else {\n 0\n }\n }\n PathSeg::Quad(quad) => {\n let p1 = quad.p1;\n if p.x < start.x.min(end.x).min(p1.x) {\n return 0;\n }\n if p.x >= start.x.max(end.x).max(p1.x) {\n return sign;\n }\n let a = end.y - 2.0 * p1.y + start.y;\n let b = 2.0 * (p1.y - start.y);\n let c = start.y - p.y;\n for t in solve_quadratic(c, b, a) {\n if (0.0..=1.0).contains(&t) {\n let x = quad.eval(t).x;\n if p.x >= x {\n return sign;\n } else {\n return 0;\n }\n }\n }\n 0\n }\n PathSeg::Cubic(cubic) => {\n let p1 = cubic.p1;\n let p2 = cubic.p2;\n if p.x < start.x.min(end.x).min(p1.x).min(p2.x) {\n return 0;\n }\n if p.x >= start.x.max(end.x).max(p1.x).max(p2.x) {\n return sign;\n }\n let a = end.y - 3.0 * p2.y + 3.0 * p1.y - start.y;\n let b = 3.0 * (p2.y - 2.0 * p1.y + start.y);\n let c = 3.0 * (p1.y - start.y);\n let d = start.y - p.y;\n for t in solve_cubic(d, c, b, a) {\n if (0.0..=1.0).contains(&t) {\n let x = cubic.eval(t).x;\n if p.x >= x {\n return sign;\n } else {\n return 0;\n }\n }\n }\n 0\n }\n }\n }\n\n /// Compute the winding number contribution of a single segment.\n ///\n /// Cast a ray to the left and count intersections.\n fn winding(&self, p: Point) -> i32 {\n self.extrema_ranges()\n .into_iter()\n .map(|range| self.subsegment(range).winding_inner(p))\n .sum()\n }\n\n /// Compute intersections against a line.\n ///\n /// Returns a vector of the intersections. For each intersection,\n /// the `t` value of the segment and line are given.\n ///\n /// Note: This test is designed to be inclusive of points near the endpoints\n /// of the segment. This is so that testing a line against multiple\n /// contiguous segments of a path will be guaranteed to catch at least one\n /// of them. In such cases, use higher level logic to coalesce the hits\n /// (the `t` value may be slightly outside the range of 0..1).\n ///\n /// # Examples\n ///\n /// ```\n /// # use kurbo::*;\n /// let seg = PathSeg::Line(Line::new((0.0, 0.0), (2.0, 0.0)));\n /// let line = Line::new((1.0, 2.0), (1.0, -2.0));\n /// let intersection = seg.intersect_line(line);\n /// assert_eq!(intersection.len(), 1);\n /// let intersection = intersection[0];\n /// assert_eq!(intersection.segment_t, 0.5);\n /// assert_eq!(intersection.line_t, 0.5);\n ///\n /// let point = seg.eval(intersection.segment_t);\n /// assert_eq!(point, Point::new(1.0, 0.0));\n /// ```\n pub fn intersect_line(&self, line: Line) -> ArrayVec {\n const EPSILON: f64 = 1e-9;\n let p0 = line.p0;\n let p1 = line.p1;\n let dx = p1.x - p0.x;\n let dy = p1.y - p0.y;\n let mut result = ArrayVec::new();\n match self {\n PathSeg::Line(l) => {\n let det = dx * (l.p1.y - l.p0.y) - dy * (l.p1.x - l.p0.x);\n if det.abs() < EPSILON {\n // Lines are coincident (or nearly so).\n return result;\n }\n let t = dx * (p0.y - l.p0.y) - dy * (p0.x - l.p0.x);\n // t = position on self\n let t = t / det;\n if (-EPSILON..=(1.0 + EPSILON)).contains(&t) {\n // u = position on probe line\n let u =\n (l.p0.x - p0.x) * (l.p1.y - l.p0.y) - (l.p0.y - p0.y) * (l.p1.x - l.p0.x);\n let u = u / det;\n if (0.0..=1.0).contains(&u) {\n result.push(LineIntersection::new(u, t));\n }\n }\n }\n PathSeg::Quad(q) => {\n // The basic technique here is to determine x and y as a quadratic polynomial\n // as a function of t. Then plug those values into the line equation for the\n // probe line (giving a sort of signed distance from the probe line) and solve\n // that for t.\n let (px0, px1, px2) = quadratic_bez_coefs(q.p0.x, q.p1.x, q.p2.x);\n let (py0, py1, py2) = quadratic_bez_coefs(q.p0.y, q.p1.y, q.p2.y);\n let c0 = dy * (px0 - p0.x) - dx * (py0 - p0.y);\n let c1 = dy * px1 - dx * py1;\n let c2 = dy * px2 - dx * py2;\n let invlen2 = (dx * dx + dy * dy).recip();\n for t in crate::common::solve_quadratic(c0, c1, c2) {\n if (-EPSILON..=(1.0 + EPSILON)).contains(&t) {\n let x = px0 + t * px1 + t * t * px2;\n let y = py0 + t * py1 + t * t * py2;\n let u = ((x - p0.x) * dx + (y - p0.y) * dy) * invlen2;\n if (0.0..=1.0).contains(&u) {\n result.push(LineIntersection::new(u, t));\n }\n }\n }\n }\n PathSeg::Cubic(c) => {\n // Same technique as above, but cubic polynomial.\n let (px0, px1, px2, px3) = cubic_bez_coefs(c.p0.x, c.p1.x, c.p2.x, c.p3.x);\n let (py0, py1, py2, py3) = cubic_bez_coefs(c.p0.y, c.p1.y, c.p2.y, c.p3.y);\n let c0 = dy * (px0 - p0.x) - dx * (py0 - p0.y);\n let c1 = dy * px1 - dx * py1;\n let c2 = dy * px2 - dx * py2;\n let c3 = dy * px3 - dx * py3;\n let invlen2 = (dx * dx + dy * dy).recip();\n for t in crate::common::solve_cubic(c0, c1, c2, c3) {\n if (-EPSILON..=(1.0 + EPSILON)).contains(&t) {\n let x = px0 + t * px1 + t * t * px2 + t * t * t * px3;\n let y = py0 + t * py1 + t * t * py2 + t * t * t * py3;\n let u = ((x - p0.x) * dx + (y - p0.y) * dy) * invlen2;\n if (0.0..=1.0).contains(&u) {\n result.push(LineIntersection::new(u, t));\n }\n }\n }\n }\n }\n result\n }\n\n /// Is this Bezier path finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n match self {\n PathSeg::Line(line) => line.is_finite(),\n PathSeg::Quad(quad_bez) => quad_bez.is_finite(),\n PathSeg::Cubic(cubic_bez) => cubic_bez.is_finite(),\n }\n }\n\n /// Is this Bezier path NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n match self {\n PathSeg::Line(line) => line.is_nan(),\n PathSeg::Quad(quad_bez) => quad_bez.is_nan(),\n PathSeg::Cubic(cubic_bez) => cubic_bez.is_nan(),\n }\n }\n\n #[inline]\n fn as_vec2_vec(&self) -> ArrayVec {\n let mut a = ArrayVec::new();\n match self {\n PathSeg::Line(l) => {\n a.push(l.p0.to_vec2());\n a.push(l.p1.to_vec2());\n }\n PathSeg::Quad(q) => {\n a.push(q.p0.to_vec2());\n a.push(q.p1.to_vec2());\n a.push(q.p2.to_vec2());\n }\n PathSeg::Cubic(c) => {\n a.push(c.p0.to_vec2());\n a.push(c.p1.to_vec2());\n a.push(c.p2.to_vec2());\n a.push(c.p3.to_vec2());\n }\n };\n a\n }\n\n /// Minimum distance between two [`PathSeg`]s.\n ///\n /// Returns a tuple of the distance, the path time `t1` of the closest point\n /// on the first `PathSeg`, and the path time `t2` of the closest point on the\n /// second `PathSeg`.\n pub fn min_dist(&self, other: PathSeg, accuracy: f64) -> MinDistance {\n let (distance, t1, t2) = crate::mindist::min_dist_param(\n &self.as_vec2_vec(),\n &other.as_vec2_vec(),\n (0.0, 1.0),\n (0.0, 1.0),\n accuracy,\n None,\n );\n MinDistance {\n distance: distance.sqrt(),\n t1,\n t2,\n }\n }\n\n /// Compute endpoint tangents of a path segment.\n ///\n /// This version is robust to the path segment not being a regular curve.\n pub(crate) fn tangents(&self) -> (Vec2, Vec2) {\n const EPS: f64 = 1e-12;\n match self {\n PathSeg::Line(l) => {\n let d = l.p1 - l.p0;\n (d, d)\n }\n PathSeg::Quad(q) => {\n let d01 = q.p1 - q.p0;\n let d0 = if d01.hypot2() > EPS { d01 } else { q.p2 - q.p0 };\n let d12 = q.p2 - q.p1;\n let d1 = if d12.hypot2() > EPS { d12 } else { q.p2 - q.p0 };\n (d0, d1)\n }\n PathSeg::Cubic(c) => {\n let d01 = c.p1 - c.p0;\n let d0 = if d01.hypot2() > EPS {\n d01\n } else {\n let d02 = c.p2 - c.p0;\n if d02.hypot2() > EPS {\n d02\n } else {\n c.p3 - c.p0\n }\n };\n let d23 = c.p3 - c.p2;\n let d1 = if d23.hypot2() > EPS {\n d23\n } else {\n let d13 = c.p3 - c.p1;\n if d13.hypot2() > EPS {\n d13\n } else {\n c.p3 - c.p0\n }\n };\n (d0, d1)\n }\n }\n }\n}\n\nimpl LineIntersection {\n fn new(line_t: f64, segment_t: f64) -> Self {\n LineIntersection { line_t, segment_t }\n }\n\n /// Is this line intersection finite?\n #[inline]\n pub fn is_finite(self) -> bool {\n self.line_t.is_finite() && self.segment_t.is_finite()\n }\n\n /// Is this line intersection NaN?\n #[inline]\n pub fn is_nan(self) -> bool {\n self.line_t.is_nan() || self.segment_t.is_nan()\n }\n}\n\n// Return polynomial coefficients given cubic bezier coordinates.\nfn quadratic_bez_coefs(x0: f64, x1: f64, x2: f64) -> (f64, f64, f64) {\n let p0 = x0;\n let p1 = 2.0 * x1 - 2.0 * x0;\n let p2 = x2 - 2.0 * x1 + x0;\n (p0, p1, p2)\n}\n\n// Return polynomial coefficients given cubic bezier coordinates.\nfn cubic_bez_coefs(x0: f64, x1: f64, x2: f64, x3: f64) -> (f64, f64, f64, f64) {\n let p0 = x0;\n let p1 = 3.0 * x1 - 3.0 * x0;\n let p2 = 3.0 * x2 - 6.0 * x1 + 3.0 * x0;\n let p3 = x3 - 3.0 * x2 + 3.0 * x1 - x0;\n (p0, p1, p2, p3)\n}\n\nimpl From for PathSeg {\n fn from(cubic_bez: CubicBez) -> PathSeg {\n PathSeg::Cubic(cubic_bez)\n }\n}\n\nimpl From for PathSeg {\n fn from(line: Line) -> PathSeg {\n PathSeg::Line(line)\n }\n}\n\nimpl From for PathSeg {\n fn from(quad_bez: QuadBez) -> PathSeg {\n PathSeg::Quad(quad_bez)\n }\n}\n\nimpl Shape for BezPath {\n type PathElementsIter<'iter> = core::iter::Copied>;\n\n fn path_elements(&self, _tolerance: f64) -> Self::PathElementsIter<'_> {\n self.0.iter().copied()\n }\n\n fn to_path(&self, _tolerance: f64) -> BezPath {\n self.clone()\n }\n\n fn into_path(self, _tolerance: f64) -> BezPath {\n self\n }\n\n /// Signed area.\n fn area(&self) -> f64 {\n self.elements().area()\n }\n\n fn perimeter(&self, accuracy: f64) -> f64 {\n self.elements().perimeter(accuracy)\n }\n\n /// Winding number of point.\n fn winding(&self, pt: Point) -> i32 {\n self.elements().winding(pt)\n }\n\n fn bounding_box(&self) -> Rect {\n self.elements().bounding_box()\n }\n\n fn as_path_slice(&self) -> Option<&[PathEl]> {\n Some(&self.0)\n }\n}\n\nimpl PathEl {\n /// Is this path element finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n match self {\n PathEl::MoveTo(p) => p.is_finite(),\n PathEl::LineTo(p) => p.is_finite(),\n PathEl::QuadTo(p, p2) => p.is_finite() && p2.is_finite(),\n PathEl::CurveTo(p, p2, p3) => p.is_finite() && p2.is_finite() && p3.is_finite(),\n PathEl::ClosePath => true,\n }\n }\n\n /// Is this path element NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n match self {\n PathEl::MoveTo(p) => p.is_nan(),\n PathEl::LineTo(p) => p.is_nan(),\n PathEl::QuadTo(p, p2) => p.is_nan() || p2.is_nan(),\n PathEl::CurveTo(p, p2, p3) => p.is_nan() || p2.is_nan() || p3.is_nan(),\n PathEl::ClosePath => false,\n }\n }\n\n /// Get the end point of the path element, if it exists.\n pub fn end_point(&self) -> Option {\n match self {\n PathEl::MoveTo(p) => Some(*p),\n PathEl::LineTo(p1) => Some(*p1),\n PathEl::QuadTo(_, p2) => Some(*p2),\n PathEl::CurveTo(_, _, p3) => Some(*p3),\n _ => None,\n }\n }\n}\n\n/// Implements [`Shape`] for a slice of [`PathEl`], provided that the first element of the slice is\n/// not a `PathEl::ClosePath`. If it is, several of these functions will panic.\n///\n/// If the slice starts with `LineTo`, `QuadTo`, or `CurveTo`, it will be treated as a `MoveTo`.\nimpl<'a> Shape for &'a [PathEl] {\n type PathElementsIter<'iter>\n\n = core::iter::Copied> where 'a: 'iter;\n\n #[inline]\n fn path_elements(&self, _tolerance: f64) -> Self::PathElementsIter<'_> {\n self.iter().copied()\n }\n\n fn to_path(&self, _tolerance: f64) -> BezPath {\n BezPath::from_vec(self.to_vec())\n }\n\n /// Signed area.\n fn area(&self) -> f64 {\n segments(self.iter().copied()).area()\n }\n\n fn perimeter(&self, accuracy: f64) -> f64 {\n segments(self.iter().copied()).perimeter(accuracy)\n }\n\n /// Winding number of point.\n fn winding(&self, pt: Point) -> i32 {\n segments(self.iter().copied()).winding(pt)\n }\n\n fn bounding_box(&self) -> Rect {\n segments(self.iter().copied()).bounding_box()\n }\n\n #[inline]\n fn as_path_slice(&self) -> Option<&[PathEl]> {\n Some(self)\n }\n}\n\n/// Implements [`Shape`] for an array of [`PathEl`], provided that the first element of the array is\n/// not a `PathEl::ClosePath`. If it is, several of these functions will panic.\n///\n/// If the array starts with `LineTo`, `QuadTo`, or `CurveTo`, it will be treated as a `MoveTo`.\nimpl Shape for [PathEl; N] {\n type PathElementsIter<'iter> = core::iter::Copied>;\n\n #[inline]\n fn path_elements(&self, _tolerance: f64) -> Self::PathElementsIter<'_> {\n self.iter().copied()\n }\n\n fn to_path(&self, _tolerance: f64) -> BezPath {\n BezPath::from_vec(self.to_vec())\n }\n\n /// Signed area.\n fn area(&self) -> f64 {\n segments(self.iter().copied()).area()\n }\n\n fn perimeter(&self, accuracy: f64) -> f64 {\n segments(self.iter().copied()).perimeter(accuracy)\n }\n\n /// Winding number of point.\n fn winding(&self, pt: Point) -> i32 {\n segments(self.iter().copied()).winding(pt)\n }\n\n fn bounding_box(&self) -> Rect {\n segments(self.iter().copied()).bounding_box()\n }\n\n #[inline]\n fn as_path_slice(&self) -> Option<&[PathEl]> {\n Some(self)\n }\n}\n\n/// An iterator for path segments.\npub struct PathSegIter {\n seg: PathSeg,\n ix: usize,\n}\n\nimpl Shape for PathSeg {\n type PathElementsIter<'iter> = PathSegIter;\n\n #[inline]\n fn path_elements(&self, _tolerance: f64) -> PathSegIter {\n PathSegIter { seg: *self, ix: 0 }\n }\n\n /// The area under the curve.\n ///\n /// We could just return `0`, but this seems more useful.\n fn area(&self) -> f64 {\n self.signed_area()\n }\n\n #[inline]\n fn perimeter(&self, accuracy: f64) -> f64 {\n self.arclen(accuracy)\n }\n\n fn winding(&self, _pt: Point) -> i32 {\n 0\n }\n\n #[inline]\n fn bounding_box(&self) -> Rect {\n ParamCurveExtrema::bounding_box(self)\n }\n\n fn as_line(&self) -> Option {\n if let PathSeg::Line(line) = self {\n Some(*line)\n } else {\n None\n }\n }\n}\n\nimpl Iterator for PathSegIter {\n type Item = PathEl;\n\n fn next(&mut self) -> Option {\n self.ix += 1;\n match (self.ix, self.seg) {\n // yes I could do some fancy bindings thing here but... :shrug:\n (1, PathSeg::Line(seg)) => Some(PathEl::MoveTo(seg.p0)),\n (1, PathSeg::Quad(seg)) => Some(PathEl::MoveTo(seg.p0)),\n (1, PathSeg::Cubic(seg)) => Some(PathEl::MoveTo(seg.p0)),\n (2, PathSeg::Line(seg)) => Some(PathEl::LineTo(seg.p1)),\n (2, PathSeg::Quad(seg)) => Some(PathEl::QuadTo(seg.p1, seg.p2)),\n (2, PathSeg::Cubic(seg)) => Some(PathEl::CurveTo(seg.p1, seg.p2, seg.p3)),\n _ => None,\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{Circle, DEFAULT_ACCURACY};\n\n use super::*;\n\n fn assert_approx_eq(x: f64, y: f64) {\n assert!((x - y).abs() < 1e-8, \"{x} != {y}\");\n }\n\n #[test]\n #[should_panic(expected = \"no open subpath\")]\n fn test_elements_to_segments_starts_on_closepath() {\n let mut path = BezPath::new();\n path.close_path();\n path.segments().next();\n }\n\n #[test]\n fn test_elements_to_segments_closepath_refers_to_last_moveto() {\n let mut path = BezPath::new();\n path.move_to((5.0, 5.0));\n path.line_to((15.0, 15.0));\n path.move_to((10.0, 10.0));\n path.line_to((15.0, 15.0));\n path.close_path();\n assert_eq!(\n path.segments().collect::>().last(),\n Some(&Line::new((15.0, 15.0), (10.0, 10.0)).into()),\n );\n }\n\n #[test]\n #[should_panic(expected = \"no open subpath\")]\n fn test_must_not_start_on_quad() {\n let mut path = BezPath::new();\n path.quad_to((5.0, 5.0), (10.0, 10.0));\n path.line_to((15.0, 15.0));\n path.close_path();\n }\n\n #[test]\n fn test_intersect_line() {\n let h_line = Line::new((0.0, 0.0), (100.0, 0.0));\n let v_line = Line::new((10.0, -10.0), (10.0, 10.0));\n let intersection = PathSeg::Line(h_line).intersect_line(v_line)[0];\n assert_approx_eq(intersection.segment_t, 0.1);\n assert_approx_eq(intersection.line_t, 0.5);\n\n let v_line = Line::new((-10.0, -10.0), (-10.0, 10.0));\n assert!(PathSeg::Line(h_line).intersect_line(v_line).is_empty());\n\n let v_line = Line::new((10.0, 10.0), (10.0, 20.0));\n assert!(PathSeg::Line(h_line).intersect_line(v_line).is_empty());\n }\n\n #[test]\n fn test_intersect_qad() {\n let q = QuadBez::new((0.0, -10.0), (10.0, 20.0), (20.0, -10.0));\n let v_line = Line::new((10.0, -10.0), (10.0, 10.0));\n assert_eq!(PathSeg::Quad(q).intersect_line(v_line).len(), 1);\n let intersection = PathSeg::Quad(q).intersect_line(v_line)[0];\n assert_approx_eq(intersection.segment_t, 0.5);\n assert_approx_eq(intersection.line_t, 0.75);\n\n let h_line = Line::new((0.0, 0.0), (100.0, 0.0));\n assert_eq!(PathSeg::Quad(q).intersect_line(h_line).len(), 2);\n }\n\n #[test]\n fn test_intersect_cubic() {\n let c = CubicBez::new((0.0, -10.0), (10.0, 20.0), (20.0, -20.0), (30.0, 10.0));\n let v_line = Line::new((10.0, -10.0), (10.0, 10.0));\n assert_eq!(PathSeg::Cubic(c).intersect_line(v_line).len(), 1);\n let intersection = PathSeg::Cubic(c).intersect_line(v_line)[0];\n assert_approx_eq(intersection.segment_t, 0.333333333);\n assert_approx_eq(intersection.line_t, 0.592592592);\n\n let h_line = Line::new((0.0, 0.0), (100.0, 0.0));\n assert_eq!(PathSeg::Cubic(c).intersect_line(h_line).len(), 3);\n }\n\n #[test]\n fn test_contains() {\n let mut path = BezPath::new();\n path.move_to((0.0, 0.0));\n path.line_to((1.0, 1.0));\n path.line_to((2.0, 0.0));\n path.close_path();\n assert_eq!(path.winding(Point::new(1.0, 0.5)), -1);\n assert!(path.contains(Point::new(1.0, 0.5)));\n }\n\n // get_seg(i) should produce the same results as path_segments().nth(i - 1).\n #[test]\n fn test_get_seg() {\n let circle = Circle::new((10.0, 10.0), 2.0).to_path(DEFAULT_ACCURACY);\n let segments = circle.path_segments(DEFAULT_ACCURACY).collect::>();\n let get_segs = (1..usize::MAX)\n .map_while(|i| circle.get_seg(i))\n .collect::>();\n assert_eq!(segments, get_segs);\n }\n\n #[test]\n fn test_control_box() {\n // a sort of map ping looking thing drawn with a single cubic\n // cbox is wildly different than tight box\n let path = BezPath::from_svg(\"M200,300 C50,50 350,50 200,300\").unwrap();\n assert_eq!(Rect::new(50.0, 50.0, 350.0, 300.0), path.control_box());\n assert!(path.control_box().area() > path.bounding_box().area());\n }\n\n #[test]\n fn test_reverse_unclosed() {\n let path = BezPath::from_svg(\"M10,10 Q40,40 60,10 L100,10 C125,10 150,50 125,60\").unwrap();\n let reversed = path.reverse_subpaths();\n assert_eq!(\n \"M125,60 C150,50 125,10 100,10 L60,10 Q40,40 10,10\",\n reversed.to_svg()\n );\n }\n\n #[test]\n fn test_reverse_closed_triangle() {\n let path = BezPath::from_svg(\"M100,100 L150,200 L50,200 Z\").unwrap();\n let reversed = path.reverse_subpaths();\n assert_eq!(\"M50,200 L150,200 L100,100 Z\", reversed.to_svg());\n }\n\n #[test]\n fn test_reverse_closed_shape() {\n let path = BezPath::from_svg(\n \"M125,100 Q200,150 175,300 C150,150 50,150 25,300 Q0,150 75,100 L100,50 Z\",\n )\n .unwrap();\n let reversed = path.reverse_subpaths();\n assert_eq!(\n \"M100,50 L75,100 Q0,150 25,300 C50,150 150,150 175,300 Q200,150 125,100 Z\",\n reversed.to_svg()\n );\n }\n\n #[test]\n fn test_reverse_multiple_subpaths() {\n let svg = \"M10,10 Q40,40 60,10 L100,10 C125,10 150,50 125,60 M100,100 L150,200 L50,200 Z M125,100 Q200,150 175,300 C150,150 50,150 25,300 Q0,150 75,100 L100,50 Z\";\n let expected_svg = \"M125,60 C150,50 125,10 100,10 L60,10 Q40,40 10,10 M50,200 L150,200 L100,100 Z M100,50 L75,100 Q0,150 25,300 C50,150 150,150 175,300 Q200,150 125,100 Z\";\n let path = BezPath::from_svg(svg).unwrap();\n let reversed = path.reverse_subpaths();\n assert_eq!(expected_svg, reversed.to_svg());\n }\n\n // https://github.com/fonttools/fonttools/blob/bf265ce49e0cae6f032420a4c80c31d8e16285b8/Tests/pens/reverseContourPen_test.py#L7\n #[test]\n fn test_reverse_lines() {\n let mut path = BezPath::new();\n path.move_to((0.0, 0.0));\n path.line_to((1.0, 1.0));\n path.line_to((2.0, 2.0));\n path.line_to((3.0, 3.0));\n path.close_path();\n let rev = path.reverse_subpaths();\n assert_eq!(\"M3,3 L2,2 L1,1 L0,0 Z\", rev.to_svg());\n }\n\n #[test]\n fn test_reverse_multiple_moves() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((2.0, 2.0).into()),\n PathEl::MoveTo((3.0, 3.0).into()),\n PathEl::ClosePath,\n PathEl::MoveTo((4.0, 4.0).into()),\n ],\n vec![\n PathEl::MoveTo((2.0, 2.0).into()),\n PathEl::MoveTo((3.0, 3.0).into()),\n PathEl::ClosePath,\n PathEl::MoveTo((4.0, 4.0).into()),\n ],\n )\n }\n\n // The following are direct port of fonttools'\n // reverseContourPen_test.py::test_reverse_pen, adapted to rust, excluding\n // test cases that don't apply because we don't implement\n // outputImpliedClosingLine=False.\n // https://github.com/fonttools/fonttools/blob/85c80be/Tests/pens/reverseContourPen_test.py#L6-L467\n\n #[test]\n fn test_reverse_closed_last_line_not_on_move() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::LineTo((1.0, 1.0).into()),\n PathEl::LineTo((2.0, 2.0).into()),\n PathEl::LineTo((3.0, 3.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((3.0, 3.0).into()),\n PathEl::LineTo((2.0, 2.0).into()),\n PathEl::LineTo((1.0, 1.0).into()),\n PathEl::LineTo((0.0, 0.0).into()), // closing line NOT implied\n PathEl::ClosePath,\n ],\n )\n }\n\n #[test]\n fn test_reverse_closed_last_line_overlaps_move() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::LineTo((1.0, 1.0).into()),\n PathEl::LineTo((2.0, 2.0).into()),\n PathEl::LineTo((0.0, 0.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::LineTo((2.0, 2.0).into()),\n PathEl::LineTo((1.0, 1.0).into()),\n PathEl::LineTo((0.0, 0.0).into()), // closing line NOT implied\n PathEl::ClosePath,\n ],\n )\n }\n\n #[test]\n fn test_reverse_closed_duplicate_line_following_move() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::LineTo((0.0, 0.0).into()),\n PathEl::LineTo((1.0, 1.0).into()),\n PathEl::LineTo((2.0, 2.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((2.0, 2.0).into()),\n PathEl::LineTo((1.0, 1.0).into()),\n PathEl::LineTo((0.0, 0.0).into()), // duplicate line retained\n PathEl::LineTo((0.0, 0.0).into()),\n PathEl::ClosePath,\n ],\n )\n }\n\n #[test]\n fn test_reverse_closed_two_lines() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::LineTo((1.0, 1.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((1.0, 1.0).into()),\n PathEl::LineTo((0.0, 0.0).into()), // closing line NOT implied\n PathEl::ClosePath,\n ],\n )\n }\n\n #[test]\n fn test_reverse_closed_last_curve_overlaps_move() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::CurveTo((1.0, 1.0).into(), (2.0, 2.0).into(), (3.0, 3.0).into()),\n PathEl::CurveTo((4.0, 4.0).into(), (5.0, 5.0).into(), (0.0, 0.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((0.0, 0.0).into()), // no extra lineTo added here\n PathEl::CurveTo((5.0, 5.0).into(), (4.0, 4.0).into(), (3.0, 3.0).into()),\n PathEl::CurveTo((2.0, 2.0).into(), (1.0, 1.0).into(), (0.0, 0.0).into()),\n PathEl::ClosePath,\n ],\n )\n }\n\n #[test]\n fn test_reverse_closed_last_curve_not_on_move() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::CurveTo((1.0, 1.0).into(), (2.0, 2.0).into(), (3.0, 3.0).into()),\n PathEl::CurveTo((4.0, 4.0).into(), (5.0, 5.0).into(), (6.0, 6.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((6.0, 6.0).into()), // the previously implied line\n PathEl::CurveTo((5.0, 5.0).into(), (4.0, 4.0).into(), (3.0, 3.0).into()),\n PathEl::CurveTo((2.0, 2.0).into(), (1.0, 1.0).into(), (0.0, 0.0).into()),\n PathEl::ClosePath,\n ],\n )\n }\n\n #[test]\n fn test_reverse_closed_line_curve_line() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::LineTo((1.0, 1.0).into()), // this line...\n PathEl::CurveTo((2.0, 2.0).into(), (3.0, 3.0).into(), (4.0, 4.0).into()),\n PathEl::CurveTo((5.0, 5.0).into(), (6.0, 6.0).into(), (7.0, 7.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((7.0, 7.0).into()),\n PathEl::CurveTo((6.0, 6.0).into(), (5.0, 5.0).into(), (4.0, 4.0).into()),\n PathEl::CurveTo((3.0, 3.0).into(), (2.0, 2.0).into(), (1.0, 1.0).into()),\n PathEl::LineTo((0.0, 0.0).into()), // ... does NOT become implied\n PathEl::ClosePath,\n ],\n )\n }\n\n #[test]\n fn test_reverse_closed_last_quad_overlaps_move() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::QuadTo((1.0, 1.0).into(), (2.0, 2.0).into()),\n PathEl::QuadTo((3.0, 3.0).into(), (0.0, 0.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((0.0, 0.0).into()), // no extra lineTo added here\n PathEl::QuadTo((3.0, 3.0).into(), (2.0, 2.0).into()),\n PathEl::QuadTo((1.0, 1.0).into(), (0.0, 0.0).into()),\n PathEl::ClosePath,\n ],\n )\n }\n\n #[test]\n fn test_reverse_closed_last_quad_not_on_move() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::QuadTo((1.0, 1.0).into(), (2.0, 2.0).into()),\n PathEl::QuadTo((3.0, 3.0).into(), (4.0, 4.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((4.0, 4.0).into()), // the previously implied line\n PathEl::QuadTo((3.0, 3.0).into(), (2.0, 2.0).into()),\n PathEl::QuadTo((1.0, 1.0).into(), (0.0, 0.0).into()),\n PathEl::ClosePath,\n ],\n )\n }\n\n #[test]\n fn test_reverse_closed_line_quad_line() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::LineTo((1.0, 1.0).into()), // this line...\n PathEl::QuadTo((2.0, 2.0).into(), (3.0, 3.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((3.0, 3.0).into()),\n PathEl::QuadTo((2.0, 2.0).into(), (1.0, 1.0).into()),\n PathEl::LineTo((0.0, 0.0).into()), // ... does NOT become implied\n PathEl::ClosePath,\n ],\n )\n }\n\n #[test]\n fn test_reverse_empty() {\n reverse_test_helper(vec![], vec![])\n }\n\n #[test]\n fn test_reverse_single_point() {\n reverse_test_helper(\n vec![PathEl::MoveTo((0.0, 0.0).into())],\n vec![PathEl::MoveTo((0.0, 0.0).into())],\n )\n }\n\n #[test]\n fn test_reverse_single_point_closed() {\n reverse_test_helper(\n vec![PathEl::MoveTo((0.0, 0.0).into()), PathEl::ClosePath],\n vec![PathEl::MoveTo((0.0, 0.0).into()), PathEl::ClosePath],\n )\n }\n\n #[test]\n fn test_reverse_single_line_open() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::LineTo((1.0, 1.0).into()),\n ],\n vec![\n PathEl::MoveTo((1.0, 1.0).into()),\n PathEl::LineTo((0.0, 0.0).into()),\n ],\n )\n }\n\n #[test]\n fn test_reverse_single_curve_open() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::CurveTo((1.0, 1.0).into(), (2.0, 2.0).into(), (3.0, 3.0).into()),\n ],\n vec![\n PathEl::MoveTo((3.0, 3.0).into()),\n PathEl::CurveTo((2.0, 2.0).into(), (1.0, 1.0).into(), (0.0, 0.0).into()),\n ],\n )\n }\n\n #[test]\n fn test_reverse_curve_line_open() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::CurveTo((1.0, 1.0).into(), (2.0, 2.0).into(), (3.0, 3.0).into()),\n PathEl::LineTo((4.0, 4.0).into()),\n ],\n vec![\n PathEl::MoveTo((4.0, 4.0).into()),\n PathEl::LineTo((3.0, 3.0).into()),\n PathEl::CurveTo((2.0, 2.0).into(), (1.0, 1.0).into(), (0.0, 0.0).into()),\n ],\n )\n }\n\n #[test]\n fn test_reverse_line_curve_open() {\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 0.0).into()),\n PathEl::LineTo((1.0, 1.0).into()),\n PathEl::CurveTo((2.0, 2.0).into(), (3.0, 3.0).into(), (4.0, 4.0).into()),\n ],\n vec![\n PathEl::MoveTo((4.0, 4.0).into()),\n PathEl::CurveTo((3.0, 3.0).into(), (2.0, 2.0).into(), (1.0, 1.0).into()),\n PathEl::LineTo((0.0, 0.0).into()),\n ],\n )\n }\n\n #[test]\n fn test_reverse_duplicate_point_after_move() {\n // Test case from: https://github.com/googlei18n/cu2qu/issues/51#issue-179370514\n // Simplified to only use atomic PathEl::QuadTo (no QuadSplines).\n reverse_test_helper(\n vec![\n PathEl::MoveTo((848.0, 348.0).into()),\n PathEl::LineTo((848.0, 348.0).into()),\n PathEl::QuadTo((848.0, 526.0).into(), (449.0, 704.0).into()),\n PathEl::QuadTo((848.0, 171.0).into(), (848.0, 348.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((848.0, 348.0).into()),\n PathEl::QuadTo((848.0, 171.0).into(), (449.0, 704.0).into()),\n PathEl::QuadTo((848.0, 526.0).into(), (848.0, 348.0).into()),\n PathEl::LineTo((848.0, 348.0).into()),\n PathEl::ClosePath,\n ],\n )\n }\n\n #[test]\n fn test_reverse_duplicate_point_at_end() {\n // Test case from: https://github.com/googlefonts/fontmake/issues/572\n reverse_test_helper(\n vec![\n PathEl::MoveTo((0.0, 651.0).into()),\n PathEl::LineTo((0.0, 101.0).into()),\n PathEl::LineTo((0.0, 101.0).into()),\n PathEl::LineTo((0.0, 651.0).into()),\n PathEl::LineTo((0.0, 651.0).into()),\n PathEl::ClosePath,\n ],\n vec![\n PathEl::MoveTo((0.0, 651.0).into()),\n PathEl::LineTo((0.0, 651.0).into()),\n PathEl::LineTo((0.0, 101.0).into()),\n PathEl::LineTo((0.0, 101.0).into()),\n PathEl::LineTo((0.0, 651.0).into()),\n PathEl::ClosePath,\n ],\n )\n }\n\n fn reverse_test_helper(contour: Vec, expected: Vec) {\n assert_eq!(BezPath(contour).reverse_subpaths().0, expected);\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/circle.rs", "sha256": "cedf586b744630a3b745b4025157df8f10d7de9540c4090d8a41e57fbbb105ab", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Implementation of circle shape.\n\nuse core::{\n f64::consts::{FRAC_PI_2, PI},\n iter,\n ops::{Add, Mul, Sub},\n};\n\nuse crate::{Affine, Arc, ArcAppendIter, Ellipse, PathEl, Point, Rect, Shape, Vec2};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// A circle.\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct Circle {\n /// The center.\n pub center: Point,\n /// The radius.\n pub radius: f64,\n}\n\nimpl Circle {\n /// A new circle from center and radius.\n #[inline]\n pub fn new(center: impl Into, radius: f64) -> Circle {\n Circle {\n center: center.into(),\n radius,\n }\n }\n\n /// Create a [`CircleSegment`] by cutting out parts of this circle.\n pub fn segment(self, inner_radius: f64, start_angle: f64, sweep_angle: f64) -> CircleSegment {\n CircleSegment {\n center: self.center,\n outer_radius: self.radius,\n inner_radius,\n start_angle,\n sweep_angle,\n }\n }\n\n /// Is this circle finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n self.center.is_finite() && self.radius.is_finite()\n }\n\n /// Is this circle NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n self.center.is_nan() || self.radius.is_nan()\n }\n}\n\nimpl Add for Circle {\n type Output = Circle;\n\n #[inline]\n fn add(self, v: Vec2) -> Circle {\n Circle {\n center: self.center + v,\n radius: self.radius,\n }\n }\n}\n\nimpl Sub for Circle {\n type Output = Circle;\n\n #[inline]\n fn sub(self, v: Vec2) -> Circle {\n Circle {\n center: self.center - v,\n radius: self.radius,\n }\n }\n}\n\nimpl Mul for Affine {\n type Output = Ellipse;\n fn mul(self, other: Circle) -> Self::Output {\n self * Ellipse::from(other)\n }\n}\n\n#[doc(hidden)]\npub struct CirclePathIter {\n circle: Circle,\n delta_th: f64,\n arm_len: f64,\n ix: usize,\n n: usize,\n}\n\nimpl Shape for Circle {\n type PathElementsIter<'iter> = CirclePathIter;\n\n fn path_elements(&self, tolerance: f64) -> CirclePathIter {\n let scaled_err = self.radius.abs() / tolerance;\n let (n, arm_len) = if scaled_err < 1.0 / 1.9608e-4 {\n // Solution from http://spencermortensen.com/articles/bezier-circle/\n (4, 0.551915024494)\n } else {\n // This is empirically determined to fall within error tolerance.\n let n = (1.1163 * scaled_err).powf(1.0 / 6.0).ceil() as usize;\n // Note: this isn't minimum error, but it is simple and we can easily\n // estimate the error.\n let arm_len = (4.0 / 3.0) * (FRAC_PI_2 / (n as f64)).tan();\n (n, arm_len)\n };\n CirclePathIter {\n circle: *self,\n delta_th: 2.0 * PI / (n as f64),\n arm_len,\n ix: 0,\n n,\n }\n }\n\n #[inline]\n fn area(&self) -> f64 {\n PI * self.radius.powi(2)\n }\n\n #[inline]\n fn perimeter(&self, _accuracy: f64) -> f64 {\n (2.0 * PI * self.radius).abs()\n }\n\n fn winding(&self, pt: Point) -> i32 {\n if (pt - self.center).hypot2() < self.radius.powi(2) {\n 1\n } else {\n 0\n }\n }\n\n #[inline]\n fn bounding_box(&self) -> Rect {\n let r = self.radius.abs();\n let (x, y) = self.center.into();\n Rect::new(x - r, y - r, x + r, y + r)\n }\n\n fn as_circle(&self) -> Option {\n Some(*self)\n }\n}\n\nimpl Iterator for CirclePathIter {\n type Item = PathEl;\n\n fn next(&mut self) -> Option {\n let a = self.arm_len;\n let r = self.circle.radius;\n let (x, y) = self.circle.center.into();\n let ix = self.ix;\n self.ix += 1;\n if ix == 0 {\n Some(PathEl::MoveTo(Point::new(x + r, y)))\n } else if ix <= self.n {\n let th1 = self.delta_th * (ix as f64);\n let th0 = th1 - self.delta_th;\n let (s0, c0) = th0.sin_cos();\n let (s1, c1) = if ix == self.n {\n (0.0, 1.0)\n } else {\n th1.sin_cos()\n };\n Some(PathEl::CurveTo(\n Point::new(x + r * (c0 - a * s0), y + r * (s0 + a * c0)),\n Point::new(x + r * (c1 + a * s1), y + r * (s1 - a * c1)),\n Point::new(x + r * c1, y + r * s1),\n ))\n } else if ix == self.n + 1 {\n Some(PathEl::ClosePath)\n } else {\n None\n }\n }\n}\n\n/// A segment of a circle.\n///\n/// If `inner_radius > 0`, then the shape will be a doughnut segment.\npub struct CircleSegment {\n /// The center.\n pub center: Point,\n /// The outer radius.\n pub outer_radius: f64,\n /// The inner radius.\n pub inner_radius: f64,\n /// The angle to start drawing the segment (in radians).\n pub start_angle: f64,\n /// The arc length of the segment (in radians).\n pub sweep_angle: f64,\n}\n\nimpl CircleSegment {\n /// Create a `CircleSegment` out of its constituent parts.\n pub fn new(\n center: impl Into,\n outer_radius: f64,\n inner_radius: f64,\n start_angle: f64,\n sweep_angle: f64,\n ) -> Self {\n CircleSegment {\n center: center.into(),\n outer_radius,\n inner_radius,\n start_angle,\n sweep_angle,\n }\n }\n\n /// Is this circle segment finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n self.center.is_finite()\n && self.outer_radius.is_finite()\n && self.inner_radius.is_finite()\n && self.start_angle.is_finite()\n && self.sweep_angle.is_finite()\n }\n\n /// Is this circle segment NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n self.center.is_nan()\n || self.outer_radius.is_nan()\n || self.inner_radius.is_nan()\n || self.start_angle.is_nan()\n || self.sweep_angle.is_nan()\n }\n}\n\nimpl Add for CircleSegment {\n type Output = CircleSegment;\n\n #[inline]\n fn add(self, v: Vec2) -> Self {\n Self {\n center: self.center + v,\n ..self\n }\n }\n}\n\nimpl Sub for CircleSegment {\n type Output = CircleSegment;\n\n #[inline]\n fn sub(self, v: Vec2) -> Self {\n Self {\n center: self.center - v,\n ..self\n }\n }\n}\n\ntype CircleSegmentPathIter = iter::Chain<\n iter::Chain<\n iter::Chain, iter::Once>, ArcAppendIter>,\n iter::Once,\n >,\n ArcAppendIter,\n>;\n\nimpl Shape for CircleSegment {\n type PathElementsIter<'iter> = CircleSegmentPathIter;\n\n fn path_elements(&self, tolerance: f64) -> CircleSegmentPathIter {\n iter::once(PathEl::MoveTo(point_on_circle(\n self.center,\n self.inner_radius,\n self.start_angle,\n )))\n // First radius\n .chain(iter::once(PathEl::LineTo(point_on_circle(\n self.center,\n self.outer_radius,\n self.start_angle,\n ))))\n // outer arc\n .chain(\n Arc {\n center: self.center,\n radii: Vec2::new(self.outer_radius, self.outer_radius),\n start_angle: self.start_angle,\n sweep_angle: self.sweep_angle,\n x_rotation: 0.0,\n }\n .append_iter(tolerance),\n )\n // second radius\n .chain(iter::once(PathEl::LineTo(point_on_circle(\n self.center,\n self.inner_radius,\n self.start_angle + self.sweep_angle,\n ))))\n // inner arc\n .chain(\n Arc {\n center: self.center,\n radii: Vec2::new(self.inner_radius, self.inner_radius),\n start_angle: self.start_angle + self.sweep_angle,\n sweep_angle: -self.sweep_angle,\n x_rotation: 0.0,\n }\n .append_iter(tolerance),\n )\n }\n\n #[inline]\n fn area(&self) -> f64 {\n 0.5 * (self.outer_radius.powi(2) - self.inner_radius.powi(2)).abs() * self.sweep_angle\n }\n\n #[inline]\n fn perimeter(&self, _accuracy: f64) -> f64 {\n 2.0 * (self.outer_radius - self.inner_radius).abs()\n + self.sweep_angle * (self.inner_radius + self.outer_radius)\n }\n\n fn winding(&self, pt: Point) -> i32 {\n let angle = (pt - self.center).atan2();\n if angle < self.start_angle || angle > self.start_angle + self.sweep_angle {\n return 0;\n }\n let dist2 = (pt - self.center).hypot2();\n if (dist2 < self.outer_radius.powi(2) && dist2 > self.inner_radius.powi(2)) ||\n // case where outer_radius < inner_radius\n (dist2 < self.inner_radius.powi(2) && dist2 > self.outer_radius.powi(2))\n {\n 1\n } else {\n 0\n }\n }\n\n #[inline]\n fn bounding_box(&self) -> Rect {\n // todo this is currently not tight\n let r = self.inner_radius.max(self.outer_radius);\n let (x, y) = self.center.into();\n Rect::new(x - r, y - r, x + r, y + r)\n }\n}\n\n#[inline]\nfn point_on_circle(center: Point, radius: f64, angle: f64) -> Point {\n let (angle_sin, angle_cos) = angle.sin_cos();\n center\n + Vec2 {\n x: angle_cos * radius,\n y: angle_sin * radius,\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{Circle, Point, Shape};\n use std::f64::consts::PI;\n\n fn assert_approx_eq(x: f64, y: f64) {\n // Note: we might want to be more rigorous in testing the accuracy\n // of the conversion into Béziers. But this seems good enough.\n assert!((x - y).abs() < 1e-7, \"{x} != {y}\");\n }\n\n #[test]\n fn area_sign() {\n let center = Point::new(5.0, 5.0);\n let c = Circle::new(center, 5.0);\n assert_approx_eq(c.area(), 25.0 * PI);\n\n assert_eq!(c.winding(center), 1);\n\n let p = c.to_path(1e-9);\n assert_approx_eq(c.area(), p.area());\n assert_eq!(c.winding(center), p.winding(center));\n\n let c_neg_radius = Circle::new(center, -5.0);\n assert_approx_eq(c_neg_radius.area(), 25.0 * PI);\n\n assert_eq!(c_neg_radius.winding(center), 1);\n\n let p_neg_radius = c_neg_radius.to_path(1e-9);\n assert_approx_eq(c_neg_radius.area(), p_neg_radius.area());\n assert_eq!(c_neg_radius.winding(center), p_neg_radius.winding(center));\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/common.rs", "sha256": "8a90e233ad3fcaeb2fe0fd73f65b6e8dd77bc7e97c6db6a8cc9fdb1d486a6b9f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Common mathematical operations\n\n#![allow(missing_docs)]\n\nuse arrayvec::ArrayVec;\n\n/// Defines a trait that chooses between libstd or libm implementations of float methods.\nmacro_rules! define_float_funcs {\n ($(\n fn $name:ident(self $(,$arg:ident: $arg_ty:ty)*) -> $ret:ty\n => $lname:ident/$lfname:ident;\n )+) => {\n #[cfg(not(feature = \"std\"))]\n pub(crate) trait FloatFuncs : Sized {\n /// Special implementation for signum, because libm doesn't have it.\n fn signum(self) -> Self;\n\n $(fn $name(self $(,$arg: $arg_ty)*) -> $ret;)+\n }\n\n #[cfg(not(feature = \"std\"))]\n impl FloatFuncs for f32 {\n #[inline]\n fn signum(self) -> f32 {\n if self.is_nan() {\n f32::NAN\n } else {\n 1.0_f32.copysign(self)\n }\n }\n\n $(fn $name(self $(,$arg: $arg_ty)*) -> $ret {\n #[cfg(feature = \"libm\")]\n return libm::$lfname(self $(,$arg as _)*);\n\n #[cfg(not(feature = \"libm\"))]\n compile_error!(\"kurbo requires either the `std` or `libm` feature\")\n })+\n }\n\n #[cfg(not(feature = \"std\"))]\n impl FloatFuncs for f64 {\n #[inline]\n fn signum(self) -> f64 {\n if self.is_nan() {\n f64::NAN\n } else {\n 1.0_f64.copysign(self)\n }\n }\n\n $(fn $name(self $(,$arg: $arg_ty)*) -> $ret {\n #[cfg(feature = \"libm\")]\n return libm::$lname(self $(,$arg as _)*);\n\n #[cfg(not(feature = \"libm\"))]\n compile_error!(\"kurbo requires either the `std` or `libm` feature\")\n })+\n }\n }\n}\n\ndefine_float_funcs! {\n fn abs(self) -> Self => fabs/fabsf;\n fn acos(self) -> Self => acos/acosf;\n fn atan2(self, other: Self) -> Self => atan2/atan2f;\n fn cbrt(self) -> Self => cbrt/cbrtf;\n fn ceil(self) -> Self => ceil/ceilf;\n fn cos(self) -> Self => cos/cosf;\n fn copysign(self, sign: Self) -> Self => copysign/copysignf;\n fn floor(self) -> Self => floor/floorf;\n fn hypot(self, other: Self) -> Self => hypot/hypotf;\n fn ln(self) -> Self => log/logf;\n fn log2(self) -> Self => log2/log2f;\n fn mul_add(self, a: Self, b: Self) -> Self => fma/fmaf;\n fn powi(self, n: i32) -> Self => pow/powf;\n fn powf(self, n: Self) -> Self => pow/powf;\n fn round(self) -> Self => round/roundf;\n fn sin_cos(self) -> (Self, Self) => sincos/sincosf;\n fn sqrt(self) -> Self => sqrt/sqrtf;\n fn tan(self) -> Self => tan/tanf;\n fn trunc(self) -> Self => trunc/truncf;\n}\n\n/// Adds convenience methods to `f32` and `f64`.\npub trait FloatExt {\n /// Rounds to the nearest integer away from zero,\n /// unless the provided value is already an integer.\n ///\n /// It is to `ceil` what `trunc` is to `floor`.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::common::FloatExt;\n ///\n /// let f = 3.7_f64;\n /// let g = 3.0_f64;\n /// let h = -3.7_f64;\n /// let i = -5.1_f32;\n ///\n /// assert_eq!(f.expand(), 4.0);\n /// assert_eq!(g.expand(), 3.0);\n /// assert_eq!(h.expand(), -4.0);\n /// assert_eq!(i.expand(), -6.0);\n /// ```\n fn expand(&self) -> T;\n}\n\nimpl FloatExt for f64 {\n #[inline]\n fn expand(&self) -> f64 {\n self.abs().ceil().copysign(*self)\n }\n}\n\nimpl FloatExt for f32 {\n #[inline]\n fn expand(&self) -> f32 {\n self.abs().ceil().copysign(*self)\n }\n}\n\n/// Find real roots of cubic equation.\n///\n/// The implementation is not (yet) fully robust, but it does handle the case\n/// where `c3` is zero (in that case, solving the quadratic equation).\n///\n/// See: \n///\n/// That implementation is in turn based on Jim Blinn's \"How to Solve a Cubic\n/// Equation\", which is masterful.\n///\n/// Return values of x for which c0 + c1 x + c2 x² + c3 x³ = 0.\npub fn solve_cubic(c0: f64, c1: f64, c2: f64, c3: f64) -> ArrayVec {\n let mut result = ArrayVec::new();\n let c3_recip = c3.recip();\n const ONETHIRD: f64 = 1. / 3.;\n let scaled_c2 = c2 * (ONETHIRD * c3_recip);\n let scaled_c1 = c1 * (ONETHIRD * c3_recip);\n let scaled_c0 = c0 * c3_recip;\n if !(scaled_c0.is_finite() && scaled_c1.is_finite() && scaled_c2.is_finite()) {\n // cubic coefficient is zero or nearly so.\n return solve_quadratic(c0, c1, c2).iter().copied().collect();\n }\n let (c0, c1, c2) = (scaled_c0, scaled_c1, scaled_c2);\n // (d0, d1, d2) is called \"Delta\" in article\n let d0 = (-c2).mul_add(c2, c1);\n let d1 = (-c1).mul_add(c2, c0);\n let d2 = c2 * c0 - c1 * c1;\n // d is called \"Discriminant\"\n let d = 4.0 * d0 * d2 - d1 * d1;\n // de is called \"Depressed.x\", Depressed.y = d0\n let de = (-2.0 * c2).mul_add(d0, d1);\n // TODO: handle the cases where these intermediate results overflow.\n if d < 0.0 {\n let sq = (-0.25 * d).sqrt();\n let r = -0.5 * de;\n let t1 = (r + sq).cbrt() + (r - sq).cbrt();\n result.push(t1 - c2);\n } else if d == 0.0 {\n let t1 = (-d0).sqrt().copysign(de);\n result.push(t1 - c2);\n result.push(-2.0 * t1 - c2);\n } else {\n let th = d.sqrt().atan2(-de) * ONETHIRD;\n // (th_cos, th_sin) is called \"CubicRoot\"\n let (th_sin, th_cos) = th.sin_cos();\n // (r0, r1, r2) is called \"Root\"\n let r0 = th_cos;\n let ss3 = th_sin * 3.0f64.sqrt();\n let r1 = 0.5 * (-th_cos + ss3);\n let r2 = 0.5 * (-th_cos - ss3);\n let t = 2.0 * (-d0).sqrt();\n result.push(t.mul_add(r0, -c2));\n result.push(t.mul_add(r1, -c2));\n result.push(t.mul_add(r2, -c2));\n }\n result\n}\n\n/// Find real roots of quadratic equation.\n///\n/// Return values of x for which c0 + c1 x + c2 x² = 0.\n///\n/// This function tries to be quite numerically robust. If the equation\n/// is nearly linear, it will return the root ignoring the quadratic term;\n/// the other root might be out of representable range. In the degenerate\n/// case where all coefficients are zero, so that all values of x satisfy\n/// the equation, a single `0.0` is returned.\npub fn solve_quadratic(c0: f64, c1: f64, c2: f64) -> ArrayVec {\n let mut result = ArrayVec::new();\n let sc0 = c0 * c2.recip();\n let sc1 = c1 * c2.recip();\n if !sc0.is_finite() || !sc1.is_finite() {\n // c2 is zero or very small, treat as linear eqn\n let root = -c0 / c1;\n if root.is_finite() {\n result.push(root);\n } else if c0 == 0.0 && c1 == 0.0 {\n // Degenerate case\n result.push(0.0);\n }\n return result;\n }\n let arg = sc1 * sc1 - 4. * sc0;\n let root1 = if !arg.is_finite() {\n // Likely, calculation of sc1 * sc1 overflowed. Find one root\n // using sc1 x + x² = 0, other root as sc0 / root1.\n -sc1\n } else {\n if arg < 0.0 {\n return result;\n } else if arg == 0.0 {\n result.push(-0.5 * sc1);\n return result;\n }\n // See https://math.stackexchange.com/questions/866331\n -0.5 * (sc1 + arg.sqrt().copysign(sc1))\n };\n let root2 = sc0 / root1;\n if root2.is_finite() {\n // Sort just to be friendly and make results deterministic.\n if root2 > root1 {\n result.push(root1);\n result.push(root2);\n } else {\n result.push(root2);\n result.push(root1);\n }\n } else {\n result.push(root1);\n }\n result\n}\n\n/// Compute epsilon relative to coefficient.\n///\n/// A helper function from the Orellana and De Michele paper.\nfn eps_rel(raw: f64, a: f64) -> f64 {\n if a == 0.0 {\n raw.abs()\n } else {\n ((raw - a) / a).abs()\n }\n}\n\n/// Find real roots of a quartic equation.\n///\n/// This is a fairly literal implementation of the method described in:\n/// Algorithm 1010: Boosting Efficiency in Solving Quartic Equations with\n/// No Compromise in Accuracy, Orellana and De Michele, ACM\n/// Transactions on Mathematical Software, Vol. 46, No. 2, May 2020.\npub fn solve_quartic(c0: f64, c1: f64, c2: f64, c3: f64, c4: f64) -> ArrayVec {\n if c4 == 0.0 {\n return solve_cubic(c0, c1, c2, c3).iter().copied().collect();\n }\n if c0 == 0.0 {\n // Note: appends 0 root at end, doesn't sort. We might want to do that.\n return solve_cubic(c1, c2, c3, c4)\n .iter()\n .copied()\n .chain(Some(0.0))\n .collect();\n }\n let a = c3 / c4;\n let b = c2 / c4;\n let c = c1 / c4;\n let d = c0 / c4;\n if let Some(result) = solve_quartic_inner(a, b, c, d, false) {\n return result;\n }\n // Do polynomial rescaling\n const K_Q: f64 = 7.16e76;\n for rescale in [false, true] {\n if let Some(result) = solve_quartic_inner(\n a / K_Q,\n b / K_Q.powi(2),\n c / K_Q.powi(3),\n d / K_Q.powi(4),\n rescale,\n ) {\n return result.iter().map(|x| x * K_Q).collect();\n }\n }\n // Overflow happened, just return no roots.\n //println!(\"overflow, no roots returned\");\n Default::default()\n}\n\nfn solve_quartic_inner(a: f64, b: f64, c: f64, d: f64, rescale: bool) -> Option> {\n factor_quartic_inner(a, b, c, d, rescale).map(|quadratics| {\n quadratics\n .iter()\n .flat_map(|(a, b)| solve_quadratic(*b, *a, 1.0))\n .collect()\n })\n}\n\n/// Factor a quartic into two quadratics.\n///\n/// Attempt to factor a quartic equation into two quadratic equations. Returns `None` either if there\n/// is overflow (in which case rescaling might succeed) or the factorization would result in\n/// complex coefficients.\n///\n/// Discussion question: distinguish the two cases in return value?\npub fn factor_quartic_inner(\n a: f64,\n b: f64,\n c: f64,\n d: f64,\n rescale: bool,\n) -> Option> {\n let calc_eps_q = |a1, b1, a2, b2| {\n let eps_a = eps_rel(a1 + a2, a);\n let eps_b = eps_rel(b1 + a1 * a2 + b2, b);\n let eps_c = eps_rel(b1 * a2 + a1 * b2, c);\n eps_a + eps_b + eps_c\n };\n let calc_eps_t = |a1, b1, a2, b2| calc_eps_q(a1, b1, a2, b2) + eps_rel(b1 * b2, d);\n let disc = 9. * a * a - 24. * b;\n let s = if disc >= 0.0 {\n -2. * b / (3. * a + disc.sqrt().copysign(a))\n } else {\n -0.25 * a\n };\n let a_prime = a + 4. * s;\n let b_prime = b + 3. * s * (a + 2. * s);\n let c_prime = c + s * (2. * b + s * (3. * a + 4. * s));\n let d_prime = d + s * (c + s * (b + s * (a + s)));\n let g_prime;\n let h_prime;\n const K_C: f64 = 3.49e102;\n if rescale {\n let a_prime_s = a_prime / K_C;\n let b_prime_s = b_prime / K_C;\n let c_prime_s = c_prime / K_C;\n let d_prime_s = d_prime / K_C;\n g_prime = a_prime_s * c_prime_s - (4. / K_C) * d_prime_s - (1. / 3.) * b_prime_s.powi(2);\n h_prime = (a_prime_s * c_prime_s + (8. / K_C) * d_prime_s - (2. / 9.) * b_prime_s.powi(2))\n * (1. / 3.)\n * b_prime_s\n - c_prime_s * (c_prime_s / K_C)\n - a_prime_s.powi(2) * d_prime_s;\n } else {\n g_prime = a_prime * c_prime - 4. * d_prime - (1. / 3.) * b_prime.powi(2);\n h_prime =\n (a_prime * c_prime + 8. * d_prime - (2. / 9.) * b_prime.powi(2)) * (1. / 3.) * b_prime\n - c_prime.powi(2)\n - a_prime.powi(2) * d_prime;\n }\n if !(g_prime.is_finite() && h_prime.is_finite()) {\n return None;\n }\n let phi = depressed_cubic_dominant(g_prime, h_prime);\n let phi = if rescale { phi * K_C } else { phi };\n let l_1 = a * 0.5;\n let l_3 = (1. / 6.) * b + 0.5 * phi;\n let delt_2 = c - a * l_3;\n let d_2_cand_1 = (2. / 3.) * b - phi - l_1 * l_1;\n let l_2_cand_1 = 0.5 * delt_2 / d_2_cand_1;\n let l_2_cand_2 = 2. * (d - l_3 * l_3) / delt_2;\n let d_2_cand_2 = 0.5 * delt_2 / l_2_cand_2;\n let d_2_cand_3 = d_2_cand_1;\n let l_2_cand_3 = l_2_cand_2;\n let mut d_2_best = 0.0;\n let mut l_2_best = 0.0;\n let mut eps_l_best = 0.0;\n for (i, (d_2, l_2)) in [\n (d_2_cand_1, l_2_cand_1),\n (d_2_cand_2, l_2_cand_2),\n (d_2_cand_3, l_2_cand_3),\n ]\n .iter()\n .enumerate()\n {\n let eps_0 = eps_rel(d_2 + l_1 * l_1 + 2. * l_3, b);\n let eps_1 = eps_rel(2. * (d_2 * l_2 + l_1 * l_3), c);\n let eps_2 = eps_rel(d_2 * l_2 * l_2 + l_3 * l_3, d);\n let eps_l = eps_0 + eps_1 + eps_2;\n if i == 0 || eps_l < eps_l_best {\n d_2_best = *d_2;\n l_2_best = *l_2;\n eps_l_best = eps_l;\n }\n }\n let d_2 = d_2_best;\n let l_2 = l_2_best;\n let mut alpha_1;\n let mut beta_1;\n let mut alpha_2;\n let mut beta_2;\n //println!(\"phi = {}, d_2 = {}\", phi, d_2);\n if d_2 < 0.0 {\n let sq = (-d_2).sqrt();\n alpha_1 = l_1 + sq;\n beta_1 = l_3 + sq * l_2;\n alpha_2 = l_1 - sq;\n beta_2 = l_3 - sq * l_2;\n if beta_2.abs() < beta_1.abs() {\n beta_2 = d / beta_1;\n } else if beta_2.abs() > beta_1.abs() {\n beta_1 = d / beta_2;\n }\n let cands;\n if alpha_1.abs() != alpha_2.abs() {\n if alpha_1.abs() < alpha_2.abs() {\n let a1_cand_1 = (c - beta_1 * alpha_2) / beta_2;\n let a1_cand_2 = (b - beta_2 - beta_1) / alpha_2;\n let a1_cand_3 = a - alpha_2;\n // Note: cand 3 is first because it is infallible, simplifying logic\n cands = [\n (a1_cand_3, alpha_2),\n (a1_cand_1, alpha_2),\n (a1_cand_2, alpha_2),\n ];\n } else {\n let a2_cand_1 = (c - alpha_1 * beta_2) / beta_1;\n let a2_cand_2 = (b - beta_2 - beta_1) / alpha_1;\n let a2_cand_3 = a - alpha_1;\n cands = [\n (alpha_1, a2_cand_3),\n (alpha_1, a2_cand_1),\n (alpha_1, a2_cand_2),\n ];\n }\n let mut eps_q_best = 0.0;\n for (i, (a1, a2)) in cands.iter().enumerate() {\n if a1.is_finite() && a2.is_finite() {\n let eps_q = calc_eps_q(*a1, beta_1, *a2, beta_2);\n if i == 0 || eps_q < eps_q_best {\n alpha_1 = *a1;\n alpha_2 = *a2;\n eps_q_best = eps_q;\n }\n }\n }\n }\n } else if d_2 == 0.0 {\n let d_3 = d - l_3 * l_3;\n alpha_1 = l_1;\n beta_1 = l_3 + (-d_3).sqrt();\n alpha_2 = l_1;\n beta_2 = l_3 - (-d_3).sqrt();\n if beta_1.abs() > beta_2.abs() {\n beta_2 = d / beta_1;\n } else if beta_2.abs() > beta_1.abs() {\n beta_1 = d / beta_2;\n }\n // TODO: handle case d_2 is very small?\n } else {\n // This case means no real roots; in the most general case we might want\n // to factor into quadratic equations with complex coefficients.\n return None;\n }\n // Newton-Raphson iteration on alpha/beta coeff's.\n let mut eps_t = calc_eps_t(alpha_1, beta_1, alpha_2, beta_2);\n for _ in 0..8 {\n //println!(\"a1 {} b1 {} a2 {} b2 {}\", alpha_1, beta_1, alpha_2, beta_2);\n //println!(\"eps_t = {:e}\", eps_t);\n if eps_t == 0.0 {\n break;\n }\n let f_0 = beta_1 * beta_2 - d;\n let f_1 = beta_1 * alpha_2 + alpha_1 * beta_2 - c;\n let f_2 = beta_1 + alpha_1 * alpha_2 + beta_2 - b;\n let f_3 = alpha_1 + alpha_2 - a;\n let c_1 = alpha_1 - alpha_2;\n let det_j = beta_1 * beta_1 - beta_1 * (alpha_2 * c_1 + 2. * beta_2)\n + beta_2 * (alpha_1 * c_1 + beta_2);\n if det_j == 0.0 {\n break;\n }\n let inv = det_j.recip();\n let c_2 = beta_2 - beta_1;\n let c_3 = beta_1 * alpha_2 - alpha_1 * beta_2;\n let dz_0 = c_1 * f_0 + c_2 * f_1 + c_3 * f_2 - (beta_1 * c_2 + alpha_1 * c_3) * f_3;\n let dz_1 = (alpha_1 * c_1 + c_2) * f_0\n - beta_1 * c_1 * f_1\n - beta_1 * c_2 * f_2\n - beta_1 * c_3 * f_3;\n let dz_2 = -c_1 * f_0 - c_2 * f_1 - c_3 * f_2 + (alpha_2 * c_3 + beta_2 * c_2) * f_3;\n let dz_3 = -(alpha_2 * c_1 + c_2) * f_0\n + beta_2 * c_1 * f_1\n + beta_2 * c_2 * f_2\n + beta_2 * c_3 * f_3;\n let a1 = alpha_1 - inv * dz_0;\n let b1 = beta_1 - inv * dz_1;\n let a2 = alpha_2 - inv * dz_2;\n let b2 = beta_2 - inv * dz_3;\n let new_eps_t = calc_eps_t(a1, b1, a2, b2);\n // We break if the new eps is equal, paper keeps going\n if new_eps_t < eps_t {\n alpha_1 = a1;\n beta_1 = b1;\n alpha_2 = a2;\n beta_2 = b2;\n eps_t = new_eps_t;\n } else {\n //println!(\"new_eps_t got worse: {:e}\", new_eps_t);\n break;\n }\n }\n Some([(alpha_1, beta_1), (alpha_2, beta_2)].into())\n}\n\n/// Dominant root of depressed cubic x^3 + gx + h = 0.\n///\n/// Section 2.2 of Orellana and De Michele.\n// Note: some of the techniques in here might be useful to improve the\n// cubic solver, and vice versa.\nfn depressed_cubic_dominant(g: f64, h: f64) -> f64 {\n let q = (-1. / 3.) * g;\n let r = 0.5 * h;\n let phi_0;\n let k = if q.abs() < 1e102 && r.abs() < 1e154 {\n None\n } else if q.abs() < r.abs() {\n Some(1. - q * (q / r).powi(2))\n } else {\n Some(q.signum() * ((r / q).powi(2) / q - 1.0))\n };\n if k.is_some() && r == 0.0 {\n if g > 0.0 {\n phi_0 = 0.0;\n } else {\n phi_0 = (-g).sqrt();\n }\n } else if k.map(|k| k < 0.0).unwrap_or_else(|| r * r < q.powi(3)) {\n let t = if k.is_some() {\n r / q / q.sqrt()\n } else {\n r / q.powi(3).sqrt()\n };\n phi_0 = -2. * q.sqrt() * (t.abs().acos() * (1. / 3.)).cos().copysign(t);\n } else {\n let a = if let Some(k) = k {\n if q.abs() < r.abs() {\n -r * (1. + k.sqrt())\n } else {\n -r - (q.abs().sqrt() * q * k.sqrt()).copysign(r)\n }\n } else {\n -r - (r * r - q.powi(3)).sqrt().copysign(r)\n }\n .cbrt();\n let b = if a == 0.0 { 0.0 } else { q / a };\n phi_0 = a + b;\n }\n // Refine with Newton-Raphson iteration\n let mut x = phi_0;\n let mut f = (x * x + g) * x + h;\n //println!(\"g = {:e}, h = {:e}, x = {:e}, f = {:e}\", g, h, x, f);\n const EPS_M: f64 = 2.22045e-16;\n if f.abs() < EPS_M * x.powi(3).max(g * x).max(h) {\n return x;\n }\n for _ in 0..8 {\n let delt_f = 3. * x * x + g;\n if delt_f == 0.0 {\n break;\n }\n let new_x = x - f / delt_f;\n let new_f = (new_x * new_x + g) * new_x + h;\n //println!(\"delt_f = {:e}, new_f = {:e}\", delt_f, new_f);\n if new_f == 0.0 {\n return new_x;\n }\n if new_f.abs() >= f.abs() {\n break;\n }\n x = new_x;\n f = new_f;\n }\n x\n}\n\n/// Solve an arbitrary function for a zero-crossing.\n///\n/// This uses the [ITP method], as described in the paper\n/// [An Enhancement of the Bisection Method Average Performance Preserving Minmax Optimality].\n///\n/// The values of `ya` and `yb` are given as arguments rather than\n/// computed from `f`, as the values may already be known, or they may\n/// be less expensive to compute as special cases.\n///\n/// It is assumed that `ya < 0.0` and `yb > 0.0`, otherwise unexpected\n/// results may occur.\n///\n/// The value of `epsilon` must be larger than 2^-63 times `b - a`,\n/// otherwise integer overflow may occur. The `a` and `b` parameters\n/// represent the lower and upper bounds of the bracket searched for a\n/// solution.\n///\n/// The ITP method has tuning parameters. This implementation hardwires\n/// k2 to 2, both because it avoids an expensive floating point\n/// exponentiation, and because this value has been tested to work well\n/// with curve fitting problems.\n///\n/// The `n0` parameter controls the relative impact of the bisection and\n/// secant components. When it is 0, the number of iterations is\n/// guaranteed to be no more than the number required by bisection (thus,\n/// this method is strictly superior to bisection). However, when the\n/// function is smooth, a value of 1 gives the secant method more of a\n/// chance to engage, so the average number of iterations is likely\n/// lower, though there can be one more iteration than bisection in the\n/// worst case.\n///\n/// The `k1` parameter is harder to characterize, and interested users\n/// are referred to the paper, as well as encouraged to do empirical\n/// testing. To match the paper, a value of `0.2 / (b - a)` is\n/// suggested, and this is confirmed to give good results.\n///\n/// When the function is monotonic, the returned result is guaranteed to\n/// be within `epsilon` of the zero crossing. For more detailed analysis,\n/// again see the paper.\n///\n/// [ITP method]: https://en.wikipedia.org/wiki/ITP_Method\n/// [An Enhancement of the Bisection Method Average Performance Preserving Minmax Optimality]: https://dl.acm.org/doi/10.1145/3423597\n#[allow(clippy::too_many_arguments)]\npub fn solve_itp(\n mut f: impl FnMut(f64) -> f64,\n mut a: f64,\n mut b: f64,\n epsilon: f64,\n n0: usize,\n k1: f64,\n mut ya: f64,\n mut yb: f64,\n) -> f64 {\n let n1_2 = (((b - a) / epsilon).log2().ceil() - 1.0).max(0.0) as usize;\n let nmax = n0 + n1_2;\n let mut scaled_epsilon = epsilon * (1u64 << nmax) as f64;\n while b - a > 2.0 * epsilon {\n let x1_2 = 0.5 * (a + b);\n let r = scaled_epsilon - 0.5 * (b - a);\n let xf = (yb * a - ya * b) / (yb - ya);\n let sigma = x1_2 - xf;\n // This has k2 = 2 hardwired for efficiency.\n let delta = k1 * (b - a).powi(2);\n let xt = if delta <= (x1_2 - xf).abs() {\n xf + delta.copysign(sigma)\n } else {\n x1_2\n };\n let xitp = if (xt - x1_2).abs() <= r {\n xt\n } else {\n x1_2 - r.copysign(sigma)\n };\n let yitp = f(xitp);\n if yitp > 0.0 {\n b = xitp;\n yb = yitp;\n } else if yitp < 0.0 {\n a = xitp;\n ya = yitp;\n } else {\n return xitp;\n }\n scaled_epsilon *= 0.5;\n }\n 0.5 * (a + b)\n}\n\n/// A variant ITP solver that allows fallible functions.\n///\n/// Another difference: it returns the bracket that contains the root,\n/// which may be important if the function has a discontinuity.\n#[allow(clippy::too_many_arguments)]\npub(crate) fn solve_itp_fallible(\n mut f: impl FnMut(f64) -> Result,\n mut a: f64,\n mut b: f64,\n epsilon: f64,\n n0: usize,\n k1: f64,\n mut ya: f64,\n mut yb: f64,\n) -> Result<(f64, f64), E> {\n let n1_2 = (((b - a) / epsilon).log2().ceil() - 1.0).max(0.0) as usize;\n let nmax = n0 + n1_2;\n let mut scaled_epsilon = epsilon * (1u64 << nmax) as f64;\n while b - a > 2.0 * epsilon {\n let x1_2 = 0.5 * (a + b);\n let r = scaled_epsilon - 0.5 * (b - a);\n let xf = (yb * a - ya * b) / (yb - ya);\n let sigma = x1_2 - xf;\n // This has k2 = 2 hardwired for efficiency.\n let delta = k1 * (b - a).powi(2);\n let xt = if delta <= (x1_2 - xf).abs() {\n xf + delta.copysign(sigma)\n } else {\n x1_2\n };\n let xitp = if (xt - x1_2).abs() <= r {\n xt\n } else {\n x1_2 - r.copysign(sigma)\n };\n let yitp = f(xitp)?;\n if yitp > 0.0 {\n b = xitp;\n yb = yitp;\n } else if yitp < 0.0 {\n a = xitp;\n ya = yitp;\n } else {\n return Ok((xitp, xitp));\n }\n scaled_epsilon *= 0.5;\n }\n Ok((a, b))\n}\n\n// Tables of Legendre-Gauss quadrature coefficients, adapted from:\n// \n\npub const GAUSS_LEGENDRE_COEFFS_3: &[(f64, f64)] = &[\n (0.8888888888888888, 0.0000000000000000),\n (0.5555555555555556, -0.7745966692414834),\n (0.5555555555555556, 0.7745966692414834),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_4: &[(f64, f64)] = &[\n (0.6521451548625461, -0.3399810435848563),\n (0.6521451548625461, 0.3399810435848563),\n (0.3478548451374538, -0.8611363115940526),\n (0.3478548451374538, 0.8611363115940526),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_5: &[(f64, f64)] = &[\n (0.5688888888888889, 0.0000000000000000),\n (0.4786286704993665, -0.5384693101056831),\n (0.4786286704993665, 0.5384693101056831),\n (0.2369268850561891, -0.9061798459386640),\n (0.2369268850561891, 0.9061798459386640),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_6: &[(f64, f64)] = &[\n (0.3607615730481386, 0.6612093864662645),\n (0.3607615730481386, -0.6612093864662645),\n (0.4679139345726910, -0.2386191860831969),\n (0.4679139345726910, 0.2386191860831969),\n (0.1713244923791704, -0.9324695142031521),\n (0.1713244923791704, 0.9324695142031521),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_7: &[(f64, f64)] = &[\n (0.4179591836734694, 0.0000000000000000),\n (0.3818300505051189, 0.4058451513773972),\n (0.3818300505051189, -0.4058451513773972),\n (0.2797053914892766, -0.7415311855993945),\n (0.2797053914892766, 0.7415311855993945),\n (0.1294849661688697, -0.9491079123427585),\n (0.1294849661688697, 0.9491079123427585),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_8: &[(f64, f64)] = &[\n (0.3626837833783620, -0.1834346424956498),\n (0.3626837833783620, 0.1834346424956498),\n (0.3137066458778873, -0.5255324099163290),\n (0.3137066458778873, 0.5255324099163290),\n (0.2223810344533745, -0.7966664774136267),\n (0.2223810344533745, 0.7966664774136267),\n (0.1012285362903763, -0.9602898564975363),\n (0.1012285362903763, 0.9602898564975363),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_8_HALF: &[(f64, f64)] = &[\n (0.3626837833783620, 0.1834346424956498),\n (0.3137066458778873, 0.5255324099163290),\n (0.2223810344533745, 0.7966664774136267),\n (0.1012285362903763, 0.9602898564975363),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_9: &[(f64, f64)] = &[\n (0.3302393550012598, 0.0000000000000000),\n (0.1806481606948574, -0.8360311073266358),\n (0.1806481606948574, 0.8360311073266358),\n (0.0812743883615744, -0.9681602395076261),\n (0.0812743883615744, 0.9681602395076261),\n (0.3123470770400029, -0.3242534234038089),\n (0.3123470770400029, 0.3242534234038089),\n (0.2606106964029354, -0.6133714327005904),\n (0.2606106964029354, 0.6133714327005904),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_11: &[(f64, f64)] = &[\n (0.2729250867779006, 0.0000000000000000),\n (0.2628045445102467, -0.2695431559523450),\n (0.2628045445102467, 0.2695431559523450),\n (0.2331937645919905, -0.5190961292068118),\n (0.2331937645919905, 0.5190961292068118),\n (0.1862902109277343, -0.7301520055740494),\n (0.1862902109277343, 0.7301520055740494),\n (0.1255803694649046, -0.8870625997680953),\n (0.1255803694649046, 0.8870625997680953),\n (0.0556685671161737, -0.9782286581460570),\n (0.0556685671161737, 0.9782286581460570),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_16: &[(f64, f64)] = &[\n (0.1894506104550685, -0.0950125098376374),\n (0.1894506104550685, 0.0950125098376374),\n (0.1826034150449236, -0.2816035507792589),\n (0.1826034150449236, 0.2816035507792589),\n (0.1691565193950025, -0.4580167776572274),\n (0.1691565193950025, 0.4580167776572274),\n (0.1495959888165767, -0.6178762444026438),\n (0.1495959888165767, 0.6178762444026438),\n (0.1246289712555339, -0.7554044083550030),\n (0.1246289712555339, 0.7554044083550030),\n (0.0951585116824928, -0.8656312023878318),\n (0.0951585116824928, 0.8656312023878318),\n (0.0622535239386479, -0.9445750230732326),\n (0.0622535239386479, 0.9445750230732326),\n (0.0271524594117541, -0.9894009349916499),\n (0.0271524594117541, 0.9894009349916499),\n];\n\n// Just the positive x_i values.\npub const GAUSS_LEGENDRE_COEFFS_16_HALF: &[(f64, f64)] = &[\n (0.1894506104550685, 0.0950125098376374),\n (0.1826034150449236, 0.2816035507792589),\n (0.1691565193950025, 0.4580167776572274),\n (0.1495959888165767, 0.6178762444026438),\n (0.1246289712555339, 0.7554044083550030),\n (0.0951585116824928, 0.8656312023878318),\n (0.0622535239386479, 0.9445750230732326),\n (0.0271524594117541, 0.9894009349916499),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_24: &[(f64, f64)] = &[\n (0.1279381953467522, -0.0640568928626056),\n (0.1279381953467522, 0.0640568928626056),\n (0.1258374563468283, -0.1911188674736163),\n (0.1258374563468283, 0.1911188674736163),\n (0.1216704729278034, -0.3150426796961634),\n (0.1216704729278034, 0.3150426796961634),\n (0.1155056680537256, -0.4337935076260451),\n (0.1155056680537256, 0.4337935076260451),\n (0.1074442701159656, -0.5454214713888396),\n (0.1074442701159656, 0.5454214713888396),\n (0.0976186521041139, -0.6480936519369755),\n (0.0976186521041139, 0.6480936519369755),\n (0.0861901615319533, -0.7401241915785544),\n (0.0861901615319533, 0.7401241915785544),\n (0.0733464814110803, -0.8200019859739029),\n (0.0733464814110803, 0.8200019859739029),\n (0.0592985849154368, -0.8864155270044011),\n (0.0592985849154368, 0.8864155270044011),\n (0.0442774388174198, -0.9382745520027328),\n (0.0442774388174198, 0.9382745520027328),\n (0.0285313886289337, -0.9747285559713095),\n (0.0285313886289337, 0.9747285559713095),\n (0.0123412297999872, -0.9951872199970213),\n (0.0123412297999872, 0.9951872199970213),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_24_HALF: &[(f64, f64)] = &[\n (0.1279381953467522, 0.0640568928626056),\n (0.1258374563468283, 0.1911188674736163),\n (0.1216704729278034, 0.3150426796961634),\n (0.1155056680537256, 0.4337935076260451),\n (0.1074442701159656, 0.5454214713888396),\n (0.0976186521041139, 0.6480936519369755),\n (0.0861901615319533, 0.7401241915785544),\n (0.0733464814110803, 0.8200019859739029),\n (0.0592985849154368, 0.8864155270044011),\n (0.0442774388174198, 0.9382745520027328),\n (0.0285313886289337, 0.9747285559713095),\n (0.0123412297999872, 0.9951872199970213),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_32: &[(f64, f64)] = &[\n (0.0965400885147278, -0.0483076656877383),\n (0.0965400885147278, 0.0483076656877383),\n (0.0956387200792749, -0.1444719615827965),\n (0.0956387200792749, 0.1444719615827965),\n (0.0938443990808046, -0.2392873622521371),\n (0.0938443990808046, 0.2392873622521371),\n (0.0911738786957639, -0.3318686022821277),\n (0.0911738786957639, 0.3318686022821277),\n (0.0876520930044038, -0.4213512761306353),\n (0.0876520930044038, 0.4213512761306353),\n (0.0833119242269467, -0.5068999089322294),\n (0.0833119242269467, 0.5068999089322294),\n (0.0781938957870703, -0.5877157572407623),\n (0.0781938957870703, 0.5877157572407623),\n (0.0723457941088485, -0.6630442669302152),\n (0.0723457941088485, 0.6630442669302152),\n (0.0658222227763618, -0.7321821187402897),\n (0.0658222227763618, 0.7321821187402897),\n (0.0586840934785355, -0.7944837959679424),\n (0.0586840934785355, 0.7944837959679424),\n (0.0509980592623762, -0.8493676137325700),\n (0.0509980592623762, 0.8493676137325700),\n (0.0428358980222267, -0.8963211557660521),\n (0.0428358980222267, 0.8963211557660521),\n (0.0342738629130214, -0.9349060759377397),\n (0.0342738629130214, 0.9349060759377397),\n (0.0253920653092621, -0.9647622555875064),\n (0.0253920653092621, 0.9647622555875064),\n (0.0162743947309057, -0.9856115115452684),\n (0.0162743947309057, 0.9856115115452684),\n (0.0070186100094701, -0.9972638618494816),\n (0.0070186100094701, 0.9972638618494816),\n];\n\npub const GAUSS_LEGENDRE_COEFFS_32_HALF: &[(f64, f64)] = &[\n (0.0965400885147278, 0.0483076656877383),\n (0.0956387200792749, 0.1444719615827965),\n (0.0938443990808046, 0.2392873622521371),\n (0.0911738786957639, 0.3318686022821277),\n (0.0876520930044038, 0.4213512761306353),\n (0.0833119242269467, 0.5068999089322294),\n (0.0781938957870703, 0.5877157572407623),\n (0.0723457941088485, 0.6630442669302152),\n (0.0658222227763618, 0.7321821187402897),\n (0.0586840934785355, 0.7944837959679424),\n (0.0509980592623762, 0.8493676137325700),\n (0.0428358980222267, 0.8963211557660521),\n (0.0342738629130214, 0.9349060759377397),\n (0.0253920653092621, 0.9647622555875064),\n (0.0162743947309057, 0.9856115115452684),\n (0.0070186100094701, 0.9972638618494816),\n];\n\n#[cfg(test)]\nmod tests {\n use crate::common::*;\n use arrayvec::ArrayVec;\n\n fn verify(mut roots: ArrayVec, expected: &[f64]) {\n assert_eq!(expected.len(), roots.len());\n let epsilon = 1e-12;\n roots.sort_by(|a, b| a.partial_cmp(b).unwrap());\n for i in 0..expected.len() {\n assert!((roots[i] - expected[i]).abs() < epsilon);\n }\n }\n\n #[test]\n fn test_solve_cubic() {\n verify(solve_cubic(-5.0, 0.0, 0.0, 1.0), &[5.0f64.cbrt()]);\n verify(solve_cubic(-5.0, -1.0, 0.0, 1.0), &[1.90416085913492]);\n verify(solve_cubic(0.0, -1.0, 0.0, 1.0), &[-1.0, 0.0, 1.0]);\n verify(solve_cubic(-2.0, -3.0, 0.0, 1.0), &[-1.0, 2.0]);\n verify(solve_cubic(2.0, -3.0, 0.0, 1.0), &[-2.0, 1.0]);\n verify(\n solve_cubic(2.0 - 1e-12, 5.0, 4.0, 1.0),\n &[\n -1.9999999999989995,\n -1.0000010000848456,\n -0.9999989999161546,\n ],\n );\n verify(solve_cubic(2.0 + 1e-12, 5.0, 4.0, 1.0), &[-2.0]);\n }\n\n #[test]\n fn test_solve_quadratic() {\n verify(\n solve_quadratic(-5.0, 0.0, 1.0),\n &[-(5.0f64.sqrt()), 5.0f64.sqrt()],\n );\n verify(solve_quadratic(5.0, 0.0, 1.0), &[]);\n verify(solve_quadratic(5.0, 1.0, 0.0), &[-5.0]);\n verify(solve_quadratic(1.0, 2.0, 1.0), &[-1.0]);\n }\n\n #[test]\n fn test_solve_quartic() {\n // These test cases are taken from Orellana and De Michele paper (Table 1).\n fn test_with_roots(coeffs: [f64; 4], roots: &[f64], rel_err: f64) {\n // Note: in paper, coefficients are in decreasing order.\n let mut actual = solve_quartic(coeffs[3], coeffs[2], coeffs[1], coeffs[0], 1.0);\n actual.sort_by(f64::total_cmp);\n assert_eq!(actual.len(), roots.len());\n for (actual, expected) in actual.iter().zip(roots) {\n assert!(\n (actual - expected).abs() < rel_err * expected.abs(),\n \"actual {:e}, expected {:e}, err {:e}\",\n actual,\n expected,\n actual - expected\n );\n }\n }\n\n fn test_vieta_roots(x1: f64, x2: f64, x3: f64, x4: f64, roots: &[f64], rel_err: f64) {\n let a = -(x1 + x2 + x3 + x4);\n let b = x1 * (x2 + x3) + x2 * (x3 + x4) + x4 * (x1 + x3);\n let c = -x1 * x2 * (x3 + x4) - x3 * x4 * (x1 + x2);\n let d = x1 * x2 * x3 * x4;\n test_with_roots([a, b, c, d], roots, rel_err);\n }\n\n fn test_vieta(x1: f64, x2: f64, x3: f64, x4: f64, rel_err: f64) {\n test_vieta_roots(x1, x2, x3, x4, &[x1, x2, x3, x4], rel_err);\n }\n\n // case 1\n test_vieta(1., 1e3, 1e6, 1e9, 1e-16);\n // case 2\n test_vieta(2., 2.001, 2.002, 2.003, 1e-6);\n // case 3\n test_vieta(1e47, 1e49, 1e50, 1e53, 2e-16);\n // case 4\n test_vieta(-1., 1., 2., 1e14, 1e-16);\n // case 5\n test_vieta(-2e7, -1., 1., 1e7, 1e-16);\n // case 6\n test_with_roots(\n [-9000002.0, -9999981999998.0, 19999982e6, -2e13],\n &[-1e6, 1e7],\n 1e-16,\n );\n // case 7\n test_with_roots(\n [2000011.0, 1010022000028.0, 11110056e6, 2828e10],\n &[-7., -4.],\n 1e-16,\n );\n // case 8\n test_with_roots(\n [-100002011.0, 201101022001.0, -102200111000011.0, 11000011e8],\n &[11., 1e8],\n 1e-16,\n );\n // cases 9-13 have no real roots\n // case 14\n test_vieta_roots(1000., 1000., 1000., 1000., &[1000., 1000.], 1e-16);\n // case 15\n test_vieta_roots(1e-15, 1000., 1000., 1000., &[1e-15, 1000., 1000.], 1e-15);\n // case 16 no real roots\n // case 17\n test_vieta(10000., 10001., 10010., 10100., 1e-6);\n // case 19\n test_vieta_roots(1., 1e30, 1e30, 1e44, &[1., 1e30, 1e44], 1e-16);\n // case 20\n // FAILS, error too big\n test_vieta(1., 1e7, 1e7, 1e14, 1e-7);\n // case 21 doesn't pick up double root\n // case 22\n test_vieta(1., 10., 1e152, 1e154, 3e-16);\n // case 23\n test_with_roots(\n [1., 1., 3. / 8., 1e-3],\n &[-0.497314148060048, -0.00268585193995149],\n 2e-15,\n );\n // case 24\n const S: f64 = 1e30;\n test_with_roots(\n [-(1. + 1. / S), 1. / S - S * S, S * S + S, -S],\n &[-S, 1e-30, 1., S],\n 2e-16,\n );\n }\n\n #[test]\n fn test_solve_itp() {\n let f = |x: f64| x.powi(3) - x - 2.0;\n let x = solve_itp(f, 1., 2., 1e-12, 0, 0.2, f(1.), f(2.));\n assert!(f(x).abs() < 6e-12);\n }\n\n #[test]\n fn test_inv_arclen() {\n use crate::{ParamCurve, ParamCurveArclen};\n let c = crate::CubicBez::new(\n (0.0, 0.0),\n (100.0 / 3.0, 0.0),\n (200.0 / 3.0, 100.0 / 3.0),\n (100.0, 100.0),\n );\n let target = 100.0;\n let _ = solve_itp(\n |t| c.subsegment(0.0..t).arclen(1e-9) - target,\n 0.,\n 1.,\n 1e-6,\n 1,\n 0.2,\n -target,\n c.arclen(1e-9) - target,\n );\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/cubicbez.rs", "sha256": "b0131f5f61f94dbc85e374c7aefc6b09961621a74aa9a08d914e8de0acf92b8f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Cubic Bézier segments.\n\nuse alloc::vec;\nuse alloc::vec::Vec;\nuse core::ops::{Mul, Range};\n\nuse crate::MAX_EXTREMA;\nuse crate::{Line, QuadSpline, Vec2};\nuse arrayvec::ArrayVec;\n\nuse crate::common::{\n solve_quadratic, GAUSS_LEGENDRE_COEFFS_16_HALF, GAUSS_LEGENDRE_COEFFS_24_HALF,\n GAUSS_LEGENDRE_COEFFS_8, GAUSS_LEGENDRE_COEFFS_8_HALF,\n};\nuse crate::{\n Affine, Nearest, ParamCurve, ParamCurveArclen, ParamCurveArea, ParamCurveCurvature,\n ParamCurveDeriv, ParamCurveExtrema, ParamCurveNearest, PathEl, Point, QuadBez, Rect, Shape,\n};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\nconst MAX_SPLINE_SPLIT: usize = 100;\n\n/// A single cubic Bézier segment.\n#[derive(Clone, Copy, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\n#[allow(missing_docs)]\npub struct CubicBez {\n pub p0: Point,\n pub p1: Point,\n pub p2: Point,\n pub p3: Point,\n}\n\n/// An iterator which produces quadratic Bézier segments.\nstruct ToQuads {\n c: CubicBez,\n i: usize,\n n: usize,\n}\n\n/// Classification result for cusp detection.\n#[derive(Debug)]\npub enum CuspType {\n /// Cusp is a loop.\n Loop,\n /// Cusp has two closely spaced inflection points.\n DoubleInflection,\n}\n\nimpl CubicBez {\n /// Create a new cubic Bézier segment.\n #[inline]\n pub fn new>(p0: P, p1: P, p2: P, p3: P) -> CubicBez {\n CubicBez {\n p0: p0.into(),\n p1: p1.into(),\n p2: p2.into(),\n p3: p3.into(),\n }\n }\n\n /// Convert to quadratic Béziers.\n ///\n /// The iterator returns the start and end parameter in the cubic of each quadratic\n /// segment, along with the quadratic.\n ///\n /// Note that the resulting quadratic Béziers are not in general G1 continuous;\n /// they are optimized for minimizing distance error.\n ///\n /// This iterator will always produce at least one `QuadBez`.\n #[inline]\n pub fn to_quads(&self, accuracy: f64) -> impl Iterator {\n // The maximum error, as a vector from the cubic to the best approximating\n // quadratic, is proportional to the third derivative, which is constant\n // across the segment. Thus, the error scales down as the third power of\n // the number of subdivisions. Our strategy then is to subdivide `t` evenly.\n //\n // This is an overestimate of the error because only the component\n // perpendicular to the first derivative is important. But the simplicity is\n // appealing.\n\n // This magic number is the square of 36 / sqrt(3).\n // See: http://caffeineowl.com/graphics/2d/vectorial/cubic2quad01.html\n let max_hypot2 = 432.0 * accuracy * accuracy;\n let p1x2 = 3.0 * self.p1.to_vec2() - self.p0.to_vec2();\n let p2x2 = 3.0 * self.p2.to_vec2() - self.p3.to_vec2();\n let err = (p2x2 - p1x2).hypot2();\n let n = ((err / max_hypot2).powf(1. / 6.0).ceil() as usize).max(1);\n\n ToQuads { c: *self, n, i: 0 }\n }\n\n /// Return a [`QuadSpline`] approximating this cubic Bézier.\n ///\n /// Returns `None` if no suitable approximation is found within the given\n /// tolerance.\n pub fn approx_spline(&self, accuracy: f64) -> Option {\n (1..=MAX_SPLINE_SPLIT).find_map(|n| self.approx_spline_n(n, accuracy))\n }\n\n // Approximate a cubic curve with a quadratic spline of `n` curves\n fn approx_spline_n(&self, n: usize, accuracy: f64) -> Option {\n if n == 1 {\n return self\n .try_approx_quadratic(accuracy)\n .map(|quad| QuadSpline::new(vec![quad.p0, quad.p1, quad.p2]));\n }\n let mut cubics = self.split_into_n(n);\n\n // The above function guarantees that the iterator returns n items,\n // which is why we're unwrapping things with wild abandon.\n let mut next_cubic = cubics.next().unwrap();\n let mut next_q1: Point = next_cubic.approx_quad_control(0.0);\n let mut q2 = self.p0;\n let mut d1 = Vec2::ZERO;\n let mut spline = vec![self.p0, next_q1];\n for i in 1..=n {\n let current_cubic: CubicBez = next_cubic;\n let q0 = q2;\n let q1 = next_q1;\n q2 = if i < n {\n next_cubic = cubics.next().unwrap();\n next_q1 = next_cubic.approx_quad_control(i as f64 / (n - 1) as f64);\n\n spline.push(next_q1);\n q1.midpoint(next_q1)\n } else {\n current_cubic.p3\n };\n let d0 = d1;\n d1 = q2.to_vec2() - current_cubic.p3.to_vec2();\n\n if d1.hypot() > accuracy\n || !CubicBez::new(\n d0.to_point(),\n q0.lerp(q1, 2.0 / 3.0) - current_cubic.p1.to_vec2(),\n q2.lerp(q1, 2.0 / 3.0) - current_cubic.p2.to_vec2(),\n d1.to_point(),\n )\n .fit_inside(accuracy)\n {\n return None;\n }\n }\n spline.push(self.p3);\n Some(QuadSpline::new(spline))\n }\n\n fn approx_quad_control(&self, t: f64) -> Point {\n let p1 = self.p0 + (self.p1 - self.p0) * 1.5;\n let p2 = self.p3 + (self.p2 - self.p3) * 1.5;\n p1.lerp(p2, t)\n }\n\n /// Approximate a cubic with a single quadratic\n ///\n /// Returns a quadratic approximating the given cubic that maintains\n /// endpoint tangents if that is within tolerance, or `None` otherwise.\n fn try_approx_quadratic(&self, accuracy: f64) -> Option {\n if let Some(q1) = Line::new(self.p0, self.p1).crossing_point(Line::new(self.p2, self.p3)) {\n let c1 = self.p0.lerp(q1, 2.0 / 3.0);\n let c2 = self.p3.lerp(q1, 2.0 / 3.0);\n if !CubicBez::new(\n Point::ZERO,\n c1 - self.p1.to_vec2(),\n c2 - self.p2.to_vec2(),\n Point::ZERO,\n )\n .fit_inside(accuracy)\n {\n return None;\n }\n return Some(QuadBez::new(self.p0, q1, self.p3));\n }\n None\n }\n\n fn split_into_n(&self, n: usize) -> impl Iterator {\n // for certain small values of `n` we precompute all our values.\n // if we have six or fewer items we precompute them.\n let mut storage = ArrayVec::<_, 6>::new();\n\n match n {\n 1 => storage.push(*self),\n 2 => {\n let (l, r) = self.subdivide();\n storage.try_extend_from_slice(&[r, l]).unwrap();\n }\n 3 => {\n let (left, mid, right) = self.subdivide_3();\n storage.try_extend_from_slice(&[right, mid, left]).unwrap();\n }\n 4 => {\n let (l, r) = self.subdivide();\n let (ll, lr) = l.subdivide();\n let (rl, rr) = r.subdivide();\n storage.try_extend_from_slice(&[rr, rl, lr, ll]).unwrap();\n }\n 6 => {\n let (l, r) = self.subdivide();\n let (l1, l2, l3) = l.subdivide_3();\n let (r1, r2, r3) = r.subdivide_3();\n storage\n .try_extend_from_slice(&[r3, r2, r1, l3, l2, l1])\n .unwrap();\n }\n _ => (),\n }\n\n // a limitation of returning 'impl Trait' is that the implementation\n // can only return a single concrete type; that is you cannot return\n // Vec::into_iter() from one branch, and then HashSet::into_iter from\n // another branch.\n //\n // This means we have to get a bit fancy, and have a single concrete\n // type that represents both of our possible cases.\n\n let mut storage = if storage.is_empty() {\n None\n } else {\n Some(storage)\n };\n\n // used in the fallback case\n let mut i = 0;\n let (a, b, c, d) = self.parameters();\n let dt = 1.0 / n as f64;\n let delta_2 = dt * dt;\n let delta_3 = dt * delta_2;\n\n core::iter::from_fn(move || {\n // if storage exists, we use it exclusively\n if let Some(storage) = storage.as_mut() {\n return storage.pop();\n }\n\n // if storage does not exist, we are exclusively working down here.\n if i >= n {\n return None;\n }\n\n let t1 = i as f64 * dt;\n let t1_2 = t1 * t1;\n let a1 = a * delta_3;\n let b1 = (3.0 * a * t1 + b) * delta_2;\n let c1 = (2.0 * b * t1 + c + 3.0 * a * t1_2) * dt;\n let d1 = a * t1 * t1_2 + b * t1_2 + c * t1 + d;\n let result = CubicBez::from_parameters(a1, b1, c1, d1);\n i += 1;\n Some(result)\n })\n }\n\n fn parameters(&self) -> (Vec2, Vec2, Vec2, Vec2) {\n let c = (self.p1 - self.p0) * 3.0;\n let b = (self.p2 - self.p1) * 3.0 - c;\n let d = self.p0.to_vec2();\n let a = self.p3.to_vec2() - d - c - b;\n (a, b, c, d)\n }\n\n /// Rust port of cu2qu [calc_cubic_points](https://github.com/fonttools/fonttools/blob/3b9a73ff8379ab49d3ce35aaaaf04b3a7d9d1655/Lib/fontTools/cu2qu/cu2qu.py#L63-L68)\n fn from_parameters(a: Vec2, b: Vec2, c: Vec2, d: Vec2) -> Self {\n let p0 = d.to_point();\n let p1 = c.div_exact(3.0).to_point() + d;\n let p2 = (b + c).div_exact(3.0).to_point() + p1.to_vec2();\n let p3 = (a + d + c + b).to_point();\n CubicBez::new(p0, p1, p2, p3)\n }\n\n fn subdivide_3(&self) -> (CubicBez, CubicBez, CubicBez) {\n let (p0, p1, p2, p3) = (\n self.p0.to_vec2(),\n self.p1.to_vec2(),\n self.p2.to_vec2(),\n self.p3.to_vec2(),\n );\n // The original Python cu2qu code here does not use division operator to divide by 27 but\n // instead uses multiplication by the reciprocal 1 / 27. We want to match it exactly\n // to avoid any floating point differences, hence in this particular case we do not use div_exact.\n // I could directly use the Vec2 Div trait (also implemented as multiplication by reciprocal)\n // but I prefer to be explicit here.\n // Source: https://github.com/fonttools/fonttools/blob/85c80be/Lib/fontTools/cu2qu/cu2qu.py#L215-L218\n // See also: https://github.com/linebender/kurbo/issues/272\n let one_27th = 27.0_f64.recip();\n let mid1 = ((8.0 * p0 + 12.0 * p1 + 6.0 * p2 + p3) * one_27th).to_point();\n let deriv1 = (p3 + 3.0 * p2 - 4.0 * p0) * one_27th;\n let mid2 = ((p0 + 6.0 * p1 + 12.0 * p2 + 8.0 * p3) * one_27th).to_point();\n let deriv2 = (4.0 * p3 - 3.0 * p1 - p0) * one_27th;\n\n let left = CubicBez::new(\n self.p0,\n (2.0 * p0 + p1).div_exact(3.0).to_point(),\n mid1 - deriv1,\n mid1,\n );\n let mid = CubicBez::new(mid1, mid1 + deriv1, mid2 - deriv2, mid2);\n let right = CubicBez::new(\n mid2,\n mid2 + deriv2,\n (p2 + 2.0 * p3).div_exact(3.0).to_point(),\n self.p3,\n );\n (left, mid, right)\n }\n\n /// Does this curve fit inside the given distance from the origin?\n ///\n /// Rust port of cu2qu [cubic_farthest_fit_inside](https://github.com/fonttools/fonttools/blob/3b9a73ff8379ab49d3ce35aaaaf04b3a7d9d1655/Lib/fontTools/cu2qu/cu2qu.py#L281)\n fn fit_inside(&self, distance: f64) -> bool {\n if self.p2.to_vec2().hypot() <= distance && self.p1.to_vec2().hypot() <= distance {\n return true;\n }\n let mid =\n (self.p0.to_vec2() + 3.0 * (self.p1.to_vec2() + self.p2.to_vec2()) + self.p3.to_vec2())\n * 0.125;\n if mid.hypot() > distance {\n return false;\n }\n // Split in two. Note that cu2qu here uses a 3/8 subdivision. I don't know why.\n let (left, right) = self.subdivide();\n left.fit_inside(distance) && right.fit_inside(distance)\n }\n\n /// Is this cubic Bezier curve finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n self.p0.is_finite() && self.p1.is_finite() && self.p2.is_finite() && self.p3.is_finite()\n }\n\n /// Is this cubic Bezier curve NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n self.p0.is_nan() || self.p1.is_nan() || self.p2.is_nan() || self.p3.is_nan()\n }\n\n /// Determine the inflection points.\n ///\n /// Return value is t parameter for the inflection points of the curve segment.\n /// There are a maximum of two for a cubic Bézier.\n ///\n /// See \n /// for the theory.\n pub fn inflections(&self) -> ArrayVec {\n let a = self.p1 - self.p0;\n let b = (self.p2 - self.p1) - a;\n let c = (self.p3 - self.p0) - 3. * (self.p2 - self.p1);\n solve_quadratic(a.cross(b), a.cross(c), b.cross(c))\n .iter()\n .copied()\n .filter(|t| *t >= 0.0 && *t <= 1.0)\n .collect()\n }\n\n /// Preprocess a cubic Bézier to ease numerical robustness.\n ///\n /// If the cubic Bézier segment has zero or near-zero derivatives, perturb\n /// the control points to make it easier to process (especially offset and\n /// stroke), avoiding numerical robustness problems.\n pub(crate) fn regularize(&self, dimension: f64) -> CubicBez {\n let mut c = *self;\n // First step: if control point is too near the endpoint, nudge it away\n // along the tangent.\n let dim2 = dimension * dimension;\n if c.p0.distance_squared(c.p1) < dim2 {\n let d02 = c.p0.distance_squared(c.p2);\n if d02 >= dim2 {\n // TODO: moderate if this would move closer to p3\n c.p1 = c.p0.lerp(c.p2, (dim2 / d02).sqrt());\n } else {\n c.p1 = c.p0.lerp(c.p3, 1.0 / 3.0);\n c.p2 = c.p3.lerp(c.p0, 1.0 / 3.0);\n return c;\n }\n }\n if c.p3.distance_squared(c.p2) < dim2 {\n let d13 = c.p1.distance_squared(c.p2);\n if d13 >= dim2 {\n // TODO: moderate if this would move closer to p0\n c.p2 = c.p3.lerp(c.p1, (dim2 / d13).sqrt());\n } else {\n c.p1 = c.p0.lerp(c.p3, 1.0 / 3.0);\n c.p2 = c.p3.lerp(c.p0, 1.0 / 3.0);\n return c;\n }\n }\n if let Some(cusp_type) = self.detect_cusp(dimension) {\n let d01 = c.p1 - c.p0;\n let d01h = d01.hypot();\n let d23 = c.p3 - c.p2;\n let d23h = d23.hypot();\n match cusp_type {\n CuspType::Loop => {\n c.p1 += (dimension / d01h) * d01;\n c.p2 -= (dimension / d23h) * d23;\n }\n CuspType::DoubleInflection => {\n // Avoid making control distance smaller than dimension\n if d01h > 2.0 * dimension {\n c.p1 -= (dimension / d01h) * d01;\n }\n if d23h > 2.0 * dimension {\n c.p2 += (dimension / d23h) * d23;\n }\n }\n }\n }\n c\n }\n\n /// Detect whether there is a cusp.\n ///\n /// Return a cusp classification if there is a cusp with curvature greater than\n /// the reciprocal of the given dimension.\n fn detect_cusp(&self, dimension: f64) -> Option {\n let d01 = self.p1 - self.p0;\n let d02 = self.p2 - self.p0;\n let d03 = self.p3 - self.p0;\n let d12 = self.p2 - self.p1;\n let d23 = self.p3 - self.p2;\n let det_012 = d01.cross(d02);\n let det_123 = d12.cross(d23);\n let det_013 = d01.cross(d03);\n let det_023 = d02.cross(d03);\n if det_012 * det_123 > 0.0 && det_012 * det_013 < 0.0 && det_012 * det_023 < 0.0 {\n let q = self.deriv();\n // accuracy isn't used for quadratic nearest\n let nearest = q.nearest(Point::ORIGIN, 1e-9);\n // detect whether curvature at minimum derivative exceeds 1/dimension,\n // without division.\n let d = q.eval(nearest.t);\n let d2 = q.deriv().eval(nearest.t);\n let cross = d.to_vec2().cross(d2.to_vec2());\n if nearest.distance_sq.powi(3) <= (cross * dimension).powi(2) {\n let a = 3. * det_012 + det_023 - 2. * det_013;\n let b = -3. * det_012 + det_013;\n let c = det_012;\n let d = b * b - 4. * a * c;\n if d > 0.0 {\n return Some(CuspType::DoubleInflection);\n } else {\n return Some(CuspType::Loop);\n }\n }\n }\n None\n }\n}\n\n/// An iterator for cubic beziers.\npub struct CubicBezIter {\n cubic: CubicBez,\n ix: usize,\n}\n\nimpl Shape for CubicBez {\n type PathElementsIter<'iter> = CubicBezIter;\n\n #[inline]\n fn path_elements(&self, _tolerance: f64) -> CubicBezIter {\n CubicBezIter {\n cubic: *self,\n ix: 0,\n }\n }\n\n fn area(&self) -> f64 {\n 0.0\n }\n\n #[inline]\n fn perimeter(&self, accuracy: f64) -> f64 {\n self.arclen(accuracy)\n }\n\n fn winding(&self, _pt: Point) -> i32 {\n 0\n }\n\n #[inline]\n fn bounding_box(&self) -> Rect {\n ParamCurveExtrema::bounding_box(self)\n }\n}\n\nimpl Iterator for CubicBezIter {\n type Item = PathEl;\n\n fn next(&mut self) -> Option {\n self.ix += 1;\n match self.ix {\n 1 => Some(PathEl::MoveTo(self.cubic.p0)),\n 2 => Some(PathEl::CurveTo(self.cubic.p1, self.cubic.p2, self.cubic.p3)),\n _ => None,\n }\n }\n}\n\nimpl ParamCurve for CubicBez {\n #[inline]\n fn eval(&self, t: f64) -> Point {\n let mt = 1.0 - t;\n let v = self.p0.to_vec2() * (mt * mt * mt)\n + (self.p1.to_vec2() * (mt * mt * 3.0)\n + (self.p2.to_vec2() * (mt * 3.0) + self.p3.to_vec2() * t) * t)\n * t;\n v.to_point()\n }\n\n fn subsegment(&self, range: Range) -> CubicBez {\n let (t0, t1) = (range.start, range.end);\n let p0 = self.eval(t0);\n let p3 = self.eval(t1);\n let d = self.deriv();\n let scale = (t1 - t0) * (1.0 / 3.0);\n let p1 = p0 + scale * d.eval(t0).to_vec2();\n let p2 = p3 - scale * d.eval(t1).to_vec2();\n CubicBez { p0, p1, p2, p3 }\n }\n\n /// Subdivide into halves, using de Casteljau.\n #[inline]\n fn subdivide(&self) -> (CubicBez, CubicBez) {\n let pm = self.eval(0.5);\n (\n CubicBez::new(\n self.p0,\n self.p0.midpoint(self.p1),\n ((self.p0.to_vec2() + self.p1.to_vec2() * 2.0 + self.p2.to_vec2()) * 0.25)\n .to_point(),\n pm,\n ),\n CubicBez::new(\n pm,\n ((self.p1.to_vec2() + self.p2.to_vec2() * 2.0 + self.p3.to_vec2()) * 0.25)\n .to_point(),\n self.p2.midpoint(self.p3),\n self.p3,\n ),\n )\n }\n\n #[inline]\n fn start(&self) -> Point {\n self.p0\n }\n\n #[inline]\n fn end(&self) -> Point {\n self.p3\n }\n}\n\nimpl ParamCurveDeriv for CubicBez {\n type DerivResult = QuadBez;\n\n #[inline]\n fn deriv(&self) -> QuadBez {\n QuadBez::new(\n (3.0 * (self.p1 - self.p0)).to_point(),\n (3.0 * (self.p2 - self.p1)).to_point(),\n (3.0 * (self.p3 - self.p2)).to_point(),\n )\n }\n}\n\nfn arclen_quadrature_core(coeffs: &[(f64, f64)], dm: Vec2, dm1: Vec2, dm2: Vec2) -> f64 {\n coeffs\n .iter()\n .map(|&(wi, xi)| {\n let d = dm + dm2 * (xi * xi);\n let dpx = (d + dm1 * xi).hypot();\n let dmx = (d - dm1 * xi).hypot();\n (2.25f64.sqrt() * wi) * (dpx + dmx)\n })\n .sum::()\n}\n\nfn arclen_rec(c: &CubicBez, accuracy: f64, depth: usize) -> f64 {\n let d03 = c.p3 - c.p0;\n let d01 = c.p1 - c.p0;\n let d12 = c.p2 - c.p1;\n let d23 = c.p3 - c.p2;\n let lp_lc = d01.hypot() + d12.hypot() + d23.hypot() - d03.hypot();\n let dd1 = d12 - d01;\n let dd2 = d23 - d12;\n // It might be faster to do direct multiplies, the data dependencies would be shorter.\n // The following values don't have the factor of 3 for first deriv\n let dm = 0.25 * (d01 + d23) + 0.5 * d12; // first derivative at midpoint\n let dm1 = 0.5 * (dd2 + dd1); // second derivative at midpoint\n let dm2 = 0.25 * (dd2 - dd1); // 0.5 * (third derivative at midpoint)\n\n let est = GAUSS_LEGENDRE_COEFFS_8\n .iter()\n .map(|&(wi, xi)| {\n wi * {\n let d_norm2 = (dm + dm1 * xi + dm2 * (xi * xi)).hypot2();\n let dd_norm2 = (dm1 + dm2 * (2.0 * xi)).hypot2();\n dd_norm2 / d_norm2\n }\n })\n .sum::();\n let est_gauss8_error = (est.powi(3) * 2.5e-6).min(3e-2) * lp_lc;\n if est_gauss8_error < accuracy {\n return arclen_quadrature_core(GAUSS_LEGENDRE_COEFFS_8_HALF, dm, dm1, dm2);\n }\n let est_gauss16_error = (est.powi(6) * 1.5e-11).min(9e-3) * lp_lc;\n if est_gauss16_error < accuracy {\n return arclen_quadrature_core(GAUSS_LEGENDRE_COEFFS_16_HALF, dm, dm1, dm2);\n }\n let est_gauss24_error = (est.powi(9) * 3.5e-16).min(3.5e-3) * lp_lc;\n if est_gauss24_error < accuracy || depth >= 20 {\n return arclen_quadrature_core(GAUSS_LEGENDRE_COEFFS_24_HALF, dm, dm1, dm2);\n }\n let (c0, c1) = c.subdivide();\n arclen_rec(&c0, accuracy * 0.5, depth + 1) + arclen_rec(&c1, accuracy * 0.5, depth + 1)\n}\n\nimpl ParamCurveArclen for CubicBez {\n /// Arclength of a cubic Bézier segment.\n ///\n /// This is an adaptive subdivision approach using Legendre-Gauss quadrature\n /// in the base case, and an error estimate to decide when to subdivide.\n fn arclen(&self, accuracy: f64) -> f64 {\n arclen_rec(self, accuracy, 0)\n }\n}\n\nimpl ParamCurveArea for CubicBez {\n #[inline]\n fn signed_area(&self) -> f64 {\n (self.p0.x * (6.0 * self.p1.y + 3.0 * self.p2.y + self.p3.y)\n + 3.0\n * (self.p1.x * (-2.0 * self.p0.y + self.p2.y + self.p3.y)\n - self.p2.x * (self.p0.y + self.p1.y - 2.0 * self.p3.y))\n - self.p3.x * (self.p0.y + 3.0 * self.p1.y + 6.0 * self.p2.y))\n * (1.0 / 20.0)\n }\n}\n\nimpl ParamCurveNearest for CubicBez {\n /// Find the nearest point, using subdivision.\n fn nearest(&self, p: Point, accuracy: f64) -> Nearest {\n let mut best_r = None;\n let mut best_t = 0.0;\n for (t0, t1, q) in self.to_quads(accuracy) {\n let nearest = q.nearest(p, accuracy);\n if best_r\n .map(|best_r| nearest.distance_sq < best_r)\n .unwrap_or(true)\n {\n best_t = t0 + nearest.t * (t1 - t0);\n best_r = Some(nearest.distance_sq);\n }\n }\n Nearest {\n t: best_t,\n distance_sq: best_r.unwrap(),\n }\n }\n}\n\nimpl ParamCurveCurvature for CubicBez {}\n\nimpl ParamCurveExtrema for CubicBez {\n fn extrema(&self) -> ArrayVec {\n fn one_coord(result: &mut ArrayVec, d0: f64, d1: f64, d2: f64) {\n let a = d0 - 2.0 * d1 + d2;\n let b = 2.0 * (d1 - d0);\n let c = d0;\n let roots = solve_quadratic(c, b, a);\n for &t in &roots {\n if t > 0.0 && t < 1.0 {\n result.push(t);\n }\n }\n }\n let mut result = ArrayVec::new();\n let d0 = self.p1 - self.p0;\n let d1 = self.p2 - self.p1;\n let d2 = self.p3 - self.p2;\n one_coord(&mut result, d0.x, d1.x, d2.x);\n one_coord(&mut result, d0.y, d1.y, d2.y);\n result.sort_by(|a, b| a.partial_cmp(b).unwrap());\n result\n }\n}\n\nimpl Mul for Affine {\n type Output = CubicBez;\n\n #[inline]\n fn mul(self, c: CubicBez) -> CubicBez {\n CubicBez {\n p0: self * c.p0,\n p1: self * c.p1,\n p2: self * c.p2,\n p3: self * c.p3,\n }\n }\n}\n\nimpl Iterator for ToQuads {\n type Item = (f64, f64, QuadBez);\n\n fn next(&mut self) -> Option<(f64, f64, QuadBez)> {\n if self.i == self.n {\n return None;\n }\n let t0 = self.i as f64 / self.n as f64;\n let t1 = (self.i + 1) as f64 / self.n as f64;\n let seg = self.c.subsegment(t0..t1);\n let p1x2 = 3.0 * seg.p1.to_vec2() - seg.p0.to_vec2();\n let p2x2 = 3.0 * seg.p2.to_vec2() - seg.p3.to_vec2();\n let result = QuadBez::new(seg.p0, ((p1x2 + p2x2) / 4.0).to_point(), seg.p3);\n self.i += 1;\n Some((t0, t1, result))\n }\n\n fn size_hint(&self) -> (usize, Option) {\n let remaining = self.n - self.i;\n (remaining, Some(remaining))\n }\n}\n\n/// Convert multiple cubic Bézier curves to quadratic splines.\n///\n/// Ensures that the resulting splines have the same number of control points.\n///\n/// Rust port of cu2qu [cubic_approx_quadratic](https://github.com/fonttools/fonttools/blob/3b9a73ff8379ab49d3ce35aaaaf04b3a7d9d1655/Lib/fontTools/cu2qu/cu2qu.py#L322)\npub fn cubics_to_quadratic_splines(curves: &[CubicBez], accuracy: f64) -> Option> {\n let mut result = Vec::new();\n let mut split_order = 0;\n\n while split_order <= MAX_SPLINE_SPLIT {\n split_order += 1;\n result.clear();\n\n for curve in curves {\n match curve.approx_spline_n(split_order, accuracy) {\n Some(spline) => result.push(spline),\n None => break,\n }\n }\n\n if result.len() == curves.len() {\n return Some(result);\n }\n }\n None\n}\n#[cfg(test)]\nmod tests {\n use crate::{\n cubics_to_quadratic_splines, Affine, CubicBez, Nearest, ParamCurve, ParamCurveArclen,\n ParamCurveArea, ParamCurveDeriv, ParamCurveExtrema, ParamCurveNearest, Point, QuadBez,\n QuadSpline,\n };\n\n #[test]\n fn cubicbez_deriv() {\n // y = x^2\n let c = CubicBez::new(\n (0.0, 0.0),\n (1.0 / 3.0, 0.0),\n (2.0 / 3.0, 1.0 / 3.0),\n (1.0, 1.0),\n );\n let deriv = c.deriv();\n\n let n = 10;\n for i in 0..=n {\n let t = (i as f64) * (n as f64).recip();\n let delta = 1e-6;\n let p = c.eval(t);\n let p1 = c.eval(t + delta);\n let d_approx = (p1 - p) * delta.recip();\n let d = deriv.eval(t).to_vec2();\n assert!((d - d_approx).hypot() < delta * 2.0);\n }\n }\n\n #[test]\n fn cubicbez_arclen() {\n // y = x^2\n let c = CubicBez::new(\n (0.0, 0.0),\n (1.0 / 3.0, 0.0),\n (2.0 / 3.0, 1.0 / 3.0),\n (1.0, 1.0),\n );\n let true_arclen = 0.5 * 5.0f64.sqrt() + 0.25 * (2.0 + 5.0f64.sqrt()).ln();\n for i in 0..12 {\n let accuracy = 0.1f64.powi(i);\n let error = c.arclen(accuracy) - true_arclen;\n assert!(error.abs() < accuracy);\n }\n }\n\n #[test]\n fn cubicbez_inv_arclen() {\n // y = x^2 / 100\n let c = CubicBez::new(\n (0.0, 0.0),\n (100.0 / 3.0, 0.0),\n (200.0 / 3.0, 100.0 / 3.0),\n (100.0, 100.0),\n );\n let true_arclen = 100.0 * (0.5 * 5.0f64.sqrt() + 0.25 * (2.0 + 5.0f64.sqrt()).ln());\n for i in 0..12 {\n let accuracy = 0.1f64.powi(i);\n let n = 10;\n for j in 0..=n {\n let arc = (j as f64) * ((n as f64).recip() * true_arclen);\n let t = c.inv_arclen(arc, accuracy * 0.5);\n let actual_arc = c.subsegment(0.0..t).arclen(accuracy * 0.5);\n assert!(\n (arc - actual_arc).abs() < accuracy,\n \"at accuracy {accuracy:e}, wanted {actual_arc} got {arc}\"\n );\n }\n }\n // corner case: user passes accuracy larger than total arc length\n let accuracy = true_arclen * 1.1;\n let arc = true_arclen * 0.5;\n let t = c.inv_arclen(arc, accuracy);\n let actual_arc = c.subsegment(0.0..t).arclen(accuracy);\n assert!(\n (arc - actual_arc).abs() < 2.0 * accuracy,\n \"at accuracy {accuracy:e}, want {actual_arc} got {arc}\"\n );\n }\n\n #[test]\n fn cubicbez_inv_arclen_accuracy() {\n let c = CubicBez::new((0.2, 0.73), (0.35, 1.08), (0.85, 1.08), (1.0, 0.73));\n let true_t = c.inv_arclen(0.5, 1e-12);\n for i in 1..12 {\n let accuracy = (0.1f64).powi(i);\n let approx_t = c.inv_arclen(0.5, accuracy);\n assert!((approx_t - true_t).abs() <= accuracy);\n }\n }\n\n #[test]\n #[allow(clippy::float_cmp)]\n fn cubicbez_signed_area_linear() {\n // y = 1 - x\n let c = CubicBez::new(\n (1.0, 0.0),\n (2.0 / 3.0, 1.0 / 3.0),\n (1.0 / 3.0, 2.0 / 3.0),\n (0.0, 1.0),\n );\n let epsilon = 1e-12;\n assert_eq!((Affine::rotate(0.5) * c).signed_area(), 0.5);\n assert!(((Affine::rotate(0.5) * c).signed_area() - 0.5).abs() < epsilon);\n assert!(((Affine::translate((0.0, 1.0)) * c).signed_area() - 1.0).abs() < epsilon);\n assert!(((Affine::translate((1.0, 0.0)) * c).signed_area() - 1.0).abs() < epsilon);\n }\n\n #[test]\n fn cubicbez_signed_area() {\n // y = 1 - x^3\n let c = CubicBez::new((1.0, 0.0), (2.0 / 3.0, 1.0), (1.0 / 3.0, 1.0), (0.0, 1.0));\n let epsilon = 1e-12;\n assert!((c.signed_area() - 0.75).abs() < epsilon);\n assert!(((Affine::rotate(0.5) * c).signed_area() - 0.75).abs() < epsilon);\n assert!(((Affine::translate((0.0, 1.0)) * c).signed_area() - 1.25).abs() < epsilon);\n assert!(((Affine::translate((1.0, 0.0)) * c).signed_area() - 1.25).abs() < epsilon);\n }\n\n #[test]\n fn cubicbez_nearest() {\n fn verify(result: Nearest, expected: f64) {\n assert!(\n (result.t - expected).abs() < 1e-6,\n \"got {result:?} expected {expected}\"\n );\n }\n // y = x^3\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 0.0), (1.0, 1.0));\n verify(c.nearest((0.1, 0.001).into(), 1e-6), 0.1);\n verify(c.nearest((0.2, 0.008).into(), 1e-6), 0.2);\n verify(c.nearest((0.3, 0.027).into(), 1e-6), 0.3);\n verify(c.nearest((0.4, 0.064).into(), 1e-6), 0.4);\n verify(c.nearest((0.5, 0.125).into(), 1e-6), 0.5);\n verify(c.nearest((0.6, 0.216).into(), 1e-6), 0.6);\n verify(c.nearest((0.7, 0.343).into(), 1e-6), 0.7);\n verify(c.nearest((0.8, 0.512).into(), 1e-6), 0.8);\n verify(c.nearest((0.9, 0.729).into(), 1e-6), 0.9);\n verify(c.nearest((1.0, 1.0).into(), 1e-6), 1.0);\n verify(c.nearest((1.1, 1.1).into(), 1e-6), 1.0);\n verify(c.nearest((-0.1, 0.0).into(), 1e-6), 0.0);\n let a = Affine::rotate(0.5);\n verify((a * c).nearest(a * Point::new(0.1, 0.001), 1e-6), 0.1);\n }\n\n // ensure to_quads returns something given colinear points\n #[test]\n fn degenerate_to_quads() {\n let c = CubicBez::new((0., 9.), (6., 6.), (12., 3.0), (18., 0.0));\n let quads = c.to_quads(1e-6).collect::>();\n assert_eq!(quads.len(), 1, \"{:?}\", &quads);\n }\n\n #[test]\n fn cubicbez_extrema() {\n // y = x^2\n let q = CubicBez::new((0.0, 0.0), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0));\n let extrema = q.extrema();\n assert_eq!(extrema.len(), 1);\n assert!((extrema[0] - 0.5).abs() < 1e-6);\n\n let q = CubicBez::new((0.4, 0.5), (0.0, 1.0), (1.0, 0.0), (0.5, 0.4));\n let extrema = q.extrema();\n assert_eq!(extrema.len(), 4);\n }\n\n #[test]\n fn cubicbez_toquads() {\n // y = x^3\n let c = CubicBez::new((0.0, 0.0), (1.0 / 3.0, 0.0), (2.0 / 3.0, 0.0), (1.0, 1.0));\n for i in 0..10 {\n let accuracy = 0.1f64.powi(i);\n let mut worst: f64 = 0.0;\n for (_count, (t0, t1, q)) in c.to_quads(accuracy).enumerate() {\n let epsilon = 1e-12;\n assert!((q.start() - c.eval(t0)).hypot() < epsilon);\n assert!((q.end() - c.eval(t1)).hypot() < epsilon);\n let n = 4;\n for j in 0..=n {\n let t = (j as f64) * (n as f64).recip();\n let p = q.eval(t);\n let err = (p.y - p.x.powi(3)).abs();\n worst = worst.max(err);\n assert!(err < accuracy, \"got {err} wanted {accuracy}\");\n }\n }\n }\n }\n\n #[test]\n fn cubicbez_approx_spline() {\n let c1 = CubicBez::new(\n (550.0, 258.0),\n (1044.0, 482.0),\n (2029.0, 1841.0),\n (1934.0, 1554.0),\n );\n\n let quad = c1.try_approx_quadratic(344.0);\n let expected = QuadBez::new(\n Point::new(550.0, 258.0),\n Point::new(1673.665720592873, 767.5164401068898),\n Point::new(1934.0, 1554.0),\n );\n assert!(quad.is_some());\n assert_eq!(quad.unwrap(), expected);\n\n let quad = c1.try_approx_quadratic(343.0);\n assert!(quad.is_none());\n\n let spline = c1.approx_spline_n(2, 343.0);\n assert!(spline.is_some());\n let spline = spline.unwrap();\n let expected = [\n Point::new(550.0, 258.0),\n Point::new(920.5, 426.0),\n Point::new(2005.25, 1769.25),\n Point::new(1934.0, 1554.0),\n ];\n assert_eq!(spline.points().len(), expected.len());\n for (got, &wanted) in spline.points().iter().zip(expected.iter()) {\n assert!(got.distance(wanted) < 5.0)\n }\n\n let spline = c1.approx_spline(5.0);\n let expected = [\n Point::new(550.0, 258.0),\n Point::new(673.5, 314.0),\n Point::new(984.8777777777776, 584.2666666666667),\n Point::new(1312.6305555555557, 927.825),\n Point::new(1613.1194444444443, 1267.425),\n Point::new(1842.7055555555555, 1525.8166666666666),\n Point::new(1957.75, 1625.75),\n Point::new(1934.0, 1554.0),\n ];\n assert!(spline.is_some());\n let spline = spline.unwrap();\n assert_eq!(spline.points().len(), expected.len());\n for (got, &wanted) in spline.points().iter().zip(expected.iter()) {\n assert!(got.distance(wanted) < 5.0)\n }\n }\n\n #[test]\n fn cubicbez_cubics_to_quadratic_splines() {\n let curves = vec![\n CubicBez::new(\n (550.0, 258.0),\n (1044.0, 482.0),\n (2029.0, 1841.0),\n (1934.0, 1554.0),\n ),\n CubicBez::new(\n (859.0, 384.0),\n (1998.0, 116.0),\n (1596.0, 1772.0),\n (8.0, 1824.0),\n ),\n CubicBez::new(\n (1090.0, 937.0),\n (418.0, 1300.0),\n (125.0, 91.0),\n (104.0, 37.0),\n ),\n ];\n let converted = cubics_to_quadratic_splines(&curves, 5.0);\n assert!(converted.is_some());\n let converted = converted.unwrap();\n assert_eq!(converted[0].points().len(), 8);\n assert_eq!(converted[1].points().len(), 8);\n assert_eq!(converted[2].points().len(), 8);\n assert!(converted[0].points()[1].distance(Point::new(673.5, 314.0)) < 0.0001);\n assert!(\n converted[0].points()[2].distance(Point::new(88639.0 / 90.0, 52584.0 / 90.0)) < 0.0001\n );\n }\n\n #[test]\n fn cubicbez_approx_spline_div_exact() {\n // Ensure rounding behavior for division matches fonttools\n // cu2qu.\n // See \n let cubic = CubicBez::new(\n Point::new(408.0, 321.0),\n Point::new(408.0, 452.0),\n Point::new(342.0, 560.0),\n Point::new(260.0, 560.0),\n );\n let spline = cubic.approx_spline(1.0).unwrap();\n assert_eq!(\n spline.points(),\n &[\n Point::new(408.0, 321.0),\n // Previous behavior produced 386.49999999999994 for the\n // y coordinate leading to inconsistent rounding.\n Point::new(408.0, 386.5),\n Point::new(368.16666666666663, 495.0833333333333),\n Point::new(301.0, 560.0),\n Point::new(260.0, 560.0)\n ]\n )\n }\n\n #[test]\n fn cubicbez_inflections() {\n let c = CubicBez::new((0., 0.), (0.8, 1.), (0.2, 1.), (1., 0.));\n let inflections = c.inflections();\n assert_eq!(inflections.len(), 2);\n assert!((inflections[0] - 0.311018).abs() < 1e-6);\n assert!((inflections[1] - 0.688982).abs() < 1e-6);\n let c = CubicBez::new((0., 0.), (1., 1.), (2., -1.), (3., 0.));\n let inflections = c.inflections();\n assert_eq!(inflections.len(), 1);\n assert!((inflections[0] - 0.5).abs() < 1e-6);\n let c = CubicBez::new((0., 0.), (1., 1.), (2., 1.), (3., 0.));\n let inflections = c.inflections();\n assert_eq!(inflections.len(), 0);\n }\n\n #[test]\n fn cubic_to_quadratic_matches_python() {\n // from https://github.com/googlefonts/fontmake-rs/issues/217\n let cubic = CubicBez {\n p0: (796.0, 319.0).into(),\n p1: (727.0, 314.0).into(),\n p2: (242.0, 303.0).into(),\n p3: (106.0, 303.0).into(),\n };\n\n // FontTools can approximate this curve successfully in 7 splits, we can too\n assert!(cubic.approx_spline_n(7, 1.0).is_some());\n\n // FontTools can solve this with accuracy 0.001, we can too\n assert!(cubics_to_quadratic_splines(&[cubic], 0.001).is_some());\n }\n\n #[test]\n fn cubics_to_quadratic_splines_matches_python() {\n // https://github.com/linebender/kurbo/pull/273\n let light = CubicBez::new((378., 608.), (378., 524.), (355., 455.), (266., 455.));\n let regular = CubicBez::new((367., 607.), (367., 511.), (338., 472.), (243., 472.));\n let bold = CubicBez::new(\n (372.425, 593.05),\n (372.425, 524.95),\n (355.05, 485.95),\n (274., 485.95),\n );\n let qsplines = cubics_to_quadratic_splines(&[light, regular, bold], 1.0).unwrap();\n assert_eq!(\n qsplines,\n [\n QuadSpline::new(vec![\n (378.0, 608.0).into(),\n (378.0, 566.0).into(),\n (359.0833333333333, 496.5).into(),\n (310.5, 455.0).into(),\n (266.0, 455.0).into(),\n ]),\n QuadSpline::new(vec![\n (367.0, 607.0).into(),\n (367.0, 559.0).into(),\n // Previous behavior produced 496.5 for the y coordinate\n (344.5833333333333, 499.49999999999994).into(),\n (290.5, 472.0).into(),\n (243.0, 472.0).into(),\n ]),\n QuadSpline::new(vec![\n (372.425, 593.05).into(),\n (372.425, 559.0).into(),\n (356.98333333333335, 511.125).into(),\n (314.525, 485.95).into(),\n (274.0, 485.95).into(),\n ]),\n ]\n )\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/ellipse.rs", "sha256": "d018c98ec9893a1e88b44142b77282a8bea680087a1df5347ff8647c2d8ba6cd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Implementation of ellipse shape.\n\nuse core::f64::consts::PI;\nuse core::{\n iter,\n ops::{Add, Mul, Sub},\n};\n\nuse crate::{Affine, Arc, ArcAppendIter, Circle, PathEl, Point, Rect, Shape, Size, Vec2};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// An ellipse.\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct Ellipse {\n /// All ellipses can be represented as an affine map of the unit circle,\n /// centred at (0, 0). Therefore we can store the ellipse as an affine map,\n /// with the implication that it be applied to the unit circle to recover the\n /// actual shape.\n inner: Affine,\n}\n\nimpl Ellipse {\n /// Create A new ellipse with a given center, radii, and rotation.\n ///\n /// The returned ellipse will be the result of taking a circle, stretching\n /// it by the `radii` along the x and y axes, then rotating it from the\n /// x axis by `rotation` radians, before finally translating the center\n /// to `center`.\n ///\n /// Rotation is clockwise in a y-down coordinate system. For more on\n /// rotation, see [`Affine::rotate`].\n #[inline]\n pub fn new(center: impl Into, radii: impl Into, x_rotation: f64) -> Ellipse {\n let Point { x: cx, y: cy } = center.into();\n let Vec2 { x: rx, y: ry } = radii.into();\n Ellipse::private_new(Vec2 { x: cx, y: cy }, rx, ry, x_rotation)\n }\n\n /// Returns the largest ellipse that can be bounded by this [`Rect`].\n ///\n /// This uses the absolute width and height of the rectangle.\n ///\n /// This ellipse is always axis-aligned; to apply rotation you can call\n /// [`with_rotation`] with the result.\n ///\n /// [`with_rotation`]: Ellipse::with_rotation\n #[inline]\n pub fn from_rect(rect: Rect) -> Self {\n let center = rect.center().to_vec2();\n let Size { width, height } = rect.size() / 2.0;\n Ellipse::private_new(center, width, height, 0.0)\n }\n\n /// Create an ellipse from an affine transformation of the unit circle.\n #[inline]\n pub fn from_affine(affine: Affine) -> Self {\n Ellipse { inner: affine }\n }\n\n /// Create a new `Ellipse` centered on the provided point.\n #[inline]\n #[must_use]\n pub fn with_center(self, new_center: Point) -> Ellipse {\n let Point { x: cx, y: cy } = new_center;\n Ellipse {\n inner: self.inner.with_translation(Vec2 { x: cx, y: cy }),\n }\n }\n\n /// Create a new `Ellipse` with the provided radii.\n #[must_use]\n pub fn with_radii(self, new_radii: Vec2) -> Ellipse {\n let rotation = self.inner.svd().1;\n let translation = self.inner.translation();\n Ellipse::private_new(translation, new_radii.x, new_radii.y, rotation)\n }\n\n /// Create a new `Ellipse`, with the rotation replaced by `rotation`\n /// radians.\n ///\n /// The rotation is clockwise, for a y-down coordinate system. For more\n /// on rotation, See [`Affine::rotate`].\n #[must_use]\n pub fn with_rotation(self, rotation: f64) -> Ellipse {\n let scale = self.inner.svd().0;\n let translation = self.inner.translation();\n Ellipse::private_new(translation, scale.x, scale.y, rotation)\n }\n\n #[deprecated(since = \"0.7.0\", note = \"use with_rotation instead\")]\n #[must_use]\n #[doc(hidden)]\n pub fn with_x_rotation(self, rotation_radians: f64) -> Ellipse {\n self.with_rotation(rotation_radians)\n }\n\n /// This gives us an internal method without any type conversions.\n #[inline]\n fn private_new(center: Vec2, scale_x: f64, scale_y: f64, x_rotation: f64) -> Ellipse {\n // Since the circle is symmetric about the x and y axes, using absolute values for the\n // radii results in the same ellipse. For simplicity we make this change here.\n Ellipse {\n inner: Affine::translate(center)\n * Affine::rotate(x_rotation)\n * Affine::scale_non_uniform(scale_x.abs(), scale_y.abs()),\n }\n }\n\n // Getters and setters.\n\n /// Returns the center of this ellipse.\n #[inline]\n pub fn center(&self) -> Point {\n let Vec2 { x: cx, y: cy } = self.inner.translation();\n Point { x: cx, y: cy }\n }\n\n /// Returns the two radii of this ellipse.\n ///\n /// The first number is the horizontal radius and the second is the vertical\n /// radius, before rotation.\n pub fn radii(&self) -> Vec2 {\n self.inner.svd().0\n }\n\n /// The ellipse's rotation, in radians.\n ///\n /// This allows all possible ellipses to be drawn by always starting with\n /// an ellipse with the two radii on the x and y axes.\n pub fn rotation(&self) -> f64 {\n self.inner.svd().1\n }\n\n /// Returns the radii and the rotation of this ellipse.\n ///\n /// Equivalent to `(self.radii(), self.rotation())` but more efficient.\n pub fn radii_and_rotation(&self) -> (Vec2, f64) {\n self.inner.svd()\n }\n\n /// Is this ellipse finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n self.inner.is_finite()\n }\n\n /// Is this ellipse NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n self.inner.is_nan()\n }\n\n #[doc(hidden)]\n #[deprecated(since = \"0.7.0\", note = \"use rotation() instead\")]\n pub fn x_rotation(&self) -> f64 {\n self.rotation()\n }\n}\n\nimpl Add for Ellipse {\n type Output = Ellipse;\n\n /// In this context adding a `Vec2` applies the corresponding translation to the ellipse.\n #[inline]\n #[allow(clippy::suspicious_arithmetic_impl)]\n fn add(self, v: Vec2) -> Ellipse {\n Ellipse {\n inner: Affine::translate(v) * self.inner,\n }\n }\n}\n\nimpl Sub for Ellipse {\n type Output = Ellipse;\n\n /// In this context subtracting a `Vec2` applies the corresponding translation to the ellipse.\n #[inline]\n fn sub(self, v: Vec2) -> Ellipse {\n Ellipse {\n inner: Affine::translate(-v) * self.inner,\n }\n }\n}\n\nimpl Mul for Affine {\n type Output = Ellipse;\n fn mul(self, other: Ellipse) -> Self::Output {\n Ellipse {\n inner: self * other.inner,\n }\n }\n}\n\nimpl From for Ellipse {\n fn from(circle: Circle) -> Self {\n Ellipse::new(circle.center, Vec2::splat(circle.radius), 0.0)\n }\n}\n\nimpl Shape for Ellipse {\n type PathElementsIter<'iter> = iter::Chain, ArcAppendIter>;\n\n fn path_elements(&self, tolerance: f64) -> Self::PathElementsIter<'_> {\n let (radii, x_rotation) = self.inner.svd();\n Arc {\n center: self.center(),\n radii,\n start_angle: 0.0,\n sweep_angle: 2.0 * PI,\n x_rotation,\n }\n .path_elements(tolerance)\n }\n\n #[inline]\n fn area(&self) -> f64 {\n let Vec2 { x, y } = self.radii();\n PI * x * y\n }\n\n #[inline]\n fn perimeter(&self, accuracy: f64) -> f64 {\n // TODO rather than delegate to the bezier path, it is possible to use various series\n // expansions to compute the perimeter to any accuracy. I believe Ramanujan authored the\n // quickest to converge. See\n // https://www.mathematica-journal.com/2009/11/23/on-the-perimeter-of-an-ellipse/\n // and https://en.wikipedia.org/wiki/Ellipse#Circumference\n //\n self.path_segments(0.1).perimeter(accuracy)\n }\n\n fn winding(&self, pt: Point) -> i32 {\n // Strategy here is to apply the inverse map to the point and see if it is in the unit\n // circle.\n let inv = self.inner.inverse();\n if (inv * pt).to_vec2().hypot2() < 1.0 {\n 1\n } else {\n 0\n }\n }\n\n // Compute a tight bounding box of the ellipse.\n //\n // See https://www.iquilezles.org/www/articles/ellipses/ellipses.htm. We can get the two\n // radius vectors by applying the affine map to the two impulses (1, 0) and (0, 1) which gives\n // (a, b) and (c, d) if the affine map is\n //\n // a | c | e\n // -----------\n // b | d | f\n //\n // We can then use the method in the link with the translation to get the bounding box.\n #[inline]\n fn bounding_box(&self) -> Rect {\n let aff = self.inner.as_coeffs();\n let a2 = aff[0] * aff[0];\n let b2 = aff[1] * aff[1];\n let c2 = aff[2] * aff[2];\n let d2 = aff[3] * aff[3];\n let cx = aff[4];\n let cy = aff[5];\n let range_x = (a2 + c2).sqrt();\n let range_y = (b2 + d2).sqrt();\n Rect {\n x0: cx - range_x,\n y0: cy - range_y,\n x1: cx + range_x,\n y1: cy + range_y,\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{Ellipse, Point, Shape};\n use std::f64::consts::PI;\n\n fn assert_approx_eq(x: f64, y: f64) {\n // Note: we might want to be more rigorous in testing the accuracy\n // of the conversion into Béziers. But this seems good enough.\n assert!((x - y).abs() < 1e-7, \"{x} != {y}\");\n }\n\n #[test]\n fn area_sign() {\n let center = Point::new(5.0, 5.0);\n let e = Ellipse::new(center, (5.0, 5.0), 1.0);\n assert_approx_eq(e.area(), 25.0 * PI);\n let e = Ellipse::new(center, (5.0, 10.0), 1.0);\n assert_approx_eq(e.area(), 50.0 * PI);\n\n assert_eq!(e.winding(center), 1);\n\n let p = e.to_path(1e-9);\n assert_approx_eq(e.area(), p.area());\n assert_eq!(e.winding(center), p.winding(center));\n\n let e_neg_radius = Ellipse::new(center, (-5.0, 10.0), 1.0);\n assert_approx_eq(e_neg_radius.area(), 50.0 * PI);\n\n assert_eq!(e_neg_radius.winding(center), 1);\n\n let p_neg_radius = e_neg_radius.to_path(1e-9);\n assert_approx_eq(e_neg_radius.area(), p_neg_radius.area());\n assert_eq!(e_neg_radius.winding(center), p_neg_radius.winding(center));\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/fit.rs", "sha256": "d03b17ca74b5eaa3cecfe30bf92b2f2bc29b11bb05e7551652045b3f3ed21a97", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! An implementation of cubic Bézier curve fitting based on a quartic\n//! solver making signed area and moment match the source curve.\n\nuse core::ops::Range;\n\nuse alloc::vec::Vec;\n\nuse arrayvec::ArrayVec;\n\nuse crate::{\n common::{\n factor_quartic_inner, solve_cubic, solve_itp_fallible, solve_quadratic,\n GAUSS_LEGENDRE_COEFFS_16,\n },\n Affine, BezPath, CubicBez, Line, ParamCurve, ParamCurveArclen, ParamCurveNearest, Point, Vec2,\n};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// The source curve for curve fitting.\n///\n/// This trait is a general representation of curves to be used as input to a curve\n/// fitting process. It can represent source curves with cusps and corners, though\n/// if the corners are known in advance, it may be better to run curve fitting on\n/// subcurves bounded by the corners.\n///\n/// The trait primarily works by sampling the source curve and computing the position\n/// and derivative at each sample. Those derivatives are then used for multiple\n/// sub-tasks, including ensuring G1 continuity at subdivision points, computing the\n/// area and moment of the curve for curve fitting, and casting rays for evaluation\n/// of a distance metric to test accuracy.\n///\n/// A major motivation is computation of offset curves, which often have cusps, but\n/// the presence and location of those cusps is not generally known. It is also\n/// intended for conversion between curve types (for example, piecewise Euler spiral\n/// or NURBS), and distortion effects such as perspective transform.\n///\n/// Note general similarities to [`ParamCurve`] but also important differences.\n/// Instead of separate [`eval`] and evaluation of derivative, have a single\n/// [`sample_pt_deriv`] method which can be more efficient and also handles cusps more\n/// robustly. Also there is no method for subsegment, as that is not needed and\n/// would be annoying to implement.\n///\n/// [`ParamCurve`]: crate::ParamCurve\n/// [`eval`]: crate::ParamCurve::eval\n/// [`sample_pt_deriv`]: ParamCurveFit::sample_pt_deriv\npub trait ParamCurveFit {\n /// Evaluate the curve and its tangent at parameter `t`.\n ///\n /// For a regular curve (one not containing a cusp or corner), the\n /// derivative is a good choice for the tangent vector and the `sign`\n /// parameter can be ignored. Otherwise, the `sign` parameter selects which\n /// side of the discontinuity the tangent will be sampled from.\n ///\n /// Generally `t` is in the range [0..1].\n fn sample_pt_tangent(&self, t: f64, sign: f64) -> CurveFitSample;\n\n /// Evaluate the point and derivative at parameter `t`.\n ///\n /// In curves with cusps, the derivative can go to zero.\n fn sample_pt_deriv(&self, t: f64) -> (Point, Vec2);\n\n /// Compute moment integrals.\n ///\n /// This method computes the integrals of y dx, x y dx, and y^2 dx over the\n /// length of this curve. From these integrals it is fairly straightforward\n /// to derive the moments needed for curve fitting.\n ///\n /// A default implementation is proved which does quadrature integration\n /// with Green's theorem, in terms of samples evaluated with\n /// [`sample_pt_deriv`].\n ///\n /// [`sample_pt_deriv`]: ParamCurveFit::sample_pt_deriv\n fn moment_integrals(&self, range: Range) -> (f64, f64, f64) {\n let t0 = 0.5 * (range.start + range.end);\n let dt = 0.5 * (range.end - range.start);\n let (a, x, y) = GAUSS_LEGENDRE_COEFFS_16\n .iter()\n .map(|(wi, xi)| {\n let t = t0 + xi * dt;\n let (p, d) = self.sample_pt_deriv(t);\n let a = wi * d.x * p.y;\n let x = p.x * a;\n let y = p.y * a;\n (a, x, y)\n })\n .fold((0.0, 0.0, 0.0), |(a0, x0, y0), (a1, x1, y1)| {\n (a0 + a1, x0 + x1, y0 + y1)\n });\n (a * dt, x * dt, y * dt)\n }\n\n /// Find a cusp or corner within the given range.\n ///\n /// If the range contains a corner or cusp, return it. If there is more\n /// than one such discontinuity, any can be reported, as the function will\n /// be called repeatedly after subdivision of the range.\n ///\n /// Do not report cusps at the endpoints of the range, as this may cause\n /// potentially infinite subdivision. In particular, when a cusp is reported\n /// and this method is called on a subdivided range bounded by the reported\n /// cusp, then the subsequent call should not report a cusp there.\n ///\n /// The definition of what exactly constitutes a cusp is somewhat loose.\n /// If a cusp is missed, then the curve fitting algorithm will attempt to\n /// fit the curve with a smooth curve, which is generally not a disaster\n /// will usually result in more subdivision. Conversely, it might be useful\n /// to report near-cusps, specifically points of curvature maxima where the\n /// curvature is large but still mathematically finite.\n fn break_cusp(&self, range: Range) -> Option;\n}\n\n/// A sample point of a curve for fitting.\npub struct CurveFitSample {\n /// A point on the curve at the sample location.\n pub p: Point,\n /// A vector tangent to the curve at the sample location.\n pub tangent: Vec2,\n}\n\nimpl CurveFitSample {\n /// Intersect a ray orthogonal to the tangent with the given cubic.\n ///\n /// Returns a vector of `t` values on the cubic.\n fn intersect(&self, c: CubicBez) -> ArrayVec {\n let p1 = 3.0 * (c.p1 - c.p0);\n let p2 = 3.0 * c.p2.to_vec2() - 6.0 * c.p1.to_vec2() + 3.0 * c.p0.to_vec2();\n let p3 = (c.p3 - c.p0) - 3.0 * (c.p2 - c.p1);\n let c0 = (c.p0 - self.p).dot(self.tangent);\n let c1 = p1.dot(self.tangent);\n let c2 = p2.dot(self.tangent);\n let c3 = p3.dot(self.tangent);\n solve_cubic(c0, c1, c2, c3)\n .into_iter()\n .filter(|t| (0.0..=1.0).contains(t))\n .collect()\n }\n}\n\n/// Generate a Bézier path that fits the source curve.\n///\n/// This function is still experimental and the signature might change; it's possible\n/// it might become a method on the [`ParamCurveFit`] trait.\n///\n/// This function recursively subdivides the curve in half by the parameter when the\n/// accuracy is not met. That gives a reasonably optimized result but not necessarily\n/// the minimum number of segments.\n///\n/// In general, the resulting Bézier path should have a Fréchet distance less than\n/// the provided `accuracy` parameter. However, this is not a rigorous guarantee, as\n/// the error metric is computed approximately.\n///\n/// This function is intended for use when the source curve is piecewise continuous,\n/// with the discontinuities reported by the cusp method. In applications (such as\n/// stroke expansion) where this property may not hold, it is up to the client to\n/// detect and handle such cases. Even so, best effort is made to avoid infinite\n/// subdivision.\n///\n/// When a higher degree of optimization is desired (at considerably more runtime cost),\n/// consider [`fit_to_bezpath_opt`] instead.\npub fn fit_to_bezpath(source: &impl ParamCurveFit, accuracy: f64) -> BezPath {\n let mut path = BezPath::new();\n fit_to_bezpath_rec(source, 0.0..1.0, accuracy, &mut path);\n path\n}\n\n// Discussion question: possibly should take endpoint samples, to avoid\n// duplication of that work.\nfn fit_to_bezpath_rec(\n source: &impl ParamCurveFit,\n range: Range,\n accuracy: f64,\n path: &mut BezPath,\n) {\n let start = range.start;\n let end = range.end;\n let start_p = source.sample_pt_tangent(range.start, 1.0).p;\n let end_p = source.sample_pt_tangent(range.end, -1.0).p;\n if start_p.distance_squared(end_p) <= accuracy * accuracy {\n if let Some((c, _)) = try_fit_line(source, accuracy, range, start_p, end_p) {\n if path.is_empty() {\n path.move_to(c.p0);\n }\n path.curve_to(c.p1, c.p2, c.p3);\n return;\n }\n }\n let t = if let Some(t) = source.break_cusp(start..end) {\n t\n } else if let Some((c, _)) = fit_to_cubic(source, start..end, accuracy) {\n if path.is_empty() {\n path.move_to(c.p0);\n }\n path.curve_to(c.p1, c.p2, c.p3);\n return;\n } else {\n // A smarter approach is possible than midpoint subdivision, but would be\n // a significant increase in complexity.\n 0.5 * (start + end)\n };\n if t == start || t == end {\n // infinite recursion, just draw a line\n let p1 = start_p.lerp(end_p, 1.0 / 3.0);\n let p2 = end_p.lerp(start_p, 1.0 / 3.0);\n if path.is_empty() {\n path.move_to(start_p);\n }\n path.curve_to(p1, p2, end_p);\n return;\n }\n fit_to_bezpath_rec(source, start..t, accuracy, path);\n fit_to_bezpath_rec(source, t..end, accuracy, path);\n}\n\nconst N_SAMPLE: usize = 20;\n\n/// An acceleration structure for estimating curve distance\nstruct CurveDist {\n samples: ArrayVec,\n arcparams: ArrayVec,\n range: Range,\n /// A \"spicy\" curve is one with potentially extreme curvature variation,\n /// so use arc length measurement for better accuracy.\n spicy: bool,\n}\n\nimpl CurveDist {\n fn from_curve(source: &impl ParamCurveFit, range: Range) -> Self {\n let step = (range.end - range.start) * (1.0 / (N_SAMPLE + 1) as f64);\n let mut last_tan = None;\n let mut spicy = false;\n const SPICY_THRESH: f64 = 0.2;\n let mut samples = ArrayVec::new();\n for i in 0..N_SAMPLE + 2 {\n let sample = source.sample_pt_tangent(range.start + i as f64 * step, 1.0);\n if let Some(last_tan) = last_tan {\n let cross = sample.tangent.cross(last_tan);\n let dot = sample.tangent.dot(last_tan);\n if cross.abs() > SPICY_THRESH * dot.abs() {\n spicy = true;\n }\n }\n last_tan = Some(sample.tangent);\n if i > 0 && i < N_SAMPLE + 1 {\n samples.push(sample);\n }\n }\n CurveDist {\n samples,\n arcparams: Default::default(),\n range,\n spicy,\n }\n }\n\n fn compute_arc_params(&mut self, source: &impl ParamCurveFit) {\n const N_SUBSAMPLE: usize = 10;\n let (start, end) = (self.range.start, self.range.end);\n let dt = (end - start) * (1.0 / ((N_SAMPLE + 1) * N_SUBSAMPLE) as f64);\n let mut arclen = 0.0;\n for i in 0..N_SAMPLE + 1 {\n for j in 0..N_SUBSAMPLE {\n let t = start + dt * ((i * N_SUBSAMPLE + j) as f64 + 0.5);\n let (_, deriv) = source.sample_pt_deriv(t);\n arclen += deriv.hypot();\n }\n if i < N_SAMPLE {\n self.arcparams.push(arclen);\n }\n }\n let arclen_inv = arclen.recip();\n for x in &mut self.arcparams {\n *x *= arclen_inv;\n }\n }\n\n /// Evaluate distance based on arc length parametrization\n fn eval_arc(&self, c: CubicBez, acc2: f64) -> Option {\n // TODO: this could perhaps be tuned.\n const EPS: f64 = 1e-9;\n let c_arclen = c.arclen(EPS);\n let mut max_err2 = 0.0;\n for (sample, s) in self.samples.iter().zip(&self.arcparams) {\n let t = c.inv_arclen(c_arclen * s, EPS);\n let err = sample.p.distance_squared(c.eval(t));\n max_err2 = err.max(max_err2);\n if max_err2 > acc2 {\n return None;\n }\n }\n Some(max_err2)\n }\n\n /// Evaluate distance to a cubic approximation.\n ///\n /// If distance exceeds stated accuracy, can return `None`. Note that\n /// `acc2` is the square of the target.\n ///\n /// Returns the square of the error, which is intended to be a good\n /// approximation of the Fréchet distance.\n fn eval_ray(&self, c: CubicBez, acc2: f64) -> Option {\n let mut max_err2 = 0.0;\n for sample in &self.samples {\n let mut best = acc2 + 1.0;\n for t in sample.intersect(c) {\n let err = sample.p.distance_squared(c.eval(t));\n best = best.min(err);\n }\n max_err2 = best.max(max_err2);\n if max_err2 > acc2 {\n return None;\n }\n }\n Some(max_err2)\n }\n\n fn eval_dist(&mut self, source: &impl ParamCurveFit, c: CubicBez, acc2: f64) -> Option {\n // Always compute cheaper distance, hoping for early-out.\n let ray_dist = self.eval_ray(c, acc2)?;\n if !self.spicy {\n return Some(ray_dist);\n }\n if self.arcparams.is_empty() {\n self.compute_arc_params(source);\n }\n self.eval_arc(c, acc2)\n }\n}\n\n/// As described in [Simplifying Bézier paths], strictly optimizing for\n/// Fréchet distance can create bumps. The problem is curves with long\n/// control arms (distance from the control point to the corresponding\n/// endpoint). We mitigate that by applying a penalty as a multiplier to\n/// the measured error (approximate Fréchet distance). This is ReLU-like,\n/// with a value of 1.0 below the elbow, and a given slope above it. The\n/// values here have been determined empirically to give good results.\n///\n/// [Simplifying Bézier paths]:\n/// https://raphlinus.github.io/curves/2023/04/18/bezpath-simplify.html\nconst D_PENALTY_ELBOW: f64 = 0.65;\nconst D_PENALTY_SLOPE: f64 = 2.0;\n\n/// Try fitting a line.\n///\n/// This is especially useful for very short chords, in which the standard\n/// cubic fit is not numerically stable. The tangents are not considered, so\n/// it's useful in cusp and near-cusp situations where the tangents are not\n/// reliable, as well.\n///\n/// Returns the line raised to a cubic and the error, if within tolerance.\nfn try_fit_line(\n source: &impl ParamCurveFit,\n accuracy: f64,\n range: Range,\n start: Point,\n end: Point,\n) -> Option<(CubicBez, f64)> {\n let acc2 = accuracy * accuracy;\n let chord_l = Line::new(start, end);\n const SHORT_N: usize = 7;\n let mut max_err2 = 0.0;\n let dt = (range.end - range.start) / (SHORT_N + 1) as f64;\n for i in 0..SHORT_N {\n let t = range.start + (i + 1) as f64 * dt;\n let p = source.sample_pt_deriv(t).0;\n let err2 = chord_l.nearest(p, accuracy).distance_sq;\n if err2 > acc2 {\n // Not in tolerance; likely subdivision will help.\n return None;\n }\n max_err2 = err2.max(max_err2);\n }\n let p1 = start.lerp(end, 1.0 / 3.0);\n let p2 = end.lerp(start, 1.0 / 3.0);\n let c = CubicBez::new(start, p1, p2, end);\n Some((c, max_err2))\n}\n\n/// Fit a single cubic to a range of the source curve.\n///\n/// Returns the cubic segment and the square of the error.\n/// Discussion question: should this be a method on the trait instead?\npub fn fit_to_cubic(\n source: &impl ParamCurveFit,\n range: Range,\n accuracy: f64,\n) -> Option<(CubicBez, f64)> {\n let start = source.sample_pt_tangent(range.start, 1.0);\n let end = source.sample_pt_tangent(range.end, -1.0);\n let d = end.p - start.p;\n let chord2 = d.hypot2();\n let acc2 = accuracy * accuracy;\n if chord2 <= acc2 {\n // Special case very short chords; try to fit a line.\n return try_fit_line(source, accuracy, range, start.p, end.p);\n }\n let th = d.atan2();\n fn mod_2pi(th: f64) -> f64 {\n let th_scaled = th * core::f64::consts::FRAC_1_PI * 0.5;\n core::f64::consts::PI * 2.0 * (th_scaled - th_scaled.round())\n }\n let th0 = mod_2pi(start.tangent.atan2() - th);\n let th1 = mod_2pi(th - end.tangent.atan2());\n\n let (mut area, mut x, mut y) = source.moment_integrals(range.clone());\n let (x0, y0) = (start.p.x, start.p.y);\n let (dx, dy) = (d.x, d.y);\n // Subtract off area of chord\n area -= dx * (y0 + 0.5 * dy);\n // `area` is signed area of closed curve segment.\n // This quantity is invariant to translation and rotation.\n\n // Subtract off moment of chord\n let dy_3 = dy * (1. / 3.);\n x -= dx * (x0 * y0 + 0.5 * (x0 * dy + y0 * dx) + dy_3 * dx);\n y -= dx * (y0 * y0 + y0 * dy + dy_3 * dy);\n // Translate start point to origin; convert raw integrals to moments.\n x -= x0 * area;\n y = 0.5 * y - y0 * area;\n // Rotate into place (this also scales up by chordlength for efficiency).\n let moment = d.x * x + d.y * y;\n // `moment` is the chordlength times the x moment of the curve translated\n // so its start point is on the origin, and rotated so its end point is on the\n // x axis.\n\n let chord2_inv = chord2.recip();\n let unit_area = area * chord2_inv;\n let mx = moment * chord2_inv.powi(2);\n // `unit_area` is signed area scaled to unit chord; `mx` is scaled x moment\n\n let chord = chord2.sqrt();\n let aff = Affine::translate(start.p.to_vec2()) * Affine::rotate(th) * Affine::scale(chord);\n let mut curve_dist = CurveDist::from_curve(source, range);\n let mut best_c = None;\n let mut best_err2 = None;\n for (cand, d0, d1) in cubic_fit(th0, th1, unit_area, mx) {\n let c = aff * cand;\n if let Some(err2) = curve_dist.eval_dist(source, c, acc2) {\n fn scale_f(d: f64) -> f64 {\n 1.0 + (d - D_PENALTY_ELBOW).max(0.0) * D_PENALTY_SLOPE\n }\n let scale = scale_f(d0).max(scale_f(d1)).powi(2);\n let err2 = err2 * scale;\n if err2 < acc2 && best_err2.map(|best| err2 < best).unwrap_or(true) {\n best_c = Some(c);\n best_err2 = Some(err2);\n }\n }\n }\n match (best_c, best_err2) {\n (Some(c), Some(err2)) => Some((c, err2)),\n _ => None,\n }\n}\n\n/// Returns curves matching area and moment, given unit chord.\nfn cubic_fit(th0: f64, th1: f64, area: f64, mx: f64) -> ArrayVec<(CubicBez, f64, f64), 4> {\n // Note: maybe we want to take unit vectors instead of angle? Shouldn't\n // matter much either way though.\n let (s0, c0) = th0.sin_cos();\n let (s1, c1) = th1.sin_cos();\n let a4 = -9.\n * c0\n * (((2. * s1 * c1 * c0 + s0 * (2. * c1 * c1 - 1.)) * c0 - 2. * s1 * c1) * c0\n - c1 * c1 * s0);\n let a3 = 12.\n * ((((c1 * (30. * area * c1 - s1) - 15. * area) * c0 + 2. * s0\n - c1 * s0 * (c1 + 30. * area * s1))\n * c0\n + c1 * (s1 - 15. * area * c1))\n * c0\n - s0 * c1 * c1);\n let a2 = 12.\n * ((((70. * mx + 15. * area) * s1 * s1 + c1 * (9. * s1 - 70. * c1 * mx - 5. * c1 * area))\n * c0\n - 5. * s0 * s1 * (3. * s1 - 4. * c1 * (7. * mx + area)))\n * c0\n - c1 * (9. * s1 - 70. * c1 * mx - 5. * c1 * area));\n let a1 = 16.\n * (((12. * s0 - 5. * c0 * (42. * mx - 17. * area)) * s1\n - 70. * c1 * (3. * mx - area) * s0\n - 75. * c0 * c1 * area * area)\n * s1\n - 75. * c1 * c1 * area * area * s0);\n let a0 = 80. * s1 * (42. * s1 * mx - 25. * area * (s1 - c1 * area));\n // TODO: \"roots\" is not a good name for this variable, as it also contains\n // the real part of complex conjugate pairs.\n let mut roots = ArrayVec::::new();\n const EPS: f64 = 1e-12;\n if a4.abs() > EPS {\n let a = a3 / a4;\n let b = a2 / a4;\n let c = a1 / a4;\n let d = a0 / a4;\n if let Some(quads) = factor_quartic_inner(a, b, c, d, false) {\n for (qc1, qc0) in quads {\n let qroots = solve_quadratic(qc0, qc1, 1.0);\n if qroots.is_empty() {\n // Real part of pair of complex roots\n roots.push(-0.5 * qc1);\n } else {\n roots.extend(qroots);\n }\n }\n }\n } else if a3.abs() > EPS {\n roots.extend(solve_cubic(a0, a1, a2, a3));\n } else {\n roots.extend(solve_quadratic(a0, a1, a2));\n }\n\n let s01 = s0 * c1 + s1 * c0;\n roots\n .iter()\n .filter_map(|&d0| {\n let (d0, d1) = if d0 > 0.0 {\n let d1 = (d0 * s0 - area * (10. / 3.)) / (0.5 * d0 * s01 - s1);\n if d1 > 0.0 {\n (d0, d1)\n } else {\n (s1 / s01, 0.0)\n }\n } else {\n (0.0, s0 / s01)\n };\n // We could implement a maximum d value here.\n if d0 >= 0.0 && d1 >= 0.0 {\n Some((\n CubicBez::new(\n (0.0, 0.0),\n (d0 * c0, d0 * s0),\n (1.0 - d1 * c1, d1 * s1),\n (1.0, 0.0),\n ),\n d0,\n d1,\n ))\n } else {\n None\n }\n })\n .collect()\n}\n\n/// Generate a highly optimized Bézier path that fits the source curve.\n///\n/// This function is still experimental and the signature might change; it's possible\n/// it might become a method on the [`ParamCurveFit`] trait.\n///\n/// This function is considerably slower than [`fit_to_bezpath`], as it computes\n/// optimal subdivision points. Its result is expected to be very close to the optimum\n/// possible Bézier path for the source curve, in that it has a minimal number of curve\n/// segments, and a minimal error over all paths with that number of segments.\n///\n/// See [`fit_to_bezpath`] for an explanation of the `accuracy` parameter.\npub fn fit_to_bezpath_opt(source: &impl ParamCurveFit, accuracy: f64) -> BezPath {\n let mut cusps = Vec::new();\n let mut path = BezPath::new();\n let mut t0 = 0.0;\n loop {\n let t1 = cusps.last().copied().unwrap_or(1.0);\n match fit_to_bezpath_opt_inner(source, accuracy, t0..t1, &mut path) {\n Some(t) => cusps.push(t),\n None => match cusps.pop() {\n Some(t) => t0 = t,\n None => break,\n },\n }\n }\n path\n}\n\n/// Fit a range without cusps.\n///\n/// On Ok return, range has been added to the path. On Err return, report a cusp (and don't\n/// mutate path).\nfn fit_to_bezpath_opt_inner(\n source: &impl ParamCurveFit,\n accuracy: f64,\n range: Range,\n path: &mut BezPath,\n) -> Option {\n if let Some(t) = source.break_cusp(range.clone()) {\n return Some(t);\n }\n let err;\n if let Some((c, err2)) = fit_to_cubic(source, range.clone(), accuracy) {\n err = err2.sqrt();\n if err < accuracy {\n if range.start == 0.0 {\n path.move_to(c.p0);\n }\n path.curve_to(c.p1, c.p2, c.p3);\n return None;\n }\n } else {\n err = 2.0 * accuracy;\n }\n let (mut t0, t1) = (range.start, range.end);\n let mut n = 0;\n let last_err;\n loop {\n n += 1;\n match fit_opt_segment(source, accuracy, t0..t1) {\n FitResult::ParamVal(t) => t0 = t,\n FitResult::SegmentError(err) => {\n last_err = err;\n break;\n }\n FitResult::CuspFound(t) => return Some(t),\n }\n }\n t0 = range.start;\n const EPS: f64 = 1e-9;\n let f = |x| fit_opt_err_delta(source, x, accuracy, t0..t1, n);\n let k1 = 0.2 / accuracy;\n let ya = -err;\n let yb = accuracy - last_err;\n let (_, x) = match solve_itp_fallible(f, 0.0, accuracy, EPS, 1, k1, ya, yb) {\n Ok(x) => x,\n Err(t) => return Some(t),\n };\n //println!(\"got fit with n={}, err={}\", n, x);\n let path_len = path.elements().len();\n for i in 0..n {\n let t1 = if i < n - 1 {\n match fit_opt_segment(source, x, t0..range.end) {\n FitResult::ParamVal(t1) => t1,\n FitResult::SegmentError(_) => range.end,\n FitResult::CuspFound(t) => {\n path.truncate(path_len);\n return Some(t);\n }\n }\n } else {\n range.end\n };\n let (c, _) = fit_to_cubic(source, t0..t1, accuracy).unwrap();\n if i == 0 && range.start == 0.0 {\n path.move_to(c.p0);\n }\n path.curve_to(c.p1, c.p2, c.p3);\n t0 = t1;\n if t0 == range.end {\n // This is unlikely but could happen when not monotonic.\n break;\n }\n }\n None\n}\n\nfn measure_one_seg(source: &impl ParamCurveFit, range: Range, limit: f64) -> Option {\n fit_to_cubic(source, range, limit).map(|(_, err2)| err2.sqrt())\n}\n\nenum FitResult {\n /// The parameter (`t`) value that meets the desired accuracy.\n ParamVal(f64),\n /// Error of the measured segment.\n SegmentError(f64),\n /// The parameter value where a cusp was found.\n CuspFound(f64),\n}\n\nfn fit_opt_segment(source: &impl ParamCurveFit, accuracy: f64, range: Range) -> FitResult {\n if let Some(t) = source.break_cusp(range.clone()) {\n return FitResult::CuspFound(t);\n }\n let missing_err = accuracy * 2.0;\n let err = measure_one_seg(source, range.clone(), accuracy).unwrap_or(missing_err);\n if err <= accuracy {\n return FitResult::SegmentError(err);\n }\n let (t0, t1) = (range.start, range.end);\n let f = |x| {\n if let Some(t) = source.break_cusp(range.clone()) {\n return Err(t);\n }\n let err = measure_one_seg(source, t0..x, accuracy).unwrap_or(missing_err);\n Ok(err - accuracy)\n };\n const EPS: f64 = 1e-9;\n let k1 = 2.0 / (t1 - t0);\n match solve_itp_fallible(f, t0, t1, EPS, 1, k1, -accuracy, err - accuracy) {\n Ok((t1, _)) => FitResult::ParamVal(t1),\n Err(t) => FitResult::CuspFound(t),\n }\n}\n\n// Ok result is delta error (accuracy - error of last seg).\n// Err result is a cusp.\nfn fit_opt_err_delta(\n source: &impl ParamCurveFit,\n accuracy: f64,\n limit: f64,\n range: Range,\n n: usize,\n) -> Result {\n let (mut t0, t1) = (range.start, range.end);\n for _ in 0..n - 1 {\n t0 = match fit_opt_segment(source, accuracy, t0..t1) {\n FitResult::ParamVal(t0) => t0,\n // In this case, n - 1 will work, which of course means the error is highly\n // non-monotonic. We should probably harvest that solution.\n FitResult::SegmentError(_) => return Ok(accuracy),\n FitResult::CuspFound(t) => return Err(t),\n }\n }\n let err = measure_one_seg(source, t0..t1, limit).unwrap_or(accuracy * 2.0);\n Ok(accuracy - err)\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/insets.rs", "sha256": "120fd9f31eea4391d5ac3b4bd2fcc03ef8ca3c2f3fc6f7ae3f2cde6d04484cc8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A description of the distances between the edges of two rectangles.\n\nuse core::ops::{Add, Neg, Sub};\n\nuse crate::{Rect, Size};\n\n/// Insets from the edges of a rectangle.\n///\n///\n/// The inset value for each edge can be thought of as a delta computed from\n/// the center of the rect to that edge. For instance, with an inset of `2.0` on\n/// the x-axis, a rectangle with the origin `(0.0, 0.0)` with that inset added\n/// will have the new origin at `(-2.0, 0.0)`.\n///\n/// Put alternatively, a positive inset represents increased distance from center,\n/// and a negative inset represents decreased distance from center.\n///\n/// # Examples\n///\n/// Positive insets added to a [`Rect`] produce a larger [`Rect`]:\n/// ```\n/// # use kurbo::{Insets, Rect};\n/// let rect = Rect::from_origin_size((0., 0.,), (10., 10.,));\n/// let insets = Insets::uniform_xy(3., 0.,);\n///\n/// let inset_rect = rect + insets;\n/// assert_eq!(inset_rect.width(), 16.0, \"10.0 + 3.0 × 2\");\n/// assert_eq!(inset_rect.x0, -3.0);\n/// ```\n///\n/// Negative insets added to a [`Rect`] produce a smaller [`Rect`]:\n///\n/// ```\n/// # use kurbo::{Insets, Rect};\n/// let rect = Rect::from_origin_size((0., 0.,), (10., 10.,));\n/// let insets = Insets::uniform_xy(-3., 0.,);\n///\n/// let inset_rect = rect + insets;\n/// assert_eq!(inset_rect.width(), 4.0, \"10.0 - 3.0 × 2\");\n/// assert_eq!(inset_rect.x0, 3.0);\n/// ```\n///\n/// [`Insets`] operate on the absolute rectangle [`Rect::abs`], and so ignore\n/// existing negative widths and heights.\n///\n/// ```\n/// # use kurbo::{Insets, Rect};\n/// let rect = Rect::new(7., 11., 0., 0.,);\n/// let insets = Insets::uniform_xy(0., 1.,);\n///\n/// assert_eq!(rect.width(), -7.0);\n///\n/// let inset_rect = rect + insets;\n/// assert_eq!(inset_rect.width(), 7.0);\n/// assert_eq!(inset_rect.x0, 0.0);\n/// assert_eq!(inset_rect.height(), 13.0);\n/// ```\n///\n/// The width and height of an inset operation can still be negative if the\n/// [`Insets`]' dimensions are greater than the dimensions of the original [`Rect`].\n///\n/// ```\n/// # use kurbo::{Insets, Rect};\n/// let rect = Rect::new(0., 0., 3., 5.);\n/// let insets = Insets::uniform_xy(0., 7.,);\n///\n/// let inset_rect = rect - insets;\n/// assert_eq!(inset_rect.height(), -9., \"5 - 7 × 2\")\n/// ```\n///\n/// `Rect - Rect = Insets`:\n///\n///\n/// ```\n/// # use kurbo::{Insets, Rect};\n/// let rect = Rect::new(0., 0., 5., 11.);\n/// let insets = Insets::uniform_xy(1., 7.,);\n///\n/// let inset_rect = rect + insets;\n/// let insets2 = inset_rect - rect;\n///\n/// assert_eq!(insets2.x0, insets.x0);\n/// assert_eq!(insets2.y1, insets.y1);\n/// assert_eq!(insets2.x_value(), insets.x_value());\n/// assert_eq!(insets2.y_value(), insets.y_value());\n/// ```\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct Insets {\n /// The minimum x coordinate (left edge).\n pub x0: f64,\n /// The minimum y coordinate (top edge in y-down spaces).\n pub y0: f64,\n /// The maximum x coordinate (right edge).\n pub x1: f64,\n /// The maximum y coordinate (bottom edge in y-down spaces).\n pub y1: f64,\n}\n\nimpl Insets {\n /// Zeroed insets.\n pub const ZERO: Insets = Insets::uniform(0.);\n\n /// New uniform insets.\n #[inline]\n pub const fn uniform(d: f64) -> Insets {\n Insets {\n x0: d,\n y0: d,\n x1: d,\n y1: d,\n }\n }\n\n /// New insets with uniform values along each axis.\n #[inline]\n pub const fn uniform_xy(x: f64, y: f64) -> Insets {\n Insets {\n x0: x,\n y0: y,\n x1: x,\n y1: y,\n }\n }\n\n /// New insets. The ordering of the arguments is \"left, top, right, bottom\",\n /// assuming a y-down coordinate space.\n #[inline]\n pub const fn new(x0: f64, y0: f64, x1: f64, y1: f64) -> Insets {\n Insets { x0, y0, x1, y1 }\n }\n\n /// The total delta on the x-axis represented by these insets.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Insets;\n ///\n /// let insets = Insets::uniform_xy(3., 8.);\n /// assert_eq!(insets.x_value(), 6.);\n ///\n /// let insets = Insets::new(5., 0., -12., 0.,);\n /// assert_eq!(insets.x_value(), -7.);\n /// ```\n #[inline]\n pub fn x_value(self) -> f64 {\n self.x0 + self.x1\n }\n\n /// The total delta on the y-axis represented by these insets.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Insets;\n ///\n /// let insets = Insets::uniform_xy(3., 7.);\n /// assert_eq!(insets.y_value(), 14.);\n ///\n /// let insets = Insets::new(5., 10., -12., 4.,);\n /// assert_eq!(insets.y_value(), 14.);\n /// ```\n #[inline]\n pub fn y_value(self) -> f64 {\n self.y0 + self.y1\n }\n\n /// Returns the total delta represented by these insets as a [`Size`].\n ///\n /// This is equivalent to creating a [`Size`] from the values returned by\n /// [`x_value`] and [`y_value`].\n ///\n /// This function may return a size with negative values.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::{Insets, Size};\n ///\n /// let insets = Insets::new(11.1, -43.3, 3.333, -0.0);\n /// assert_eq!(insets.size(), Size::new(insets.x_value(), insets.y_value()));\n /// ```\n ///\n /// [`x_value`]: Insets::x_value\n /// [`y_value`]: Insets::y_value\n pub fn size(self) -> Size {\n Size::new(self.x_value(), self.y_value())\n }\n\n /// Return `true` iff all values are nonnegative.\n pub fn are_nonnegative(self) -> bool {\n let Insets { x0, y0, x1, y1 } = self;\n x0 >= 0.0 && y0 >= 0.0 && x1 >= 0.0 && y1 >= 0.0\n }\n\n /// Return new `Insets` with all negative values replaced with `0.0`.\n ///\n /// This is provided as a convenience for applications where negative insets\n /// are not meaningful.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Insets;\n ///\n /// let insets = Insets::new(-10., 3., -0.2, 4.);\n /// let nonnegative = insets.nonnegative();\n /// assert_eq!(nonnegative.x_value(), 0.0);\n /// assert_eq!(nonnegative.y_value(), 7.0);\n /// ```\n pub fn nonnegative(self) -> Insets {\n let Insets { x0, y0, x1, y1 } = self;\n Insets {\n x0: x0.max(0.0),\n y0: y0.max(0.0),\n x1: x1.max(0.0),\n y1: y1.max(0.0),\n }\n }\n\n /// Are these insets finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n self.x0.is_finite() && self.y0.is_finite() && self.x1.is_finite() && self.y1.is_finite()\n }\n\n /// Are these insets NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n self.x0.is_nan() || self.y0.is_nan() || self.x1.is_nan() || self.y1.is_nan()\n }\n}\n\nimpl Neg for Insets {\n type Output = Insets;\n\n #[inline]\n fn neg(self) -> Insets {\n Insets::new(-self.x0, -self.y0, -self.x1, -self.y1)\n }\n}\n\nimpl Add for Insets {\n type Output = Rect;\n\n #[inline]\n #[allow(clippy::suspicious_arithmetic_impl)]\n fn add(self, other: Rect) -> Rect {\n let other = other.abs();\n Rect::new(\n other.x0 - self.x0,\n other.y0 - self.y0,\n other.x1 + self.x1,\n other.y1 + self.y1,\n )\n }\n}\n\nimpl Add for Rect {\n type Output = Rect;\n\n #[inline]\n fn add(self, other: Insets) -> Rect {\n other + self\n }\n}\n\nimpl Sub for Insets {\n type Output = Rect;\n\n #[inline]\n fn sub(self, other: Rect) -> Rect {\n other + -self\n }\n}\n\nimpl Sub for Rect {\n type Output = Rect;\n\n #[inline]\n fn sub(self, other: Insets) -> Rect {\n other - self\n }\n}\n\nimpl From for Insets {\n fn from(src: f64) -> Insets {\n Insets::uniform(src)\n }\n}\n\nimpl From<(f64, f64)> for Insets {\n fn from(src: (f64, f64)) -> Insets {\n Insets::uniform_xy(src.0, src.1)\n }\n}\n\nimpl From<(f64, f64, f64, f64)> for Insets {\n fn from(src: (f64, f64, f64, f64)) -> Insets {\n Insets::new(src.0, src.1, src.2, src.3)\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/lib.rs", "sha256": "67cff58c27c0852496a841a9f507347b63d5d00313ea2cd60b6df7d3345b0751", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! 2D geometry, with a focus on curves.\n//!\n//! The kurbo library contains data structures and algorithms for curves and\n//! vector paths. It was designed to serve the needs of 2D graphics applications,\n//! but it is intended to be general enough to be useful for other applications.\n//! It can be used as \"vocabulary types\" for representing curves and paths, and\n//! also contains a number of computational geometry methods.\n//!\n//! # Examples\n//!\n//! Basic UI-style geometry:\n//! ```\n//! use kurbo::{Insets, Point, Rect, Size, Vec2};\n//!\n//! let pt = Point::new(10.0, 10.0);\n//! let vector = Vec2::new(5.0, -5.0);\n//! let pt2 = pt + vector;\n//! assert_eq!(pt2, Point::new(15.0, 5.0));\n//!\n//! let rect = Rect::from_points(pt, pt2);\n//! assert_eq!(rect, Rect::from_origin_size((10.0, 5.0), (5.0, 5.0)));\n//!\n//! let insets = Insets::uniform(1.0);\n//! let inset_rect = rect - insets;\n//! assert_eq!(inset_rect.size(), Size::new(3.0, 3.0));\n//! ```\n//!\n//! Finding the closest position on a [`Shape`]'s perimeter to a [`Point`]:\n//!\n//! ```\n//! use kurbo::{Circle, ParamCurve, ParamCurveNearest, Point, Shape};\n//!\n//! const DESIRED_ACCURACY: f64 = 0.1;\n//!\n//! /// Given a shape and a point, returns the closest position on the shape's\n//! /// perimeter, or `None` if the shape is malformed.\n//! fn closest_perimeter_point(shape: impl Shape, pt: Point) -> Option {\n//! let mut best: Option<(Point, f64)> = None;\n//! for segment in shape.path_segments(DESIRED_ACCURACY) {\n//! let nearest = segment.nearest(pt, DESIRED_ACCURACY);\n//! if best.map(|(_, best_d)| nearest.distance_sq < best_d).unwrap_or(true) {\n//! best = Some((segment.eval(nearest.t), nearest.distance_sq))\n//! }\n//! }\n//! best.map(|(point, _)| point)\n//! }\n//!\n//! let circle = Circle::new((5.0, 5.0), 5.0);\n//! let hit_point = Point::new(5.0, -2.0);\n//! let expectation = Point::new(5.0, 0.0);\n//! let hit = closest_perimeter_point(circle, hit_point).unwrap();\n//! assert!(hit.distance(expectation) <= DESIRED_ACCURACY);\n//! ```\n//!\n//! # Features\n//!\n//! This crate either uses the standard library or the [`libm`] crate for\n//! math functionality. The `std` feature is enabled by default, but can be\n//! disabled, as long as the `libm` feature is enabled. This is useful for\n//! `no_std` environments. However, note that the `libm` crate is not as\n//! efficient as the standard library, and that this crate still uses the\n//! `alloc` crate regardless.\n//!\n//! [`Piet`]: https://docs.rs/piet\n//! [`Druid`]: https://docs.rs/druid\n//! [`libm`]: https://docs.rs/libm\n\n#![forbid(unsafe_code)]\n#![deny(missing_docs, clippy::trivially_copy_pass_by_ref)]\n#![warn(clippy::doc_markdown, rustdoc::broken_intra_doc_links)]\n#![allow(\n clippy::unreadable_literal,\n clippy::many_single_char_names,\n clippy::excessive_precision,\n clippy::bool_to_int_with_if\n)]\n#![cfg_attr(all(not(feature = \"std\"), not(test)), no_std)]\n\n#[cfg(not(any(feature = \"std\", feature = \"libm\")))]\ncompile_error!(\"kurbo requires either the `std` or `libm` feature\");\n\nextern crate alloc;\n\nmod affine;\nmod arc;\nmod bezpath;\nmod circle;\npub mod common;\nmod cubicbez;\nmod ellipse;\nmod fit;\nmod insets;\nmod line;\nmod mindist;\npub mod offset;\nmod param_curve;\nmod point;\nmod quadbez;\nmod quadspline;\nmod rect;\nmod rounded_rect;\nmod rounded_rect_radii;\nmod shape;\npub mod simplify;\nmod size;\nmod stroke;\n#[cfg(feature = \"std\")]\nmod svg;\nmod translate_scale;\nmod vec2;\n\npub use crate::affine::*;\npub use crate::arc::*;\npub use crate::bezpath::*;\npub use crate::circle::*;\npub use crate::cubicbez::*;\npub use crate::ellipse::*;\npub use crate::fit::*;\npub use crate::insets::*;\npub use crate::line::*;\npub use crate::param_curve::*;\npub use crate::point::*;\npub use crate::quadbez::*;\npub use crate::quadspline::*;\npub use crate::rect::*;\npub use crate::rounded_rect::*;\npub use crate::rounded_rect_radii::*;\npub use crate::shape::*;\npub use crate::size::*;\npub use crate::stroke::*;\n#[cfg(feature = \"std\")]\npub use crate::svg::*;\npub use crate::translate_scale::*;\npub use crate::vec2::*;\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/line.rs", "sha256": "496292c066a56bfd7f49f3b28c5cd38b94b53a4f9c97b985d783dea5078ed54c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Lines.\n\nuse core::ops::{Add, Mul, Range, Sub};\n\nuse arrayvec::ArrayVec;\n\nuse crate::{\n Affine, Nearest, ParamCurve, ParamCurveArclen, ParamCurveArea, ParamCurveCurvature,\n ParamCurveDeriv, ParamCurveExtrema, ParamCurveNearest, PathEl, Point, Rect, Shape, Vec2,\n DEFAULT_ACCURACY, MAX_EXTREMA,\n};\n\n/// A single line.\n#[derive(Clone, Copy, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct Line {\n /// The line's start point.\n pub p0: Point,\n /// The line's end point.\n pub p1: Point,\n}\n\nimpl Line {\n /// Create a new line.\n #[inline]\n pub fn new(p0: impl Into, p1: impl Into) -> Line {\n Line {\n p0: p0.into(),\n p1: p1.into(),\n }\n }\n\n /// The length of the line.\n #[inline]\n pub fn length(self) -> f64 {\n self.arclen(DEFAULT_ACCURACY)\n }\n\n /// Computes the point where two lines, if extended to infinity, would cross.\n pub fn crossing_point(self, other: Line) -> Option {\n let ab = self.p1 - self.p0;\n let cd = other.p1 - other.p0;\n let pcd = ab.cross(cd);\n if pcd == 0.0 {\n return None;\n }\n let h = ab.cross(self.p0 - other.p0) / pcd;\n Some(other.p0 + cd * h)\n }\n\n /// Is this line finite?\n #[inline]\n pub fn is_finite(self) -> bool {\n self.p0.is_finite() && self.p1.is_finite()\n }\n\n /// Is this line NaN?\n #[inline]\n pub fn is_nan(self) -> bool {\n self.p0.is_nan() || self.p1.is_nan()\n }\n}\n\nimpl ParamCurve for Line {\n #[inline]\n fn eval(&self, t: f64) -> Point {\n self.p0.lerp(self.p1, t)\n }\n\n #[inline]\n fn subsegment(&self, range: Range) -> Line {\n Line {\n p0: self.eval(range.start),\n p1: self.eval(range.end),\n }\n }\n\n #[inline]\n fn start(&self) -> Point {\n self.p0\n }\n\n #[inline]\n fn end(&self) -> Point {\n self.p1\n }\n}\n\nimpl ParamCurveDeriv for Line {\n type DerivResult = ConstPoint;\n\n #[inline]\n fn deriv(&self) -> ConstPoint {\n ConstPoint((self.p1 - self.p0).to_point())\n }\n}\n\nimpl ParamCurveArclen for Line {\n #[inline]\n fn arclen(&self, _accuracy: f64) -> f64 {\n (self.p1 - self.p0).hypot()\n }\n\n #[inline]\n fn inv_arclen(&self, arclen: f64, _accuracy: f64) -> f64 {\n arclen / (self.p1 - self.p0).hypot()\n }\n}\n\nimpl ParamCurveArea for Line {\n #[inline]\n fn signed_area(&self) -> f64 {\n self.p0.to_vec2().cross(self.p1.to_vec2()) * 0.5\n }\n}\n\nimpl ParamCurveNearest for Line {\n fn nearest(&self, p: Point, _accuracy: f64) -> Nearest {\n let d = self.p1 - self.p0;\n let dotp = d.dot(p - self.p0);\n let d_squared = d.dot(d);\n let (t, distance_sq) = if dotp <= 0.0 {\n (0.0, (p - self.p0).hypot2())\n } else if dotp >= d_squared {\n (1.0, (p - self.p1).hypot2())\n } else {\n let t = dotp / d_squared;\n let dist = (p - self.eval(t)).hypot2();\n (t, dist)\n };\n Nearest { distance_sq, t }\n }\n}\n\nimpl ParamCurveCurvature for Line {\n #[inline]\n fn curvature(&self, _t: f64) -> f64 {\n 0.0\n }\n}\n\nimpl ParamCurveExtrema for Line {\n #[inline]\n fn extrema(&self) -> ArrayVec {\n ArrayVec::new()\n }\n}\n\n/// A trivial \"curve\" that is just a constant.\n#[derive(Clone, Copy, Debug)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct ConstPoint(Point);\n\nimpl ConstPoint {\n /// Is this point finite?\n #[inline]\n pub fn is_finite(self) -> bool {\n self.0.is_finite()\n }\n\n /// Is this point NaN?\n #[inline]\n pub fn is_nan(self) -> bool {\n self.0.is_nan()\n }\n}\n\nimpl ParamCurve for ConstPoint {\n #[inline]\n fn eval(&self, _t: f64) -> Point {\n self.0\n }\n\n #[inline]\n fn subsegment(&self, _range: Range) -> ConstPoint {\n *self\n }\n}\n\nimpl ParamCurveDeriv for ConstPoint {\n type DerivResult = ConstPoint;\n\n #[inline]\n fn deriv(&self) -> ConstPoint {\n ConstPoint(Point::new(0.0, 0.0))\n }\n}\n\nimpl ParamCurveArclen for ConstPoint {\n #[inline]\n fn arclen(&self, _accuracy: f64) -> f64 {\n 0.0\n }\n\n #[inline]\n fn inv_arclen(&self, _arclen: f64, _accuracy: f64) -> f64 {\n 0.0\n }\n}\n\nimpl Mul for Affine {\n type Output = Line;\n\n #[inline]\n fn mul(self, other: Line) -> Line {\n Line {\n p0: self * other.p0,\n p1: self * other.p1,\n }\n }\n}\n\nimpl Add for Line {\n type Output = Line;\n\n #[inline]\n fn add(self, v: Vec2) -> Line {\n Line::new(self.p0 + v, self.p1 + v)\n }\n}\n\nimpl Sub for Line {\n type Output = Line;\n\n #[inline]\n fn sub(self, v: Vec2) -> Line {\n Line::new(self.p0 - v, self.p1 - v)\n }\n}\n\n/// An iterator yielding the path for a single line.\n#[doc(hidden)]\npub struct LinePathIter {\n line: Line,\n ix: usize,\n}\n\nimpl Shape for Line {\n type PathElementsIter<'iter> = LinePathIter;\n\n #[inline]\n fn path_elements(&self, _tolerance: f64) -> LinePathIter {\n LinePathIter { line: *self, ix: 0 }\n }\n\n /// Returning zero here is consistent with the contract (area is\n /// only meaningful for closed shapes), but an argument can be made\n /// that the contract should be tightened to include the Green's\n /// theorem contribution.\n fn area(&self) -> f64 {\n 0.0\n }\n\n #[inline]\n fn perimeter(&self, _accuracy: f64) -> f64 {\n (self.p1 - self.p0).hypot()\n }\n\n /// Same consideration as `area`.\n fn winding(&self, _pt: Point) -> i32 {\n 0\n }\n\n #[inline]\n fn bounding_box(&self) -> Rect {\n Rect::from_points(self.p0, self.p1)\n }\n\n #[inline]\n fn as_line(&self) -> Option {\n Some(*self)\n }\n}\n\nimpl Iterator for LinePathIter {\n type Item = PathEl;\n\n fn next(&mut self) -> Option {\n self.ix += 1;\n match self.ix {\n 1 => Some(PathEl::MoveTo(self.line.p0)),\n 2 => Some(PathEl::LineTo(self.line.p1)),\n _ => None,\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{Line, ParamCurveArclen, Point};\n\n #[test]\n fn line_arclen() {\n let l = Line::new((0.0, 0.0), (1.0, 1.0));\n let true_len = 2.0f64.sqrt();\n let epsilon = 1e-9;\n assert!(l.arclen(epsilon) - true_len < epsilon);\n\n let t = l.inv_arclen(true_len / 3.0, epsilon);\n assert!((t - 1.0 / 3.0).abs() < epsilon);\n }\n\n #[test]\n fn line_is_finite() {\n assert!((Line {\n p0: Point { x: 0., y: 0. },\n p1: Point { x: 1., y: 1. }\n })\n .is_finite());\n\n assert!(!(Line {\n p0: Point { x: 0., y: 0. },\n p1: Point {\n x: f64::INFINITY,\n y: 1.\n }\n })\n .is_finite());\n\n assert!(!(Line {\n p0: Point { x: 0., y: 0. },\n p1: Point {\n x: 0.,\n y: f64::INFINITY\n }\n })\n .is_finite());\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/mindist.rs", "sha256": "ca09ddd52dd46a872e6feb9d4036d12823f673ee2dc81854aa20d051fb4c83b6", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2021 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Minimum distance between two Bézier curves\n//!\n//! This implements the algorithm in \"Computing the minimum distance between\n//! two Bézier curves\", Chen et al., *Journal of Computational and Applied\n//! Mathematics* 229(2009), 294-301\n\nuse crate::Vec2;\nuse core::cmp::Ordering;\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\npub(crate) fn min_dist_param(\n bez1: &[Vec2],\n bez2: &[Vec2],\n u: (f64, f64),\n v: (f64, f64),\n epsilon: f64,\n best_alpha: Option,\n) -> (f64, f64, f64) {\n assert!(!bez1.is_empty() && !bez2.is_empty());\n let n = bez1.len() - 1;\n let m = bez2.len() - 1;\n let (umin, umax) = u;\n let (vmin, vmax) = v;\n let umid = (umin + umax) / 2.0;\n let vmid = (vmin + vmax) / 2.0;\n let svalues: [(f64, f64, f64); 4] = [\n (S(umin, vmin, bez1, bez2), umin, vmin),\n (S(umin, vmax, bez1, bez2), umin, vmax),\n (S(umax, vmin, bez1, bez2), umax, vmin),\n (S(umax, vmax, bez1, bez2), umax, vmax),\n ];\n let alpha: f64 = svalues.iter().map(|(a, _, _)| *a).reduce(f64::min).unwrap();\n if let Some(best) = best_alpha {\n if alpha > best {\n return (alpha, umid, vmid);\n }\n }\n\n if (umax - umin).abs() < epsilon || (vmax - vmin).abs() < epsilon {\n return (alpha, umid, vmid);\n }\n\n // Property one: D(r>k) > alpha\n let mut is_outside = true;\n let mut min_drk = None;\n let mut min_ij = None;\n for r in 0..(2 * n) {\n for k in 0..(2 * m) {\n let d_rk = D_rk(r, k, bez1, bez2);\n if d_rk < alpha {\n is_outside = false;\n }\n if min_drk.is_none() || Some(d_rk) < min_drk {\n min_drk = Some(d_rk);\n min_ij = Some((r, k));\n }\n }\n }\n if is_outside {\n return (alpha, umid, vmid);\n }\n\n // Property two: boundary check\n let mut at_boundary0on_bez1 = true;\n let mut at_boundary1on_bez1 = true;\n let mut at_boundary0on_bez2 = true;\n let mut at_boundary1on_bez2 = true;\n for i in 0..2 * n {\n for j in 0..2 * m {\n let dij = D_rk(i, j, bez1, bez2);\n let dkj = D_rk(0, j, bez1, bez2);\n if dij < dkj {\n at_boundary0on_bez1 = false\n }\n let dkj = D_rk(2 * n, j, bez1, bez2);\n if dij < dkj {\n at_boundary1on_bez1 = false\n }\n let dkj = D_rk(i, 0, bez1, bez2);\n if dij < dkj {\n at_boundary0on_bez2 = false\n }\n let dkj = D_rk(i, 2 * n, bez1, bez2);\n if dij < dkj {\n at_boundary1on_bez2 = false\n }\n }\n }\n if at_boundary0on_bez1 && at_boundary0on_bez2 {\n return svalues[0];\n }\n if at_boundary0on_bez1 && at_boundary1on_bez2 {\n return svalues[1];\n }\n if at_boundary1on_bez1 && at_boundary0on_bez2 {\n return svalues[2];\n }\n if at_boundary1on_bez1 && at_boundary1on_bez2 {\n return svalues[3];\n }\n\n let (min_i, min_j) = min_ij.unwrap();\n let new_umid = umin + (umax - umin) * (min_i as f64 / (2 * n) as f64);\n let new_vmid = vmin + (vmax - vmin) * (min_j as f64 / (2 * m) as f64);\n\n // Subdivide\n let results = [\n min_dist_param(\n bez1,\n bez2,\n (umin, new_umid),\n (vmin, new_vmid),\n epsilon,\n Some(alpha),\n ),\n min_dist_param(\n bez1,\n bez2,\n (umin, new_umid),\n (new_vmid, vmax),\n epsilon,\n Some(alpha),\n ),\n min_dist_param(\n bez1,\n bez2,\n (new_umid, umax),\n (vmin, new_vmid),\n epsilon,\n Some(alpha),\n ),\n min_dist_param(\n bez1,\n bez2,\n (new_umid, umax),\n (new_vmid, vmax),\n epsilon,\n Some(alpha),\n ),\n ];\n\n *results\n .iter()\n .min_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(Ordering::Less))\n .unwrap()\n}\n\n// $ S(u,v)=\\sum_{r=0}^{2n} \\sum _{k=0}^{2m} D_{r,k} B_{2n}^r(u) B_{2m}^k(v) $\n\n#[allow(non_snake_case)]\nfn S(u: f64, v: f64, bez1: &[Vec2], bez2: &[Vec2]) -> f64 {\n let n = bez1.len() - 1;\n let m = bez2.len() - 1;\n let mut summand = 0.0;\n for r in 0..=2 * n {\n for k in 0..=2 * m {\n summand +=\n D_rk(r, k, bez1, bez2) * basis_function(2 * n, r, u) * basis_function(2 * m, k, v)\n }\n }\n summand\n}\n\n// $ C_{r,k} = ( \\sum_{i=\\theta}^\\upsilon P_i C_n^i C_n^{r-i} / C_{2n}^r ) \\cdot (\\sum_{j=\\sigma}^\\varsigma Q_j C_m^j C_m^{k-j} / C_{2m}^k ) $\n#[allow(non_snake_case)]\nfn C_rk(r: usize, k: usize, bez1: &[Vec2], bez2: &[Vec2]) -> f64 {\n let n = bez1.len() - 1;\n let upsilon = r.min(n);\n let theta = r - n.min(r);\n let mut left: Vec2 = Vec2::ZERO;\n for (i, &item) in bez1.iter().enumerate().take(upsilon + 1).skip(theta) {\n left += item * (choose(n, i) * choose(n, r - i)) as f64 / (choose(2 * n, r) as f64);\n }\n\n let m = bez2.len() - 1;\n let varsigma = k.min(m);\n let sigma = k - m.min(k);\n let mut right: Vec2 = Vec2::ZERO;\n for (j, &item) in bez2.iter().enumerate().take(varsigma + 1).skip(sigma) {\n right += item * (choose(m, j) * choose(m, k - j)) as f64 / (choose(2 * m, k) as f64);\n }\n left.dot(right)\n}\n\n// $ A_r=\\sum _{i=\\theta} ^\\upsilon (P_i \\cdot P_{r-i}) C_n^i C_n^{r-i} / C_{2n}^r $\n// ($ B_k $ is just the same as $ A_r $ but for the other curve.)\n\n#[allow(non_snake_case)]\nfn A_r(r: usize, p: &[Vec2]) -> f64 {\n let n = p.len() - 1;\n let upsilon = r.min(n);\n let theta = r - n.min(r);\n (theta..=upsilon)\n .map(|i| {\n let dot = p[i].dot(p[r - i]); // These are bounds checked by the sum limits\n let factor = (choose(n, i) * choose(n, r - i)) as f64 / (choose(2 * n, r) as f64);\n dot * factor\n })\n .sum()\n}\n\n#[allow(non_snake_case)]\nfn D_rk(r: usize, k: usize, bez1: &[Vec2], bez2: &[Vec2]) -> f64 {\n // In the paper, B_k is used for the second factor, but it's the same thing\n A_r(r, bez1) + A_r(k, bez2) - 2.0 * C_rk(r, k, bez1, bez2)\n}\n\n// Bezier basis function\nfn basis_function(n: usize, i: usize, u: f64) -> f64 {\n choose(n, i) as f64 * (1.0 - u).powi((n - i) as i32) * u.powi(i as i32)\n}\n\n// Binomial co-efficient, but returning zeros for values outside of domain\nfn choose(n: usize, k: usize) -> u32 {\n let mut n = n;\n if k > n {\n return 0;\n }\n let mut p = 1;\n for i in 1..=(n - k) {\n p *= n;\n p /= i;\n n -= 1;\n }\n p as u32\n}\n\n#[cfg(test)]\nmod tests {\n use crate::mindist::A_r;\n use crate::mindist::{choose, D_rk};\n use crate::{CubicBez, Line, PathSeg, Vec2};\n\n #[test]\n fn test_choose() {\n assert_eq!(choose(6, 0), 1);\n assert_eq!(choose(6, 1), 6);\n assert_eq!(choose(6, 2), 15);\n }\n\n #[test]\n fn test_d_rk() {\n let bez1 = vec![\n Vec2::new(129.0, 139.0),\n Vec2::new(190.0, 139.0),\n Vec2::new(201.0, 364.0),\n Vec2::new(90.0, 364.0),\n ];\n let bez2 = vec![\n Vec2::new(309.0, 159.0),\n Vec2::new(178.0, 159.0),\n Vec2::new(215.0, 408.0),\n Vec2::new(309.0, 408.0),\n ];\n let b = A_r(1, &bez2);\n assert!((b - 80283.0).abs() < 0.005, \"B_1(Q)={b:?}\");\n let d = D_rk(0, 1, &bez1, &bez2);\n assert!((d - 9220.0).abs() < 0.005, \"D={d:?}\");\n }\n\n #[test]\n fn test_mindist() {\n let bez1 = PathSeg::Cubic(CubicBez::new(\n (129.0, 139.0),\n (190.0, 139.0),\n (201.0, 364.0),\n (90.0, 364.0),\n ));\n let bez2 = PathSeg::Cubic(CubicBez::new(\n (309.0, 159.0),\n (178.0, 159.0),\n (215.0, 408.0),\n (309.0, 408.0),\n ));\n let mindist = bez1.min_dist(bez2, 0.001);\n assert!((mindist.distance - 50.9966).abs() < 0.5);\n }\n\n #[test]\n fn test_overflow() {\n let bez1 = PathSeg::Cubic(CubicBez::new(\n (232.0, 126.0),\n (134.0, 126.0),\n (139.0, 232.0),\n (141.0, 301.0),\n ));\n let bez2 = PathSeg::Line(Line::new((359.0, 416.0), (367.0, 755.0)));\n let mindist = bez1.min_dist(bez2, 0.001);\n assert!((mindist.distance - 246.4731222669117).abs() < 0.5);\n }\n\n #[test]\n fn test_out_of_order() {\n let bez1 = PathSeg::Cubic(CubicBez::new(\n (287.0, 182.0),\n (346.0, 277.0),\n (356.0, 299.0),\n (359.0, 416.0),\n ));\n let bez2 = PathSeg::Line(Line::new((141.0, 301.0), (152.0, 709.0)));\n let mindist1 = bez1.min_dist(bez2, 0.5);\n let mindist2 = bez2.min_dist(bez1, 0.5);\n assert!((mindist1.distance - mindist2.distance).abs() < 0.5);\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/offset.rs", "sha256": "7cfc29fc66af0a9d1e255b576a2d3f247640ce48e5cf2c253730648f8c4951c1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 The Kurbo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// https://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Computation of offset curves of cubic Béziers, based on a curve fitting\n//! approach.\n//!\n//! See the [Parallel curves of cubic Béziers] blog post for a discussion of how\n//! this algorithm works and what kind of results can be expected. In general, it\n//! is expected to perform much better than most published algorithms. The number\n//! of curve segments needed to attain a given accuracy scales as O(n^6) with\n//! accuracy.\n//!\n//! In general, to compute the offset curve (also known as parallel curve) of\n//! a cubic Bézier segment, create a [`CubicOffset`] struct with the curve\n//! segment and offset, then use [`fit_to_bezpath`] or [`fit_to_bezpath_opt`]\n//! depending on how much time to spend optimizing the resulting path.\n//!\n//! [`fit_to_bezpath`]: crate::fit_to_bezpath\n//! [`fit_to_bezpath_opt`]: crate::fit_to_bezpath_opt\n//! [Parallel curves of cubic Béziers]: https://raphlinus.github.io/curves/2022/09/09/parallel-beziers.html\nuse core::ops::Range;\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\nuse crate::{\n common::solve_itp, CubicBez, CurveFitSample, ParamCurve, ParamCurveDeriv, ParamCurveFit, Point,\n QuadBez, Vec2,\n};\n\n/// The offset curve of a cubic Bézier.\n///\n/// This is a representation of the offset curve of a cubic Bézier segment, for\n/// purposes of curve fitting.\n///\n/// See the [module-level documentation] for a bit more discussion of the approach,\n/// and how this struct is to be used.\n///\n/// [module-level documentation]: crate::offset\npub struct CubicOffset {\n /// Source curve.\n c: CubicBez,\n /// Derivative of source curve.\n q: QuadBez,\n /// Offset.\n d: f64,\n // c0 + c1 t + c2 t^2 is the cross product of second and first\n // derivatives of the underlying cubic, multiplied by offset (for\n // computing cusp).\n c0: f64,\n c1: f64,\n c2: f64,\n}\n\nimpl CubicOffset {\n /// Create a new curve from Bézier segment and offset.\n ///\n /// This method should only be used if the Bézier is smooth. Use\n /// [`new_regularized`] instead to deal with a wider range of inputs.\n ///\n /// [`new_regularized`]: Self::new_regularized\n pub fn new(c: CubicBez, d: f64) -> Self {\n let q = c.deriv();\n let d0 = q.p0.to_vec2();\n let d1 = 2.0 * (q.p1 - q.p0);\n let d2 = q.p0.to_vec2() - 2.0 * q.p1.to_vec2() + q.p2.to_vec2();\n CubicOffset {\n c,\n q,\n d,\n c0: d * d1.cross(d0),\n c1: d * 2.0 * d2.cross(d0),\n c2: d * d2.cross(d1),\n }\n }\n\n /// Create a new curve from Bézier segment and offset, with numerical robustness tweaks.\n ///\n /// The dimension represents a minimum feature size; the regularization is allowed to\n /// perturb the curve by this amount in order to improve the robustness.\n pub fn new_regularized(c: CubicBez, d: f64, dimension: f64) -> Self {\n Self::new(c.regularize(dimension), d)\n }\n\n fn eval_offset(&self, t: f64) -> Vec2 {\n let dp = self.q.eval(t).to_vec2();\n let norm = Vec2::new(-dp.y, dp.x);\n // TODO: deal with hypot = 0\n norm * self.d / dp.hypot()\n }\n\n fn eval(&self, t: f64) -> Point {\n // Point on source curve.\n self.c.eval(t) + self.eval_offset(t)\n }\n\n /// Evaluate derivative of curve.\n fn eval_deriv(&self, t: f64) -> Vec2 {\n self.cusp_sign(t) * self.q.eval(t).to_vec2()\n }\n\n // Compute a function which has a zero-crossing at cusps, and is\n // positive at low curvatures on the source curve.\n fn cusp_sign(&self, t: f64) -> f64 {\n let ds2 = self.q.eval(t).to_vec2().hypot2();\n ((self.c2 * t + self.c1) * t + self.c0) / (ds2 * ds2.sqrt()) + 1.0\n }\n}\n\nimpl ParamCurveFit for CubicOffset {\n fn sample_pt_tangent(&self, t: f64, sign: f64) -> CurveFitSample {\n let p = self.eval(t);\n const CUSP_EPS: f64 = 1e-8;\n let mut cusp = self.cusp_sign(t);\n if cusp.abs() < CUSP_EPS {\n // This is a numerical derivative, which is probably good enough\n // for all practical purposes, but an analytical derivative would\n // be more elegant.\n //\n // Also, we're not dealing with second or higher order cusps.\n cusp = sign * (self.cusp_sign(t + CUSP_EPS) - self.cusp_sign(t - CUSP_EPS));\n }\n let tangent = self.q.eval(t).to_vec2() * cusp.signum();\n CurveFitSample { p, tangent }\n }\n\n fn sample_pt_deriv(&self, t: f64) -> (Point, Vec2) {\n (self.eval(t), self.eval_deriv(t))\n }\n\n fn break_cusp(&self, range: Range) -> Option {\n const CUSP_EPS: f64 = 1e-8;\n // When an endpoint is on (or very near) a cusp, move just far enough\n // away from the cusp that we're confident we have the right sign.\n let break_cusp_help = |mut x, mut d| {\n let mut cusp = self.cusp_sign(x);\n while cusp.abs() < CUSP_EPS && d < 1.0 {\n x += d;\n let old_cusp = cusp;\n cusp = self.cusp_sign(x);\n if cusp.abs() > old_cusp.abs() {\n break;\n }\n d *= 2.0;\n }\n (x, cusp)\n };\n let (a, cusp0) = break_cusp_help(range.start, 1e-12);\n let (b, cusp1) = break_cusp_help(range.end, -1e-12);\n if a >= b || cusp0 * cusp1 >= 0.0 {\n // Discussion point: maybe we should search for double cusps in the interior\n // of the range.\n return None;\n }\n let s = cusp1.signum();\n let f = |t| s * self.cusp_sign(t);\n let k1 = 0.2 / (b - a);\n const ITP_EPS: f64 = 1e-12;\n let x = solve_itp(f, a, b, ITP_EPS, 1, k1, s * cusp0, s * cusp1);\n Some(x)\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::CubicOffset;\n use crate::{fit_to_bezpath, fit_to_bezpath_opt, CubicBez, PathEl};\n\n // This test tries combinations of parameters that have caused problems in the past.\n #[test]\n fn pathological_curves() {\n let curve = CubicBez {\n p0: (-1236.3746269978635, 152.17981429574826).into(),\n p1: (-1175.18662093517, 108.04721798590596).into(),\n p2: (-1152.142883879584, 105.76260301083356).into(),\n p3: (-1151.842639804639, 105.73040758939104).into(),\n };\n let offset = 3603.7267536453924;\n let accuracy = 0.1;\n let offset_path = CubicOffset::new(curve, offset);\n let path = fit_to_bezpath_opt(&offset_path, accuracy);\n assert!(matches!(path.iter().next(), Some(PathEl::MoveTo(_))));\n let path_opt = fit_to_bezpath(&offset_path, accuracy);\n assert!(matches!(path_opt.iter().next(), Some(PathEl::MoveTo(_))));\n }\n\n /// Cubic offset that used to trigger infinite recursion.\n #[test]\n fn infinite_recursion() {\n const DIM_TUNE: f64 = 0.25;\n const TOLERANCE: f64 = 0.1;\n let c = CubicBez::new(\n (1096.2962962962963, 593.90243902439033),\n (1043.6213991769548, 593.90243902439033),\n (1030.4526748971193, 593.90243902439033),\n (1056.7901234567901, 593.90243902439033),\n );\n let co = CubicOffset::new_regularized(c, -0.5, DIM_TUNE * TOLERANCE);\n fit_to_bezpath(&co, TOLERANCE);\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/param_curve.rs", "sha256": "5c48fec80c8c67f8e2aa7f23b2d0aece5f90ca2e56579370556eb244a247353f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A trait for curves parametrized by a scalar.\n\nuse core::ops::Range;\n\nuse arrayvec::ArrayVec;\n\nuse crate::{common, Point, Rect};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// A default value for methods that take an 'accuracy' argument.\n///\n/// This value is intended to be suitable for general-purpose use, such as\n/// 2d graphics.\npub const DEFAULT_ACCURACY: f64 = 1e-6;\n\n/// A curve parametrized by a scalar.\n///\n/// If the result is interpreted as a point, this represents a curve.\n/// But the result can be interpreted as a vector as well.\npub trait ParamCurve: Sized {\n /// Evaluate the curve at parameter `t`.\n ///\n /// Generally `t` is in the range [0..1].\n fn eval(&self, t: f64) -> Point;\n\n /// Get a subsegment of the curve for the given parameter range.\n fn subsegment(&self, range: Range) -> Self;\n\n /// Subdivide into (roughly) halves.\n #[inline]\n fn subdivide(&self) -> (Self, Self) {\n (self.subsegment(0.0..0.5), self.subsegment(0.5..1.0))\n }\n\n /// The start point.\n fn start(&self) -> Point {\n self.eval(0.0)\n }\n\n /// The end point.\n fn end(&self) -> Point {\n self.eval(1.0)\n }\n}\n\n// TODO: I might not want to have separate traits for all these.\n\n/// A differentiable parametrized curve.\npub trait ParamCurveDeriv {\n /// The parametric curve obtained by taking the derivative of this one.\n type DerivResult: ParamCurve;\n\n /// The derivative of the curve.\n ///\n /// Note that the type of the return value is somewhat inaccurate, as\n /// the derivative of a curve (mapping of param to point) is a mapping\n /// of param to vector. We choose to accept this rather than have a\n /// more complex type scheme.\n fn deriv(&self) -> Self::DerivResult;\n\n /// Estimate arclength using Gaussian quadrature.\n ///\n /// The coefficients are assumed to cover the range (-1..1), which is\n /// traditional.\n #[inline]\n fn gauss_arclen(&self, coeffs: &[(f64, f64)]) -> f64 {\n let d = self.deriv();\n coeffs\n .iter()\n .map(|(wi, xi)| wi * d.eval(0.5 * (xi + 1.0)).to_vec2().hypot())\n .sum::()\n * 0.5\n }\n}\n\n/// A parametrized curve that can have its arc length measured.\npub trait ParamCurveArclen: ParamCurve {\n /// The arc length of the curve.\n ///\n /// The result is accurate to the given accuracy (subject to\n /// roundoff errors for ridiculously low values). Compute time\n /// may vary with accuracy, if the curve needs to be subdivided.\n fn arclen(&self, accuracy: f64) -> f64;\n\n /// Solve for the parameter that has the given arc length from the start.\n ///\n /// This implementation uses the IPT method, as provided by\n /// [`common::solve_itp`]. This is as robust as bisection but\n /// typically converges faster. In addition, the method takes\n /// care to compute arc lengths of increasingly smaller segments\n /// of the curve, as that is likely faster than repeatedly\n /// computing the arc length of the segment starting at t=0.\n fn inv_arclen(&self, arclen: f64, accuracy: f64) -> f64 {\n if arclen <= 0.0 {\n return 0.0;\n }\n let total_arclen = self.arclen(accuracy);\n if arclen >= total_arclen {\n return 1.0;\n }\n let mut t_last = 0.0;\n let mut arclen_last = 0.0;\n let epsilon = accuracy / total_arclen;\n let n = 1.0 - epsilon.log2().ceil().min(0.0);\n let inner_accuracy = accuracy / n;\n let f = |t: f64| {\n let (range, dir) = if t > t_last {\n (t_last..t, 1.0)\n } else {\n (t..t_last, -1.0)\n };\n let arc = self.subsegment(range).arclen(inner_accuracy);\n arclen_last += arc * dir;\n t_last = t;\n arclen_last - arclen\n };\n common::solve_itp(f, 0.0, 1.0, epsilon, 1, 0.2, -arclen, total_arclen - arclen)\n }\n}\n\n/// A parametrized curve that can have its signed area measured.\npub trait ParamCurveArea {\n /// Compute the signed area under the curve.\n ///\n /// For a closed path, the signed area of the path is the sum of signed\n /// areas of the segments. This is a variant of the \"shoelace formula.\"\n /// See:\n /// and\n /// \n ///\n /// This can be computed exactly for Béziers thanks to Green's theorem,\n /// and also for simple curves such as circular arcs. For more exotic\n /// curves, it's probably best to subdivide to cubics. We leave that\n /// to the caller, which is why we don't give an accuracy param here.\n fn signed_area(&self) -> f64;\n}\n\n/// The nearest position on a curve to some point.\n///\n/// This is returned by [`ParamCurveNearest::nearest`]\n#[derive(Debug, Clone, Copy)]\npub struct Nearest {\n /// The square of the distance from the nearest position on the curve\n /// to the given point.\n pub distance_sq: f64,\n /// The position on the curve of the nearest point, as a parameter.\n ///\n /// To resolve this to a [`Point`], use [`ParamCurve::eval`].\n pub t: f64,\n}\n\n/// A parametrized curve that reports the nearest point.\npub trait ParamCurveNearest {\n /// Find the position on the curve that is nearest to the given point.\n ///\n /// This returns a [`Nearest`] struct that contains information about\n /// the position.\n fn nearest(&self, p: Point, accuracy: f64) -> Nearest;\n}\n\n/// A parametrized curve that reports its curvature.\npub trait ParamCurveCurvature: ParamCurveDeriv\nwhere\n Self::DerivResult: ParamCurveDeriv,\n{\n /// Compute the signed curvature at parameter `t`.\n #[inline]\n fn curvature(&self, t: f64) -> f64 {\n let deriv = self.deriv();\n let deriv2 = deriv.deriv();\n let d = deriv.eval(t).to_vec2();\n let d2 = deriv2.eval(t).to_vec2();\n // TODO: What's the convention for sign? I think it should match signed\n // area - a positive area curve should have positive curvature.\n d2.cross(d) * d.hypot2().powf(-1.5)\n }\n}\n\n/// The maximum number of extrema that can be reported in the `ParamCurveExtrema` trait.\n///\n/// This is 4 to support cubic Béziers. If other curves are used, they should be\n/// subdivided to limit the number of extrema.\npub const MAX_EXTREMA: usize = 4;\n\n/// A parametrized curve that reports its extrema.\npub trait ParamCurveExtrema: ParamCurve {\n /// Compute the extrema of the curve.\n ///\n /// Only extrema within the interior of the curve count.\n /// At most four extrema can be reported, which is sufficient for\n /// cubic Béziers.\n ///\n /// The extrema should be reported in increasing parameter order.\n fn extrema(&self) -> ArrayVec;\n\n /// Return parameter ranges, each of which is monotonic within the range.\n fn extrema_ranges(&self) -> ArrayVec, { MAX_EXTREMA + 1 }> {\n let mut result = ArrayVec::new();\n let mut t0 = 0.0;\n for t in self.extrema() {\n result.push(t0..t);\n t0 = t;\n }\n result.push(t0..1.0);\n result\n }\n\n /// The smallest rectangle that encloses the curve in the range (0..1).\n fn bounding_box(&self) -> Rect {\n let mut bbox = Rect::from_points(self.start(), self.end());\n for t in self.extrema() {\n bbox = bbox.union_pt(self.eval(t))\n }\n bbox\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/point.rs", "sha256": "e686b62dcac07277d6e189bfa570b8b5fffa119a34d970e3e5eae9de466090b0", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A 2D point.\n\nuse core::fmt;\nuse core::ops::{Add, AddAssign, Sub, SubAssign};\n\nuse crate::common::FloatExt;\nuse crate::Vec2;\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// A 2D point.\n///\n/// This type represents a point in 2D space. It has the same layout as [`Vec2`][crate::Vec2], but\n/// its meaning is different: `Vec2` represents a change in location (for example velocity).\n///\n/// In general, `kurbo` overloads math operators where it makes sense, for example implementing\n/// `Affine * Point` as the point under the affine transformation. However `Point + Point` and\n/// `f64 * Point` are not implemented, because the operations do not make geometric sense. If you\n/// need to apply these operations, then 1) check what you're doing makes geometric sense, then 2)\n/// use [`Point::to_vec2`] to convert the point to a `Vec2`.\n#[derive(Clone, Copy, Default, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct Point {\n /// The x coordinate.\n pub x: f64,\n /// The y coordinate.\n pub y: f64,\n}\n\nimpl Point {\n /// The point (0, 0).\n pub const ZERO: Point = Point::new(0., 0.);\n\n /// The point at the origin; (0, 0).\n pub const ORIGIN: Point = Point::new(0., 0.);\n\n /// Create a new `Point` with the provided `x` and `y` coordinates.\n #[inline]\n pub const fn new(x: f64, y: f64) -> Self {\n Point { x, y }\n }\n\n /// Convert this point into a `Vec2`.\n #[inline]\n pub const fn to_vec2(self) -> Vec2 {\n Vec2::new(self.x, self.y)\n }\n\n /// Linearly interpolate between two points.\n #[inline]\n pub fn lerp(self, other: Point, t: f64) -> Point {\n self.to_vec2().lerp(other.to_vec2(), t).to_point()\n }\n\n /// Determine the midpoint of two points.\n #[inline]\n pub fn midpoint(self, other: Point) -> Point {\n Point::new(0.5 * (self.x + other.x), 0.5 * (self.y + other.y))\n }\n\n /// Euclidean distance.\n #[inline]\n pub fn distance(self, other: Point) -> f64 {\n (self - other).hypot()\n }\n\n /// Squared Euclidean distance.\n #[inline]\n pub fn distance_squared(self, other: Point) -> f64 {\n (self - other).hypot2()\n }\n\n /// Returns a new `Point`,\n /// with `x` and `y` rounded to the nearest integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Point;\n /// let a = Point::new(3.3, 3.6).round();\n /// let b = Point::new(3.0, -3.1).round();\n /// assert_eq!(a.x, 3.0);\n /// assert_eq!(a.y, 4.0);\n /// assert_eq!(b.x, 3.0);\n /// assert_eq!(b.y, -3.0);\n /// ```\n #[inline]\n pub fn round(self) -> Point {\n Point::new(self.x.round(), self.y.round())\n }\n\n /// Returns a new `Point`,\n /// with `x` and `y` rounded up to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Point;\n /// let a = Point::new(3.3, 3.6).ceil();\n /// let b = Point::new(3.0, -3.1).ceil();\n /// assert_eq!(a.x, 4.0);\n /// assert_eq!(a.y, 4.0);\n /// assert_eq!(b.x, 3.0);\n /// assert_eq!(b.y, -3.0);\n /// ```\n #[inline]\n pub fn ceil(self) -> Point {\n Point::new(self.x.ceil(), self.y.ceil())\n }\n\n /// Returns a new `Point`,\n /// with `x` and `y` rounded down to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Point;\n /// let a = Point::new(3.3, 3.6).floor();\n /// let b = Point::new(3.0, -3.1).floor();\n /// assert_eq!(a.x, 3.0);\n /// assert_eq!(a.y, 3.0);\n /// assert_eq!(b.x, 3.0);\n /// assert_eq!(b.y, -4.0);\n /// ```\n #[inline]\n pub fn floor(self) -> Point {\n Point::new(self.x.floor(), self.y.floor())\n }\n\n /// Returns a new `Point`,\n /// with `x` and `y` rounded away from zero to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Point;\n /// let a = Point::new(3.3, 3.6).expand();\n /// let b = Point::new(3.0, -3.1).expand();\n /// assert_eq!(a.x, 4.0);\n /// assert_eq!(a.y, 4.0);\n /// assert_eq!(b.x, 3.0);\n /// assert_eq!(b.y, -4.0);\n /// ```\n #[inline]\n pub fn expand(self) -> Point {\n Point::new(self.x.expand(), self.y.expand())\n }\n\n /// Returns a new `Point`,\n /// with `x` and `y` rounded towards zero to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Point;\n /// let a = Point::new(3.3, 3.6).trunc();\n /// let b = Point::new(3.0, -3.1).trunc();\n /// assert_eq!(a.x, 3.0);\n /// assert_eq!(a.y, 3.0);\n /// assert_eq!(b.x, 3.0);\n /// assert_eq!(b.y, -3.0);\n /// ```\n #[inline]\n pub fn trunc(self) -> Point {\n Point::new(self.x.trunc(), self.y.trunc())\n }\n\n /// Is this point finite?\n #[inline]\n pub fn is_finite(self) -> bool {\n self.x.is_finite() && self.y.is_finite()\n }\n\n /// Is this point NaN?\n #[inline]\n pub fn is_nan(self) -> bool {\n self.x.is_nan() || self.y.is_nan()\n }\n}\n\nimpl From<(f64, f64)> for Point {\n #[inline]\n fn from(v: (f64, f64)) -> Point {\n Point { x: v.0, y: v.1 }\n }\n}\n\nimpl From for (f64, f64) {\n #[inline]\n fn from(v: Point) -> (f64, f64) {\n (v.x, v.y)\n }\n}\n\nimpl Add for Point {\n type Output = Point;\n\n #[inline]\n fn add(self, other: Vec2) -> Self {\n Point::new(self.x + other.x, self.y + other.y)\n }\n}\n\nimpl AddAssign for Point {\n #[inline]\n fn add_assign(&mut self, other: Vec2) {\n *self = Point::new(self.x + other.x, self.y + other.y)\n }\n}\n\nimpl Sub for Point {\n type Output = Point;\n\n #[inline]\n fn sub(self, other: Vec2) -> Self {\n Point::new(self.x - other.x, self.y - other.y)\n }\n}\n\nimpl SubAssign for Point {\n #[inline]\n fn sub_assign(&mut self, other: Vec2) {\n *self = Point::new(self.x - other.x, self.y - other.y)\n }\n}\n\nimpl Add<(f64, f64)> for Point {\n type Output = Point;\n\n #[inline]\n fn add(self, (x, y): (f64, f64)) -> Self {\n Point::new(self.x + x, self.y + y)\n }\n}\n\nimpl AddAssign<(f64, f64)> for Point {\n #[inline]\n fn add_assign(&mut self, (x, y): (f64, f64)) {\n *self = Point::new(self.x + x, self.y + y)\n }\n}\n\nimpl Sub<(f64, f64)> for Point {\n type Output = Point;\n\n #[inline]\n fn sub(self, (x, y): (f64, f64)) -> Self {\n Point::new(self.x - x, self.y - y)\n }\n}\n\nimpl SubAssign<(f64, f64)> for Point {\n #[inline]\n fn sub_assign(&mut self, (x, y): (f64, f64)) {\n *self = Point::new(self.x - x, self.y - y)\n }\n}\n\nimpl Sub for Point {\n type Output = Vec2;\n\n #[inline]\n fn sub(self, other: Point) -> Vec2 {\n Vec2::new(self.x - other.x, self.y - other.y)\n }\n}\n\nimpl fmt::Debug for Point {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n write!(f, \"({:?}, {:?})\", self.x, self.y)\n }\n}\n\nimpl fmt::Display for Point {\n fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n write!(formatter, \"(\")?;\n fmt::Display::fmt(&self.x, formatter)?;\n write!(formatter, \", \")?;\n fmt::Display::fmt(&self.y, formatter)?;\n write!(formatter, \")\")\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n #[test]\n fn point_arithmetic() {\n assert_eq!(\n Point::new(0., 0.) - Vec2::new(10., 0.),\n Point::new(-10., 0.)\n );\n assert_eq!(\n Point::new(0., 0.) - Point::new(-5., 101.),\n Vec2::new(5., -101.)\n );\n }\n\n #[test]\n #[allow(clippy::float_cmp)]\n fn distance() {\n let p1 = Point::new(0., 10.);\n let p2 = Point::new(0., 5.);\n assert_eq!(p1.distance(p2), 5.);\n\n let p1 = Point::new(-11., 1.);\n let p2 = Point::new(-7., -2.);\n assert_eq!(p1.distance(p2), 5.);\n }\n\n #[test]\n fn display() {\n let p = Point::new(0.12345, 9.87654);\n assert_eq!(format!(\"{p}\"), \"(0.12345, 9.87654)\");\n\n let p = Point::new(0.12345, 9.87654);\n assert_eq!(format!(\"{p:.2}\"), \"(0.12, 9.88)\");\n }\n}\n\n#[cfg(feature = \"mint\")]\nimpl From for mint::Point2 {\n #[inline]\n fn from(p: Point) -> mint::Point2 {\n mint::Point2 { x: p.x, y: p.y }\n }\n}\n\n#[cfg(feature = \"mint\")]\nimpl From> for Point {\n #[inline]\n fn from(p: mint::Point2) -> Point {\n Point { x: p.x, y: p.y }\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/quadbez.rs", "sha256": "0faf471969966e1659da6af2ac28359f690b8e41f52aced64b53b19865329945", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Quadratic Bézier segments.\n\nuse core::ops::{Mul, Range};\n\nuse arrayvec::ArrayVec;\n\nuse crate::common::solve_cubic;\nuse crate::MAX_EXTREMA;\nuse crate::{\n Affine, CubicBez, Line, Nearest, ParamCurve, ParamCurveArclen, ParamCurveArea,\n ParamCurveCurvature, ParamCurveDeriv, ParamCurveExtrema, ParamCurveNearest, PathEl, Point,\n Rect, Shape,\n};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// A single quadratic Bézier segment.\n#[derive(Clone, Copy, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\n#[allow(missing_docs)]\npub struct QuadBez {\n pub p0: Point,\n pub p1: Point,\n pub p2: Point,\n}\n\nimpl QuadBez {\n /// Create a new quadratic Bézier segment.\n #[inline]\n pub fn new>(p0: V, p1: V, p2: V) -> QuadBez {\n QuadBez {\n p0: p0.into(),\n p1: p1.into(),\n p2: p2.into(),\n }\n }\n\n /// Raise the order by 1.\n ///\n /// Returns a cubic Bézier segment that exactly represents this quadratic.\n #[inline]\n pub fn raise(&self) -> CubicBez {\n CubicBez::new(\n self.p0,\n self.p0 + (2.0 / 3.0) * (self.p1 - self.p0),\n self.p2 + (2.0 / 3.0) * (self.p1 - self.p2),\n self.p2,\n )\n }\n\n /// Estimate the number of subdivisions for flattening.\n pub(crate) fn estimate_subdiv(&self, sqrt_tol: f64) -> FlattenParams {\n // Determine transformation to $y = x^2$ parabola.\n let d01 = self.p1 - self.p0;\n let d12 = self.p2 - self.p1;\n let dd = d01 - d12;\n let cross = (self.p2 - self.p0).cross(dd);\n let x0 = d01.dot(dd) * cross.recip();\n let x2 = d12.dot(dd) * cross.recip();\n let scale = (cross / (dd.hypot() * (x2 - x0))).abs();\n\n // Compute number of subdivisions needed.\n let a0 = approx_parabola_integral(x0);\n let a2 = approx_parabola_integral(x2);\n let val = if scale.is_finite() {\n let da = (a2 - a0).abs();\n let sqrt_scale = scale.sqrt();\n if x0.signum() == x2.signum() {\n da * sqrt_scale\n } else {\n // Handle cusp case (segment contains curvature maximum)\n let xmin = sqrt_tol / sqrt_scale;\n sqrt_tol * da / approx_parabola_integral(xmin)\n }\n } else {\n 0.0\n };\n let u0 = approx_parabola_inv_integral(a0);\n let u2 = approx_parabola_inv_integral(a2);\n let uscale = (u2 - u0).recip();\n FlattenParams {\n a0,\n a2,\n u0,\n uscale,\n val,\n }\n }\n\n // Maps a value from 0..1 to 0..1.\n pub(crate) fn determine_subdiv_t(&self, params: &FlattenParams, x: f64) -> f64 {\n let a = params.a0 + (params.a2 - params.a0) * x;\n let u = approx_parabola_inv_integral(a);\n (u - params.u0) * params.uscale\n }\n\n /// Is this quadratic Bezier curve finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n self.p0.is_finite() && self.p1.is_finite() && self.p2.is_finite()\n }\n\n /// Is this quadratic Bezier curve NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n self.p0.is_nan() || self.p1.is_nan() || self.p2.is_nan()\n }\n}\n\n/// An iterator for quadratic beziers.\npub struct QuadBezIter {\n quad: QuadBez,\n ix: usize,\n}\n\nimpl Shape for QuadBez {\n type PathElementsIter<'iter> = QuadBezIter;\n\n #[inline]\n fn path_elements(&self, _tolerance: f64) -> QuadBezIter {\n QuadBezIter { quad: *self, ix: 0 }\n }\n\n fn area(&self) -> f64 {\n 0.0\n }\n\n #[inline]\n fn perimeter(&self, accuracy: f64) -> f64 {\n self.arclen(accuracy)\n }\n\n fn winding(&self, _pt: Point) -> i32 {\n 0\n }\n\n #[inline]\n fn bounding_box(&self) -> Rect {\n ParamCurveExtrema::bounding_box(self)\n }\n}\n\nimpl Iterator for QuadBezIter {\n type Item = PathEl;\n\n fn next(&mut self) -> Option {\n self.ix += 1;\n match self.ix {\n 1 => Some(PathEl::MoveTo(self.quad.p0)),\n 2 => Some(PathEl::QuadTo(self.quad.p1, self.quad.p2)),\n _ => None,\n }\n }\n}\n\npub(crate) struct FlattenParams {\n a0: f64,\n a2: f64,\n u0: f64,\n uscale: f64,\n /// The number of subdivisions * 2 * sqrt_tol.\n pub(crate) val: f64,\n}\n\n/// An approximation to $\\int (1 + 4x^2) ^ -0.25 dx$\n///\n/// This is used for flattening curves.\nfn approx_parabola_integral(x: f64) -> f64 {\n const D: f64 = 0.67;\n x / (1.0 - D + (D.powi(4) + 0.25 * x * x).sqrt().sqrt())\n}\n\n/// An approximation to the inverse parabola integral.\nfn approx_parabola_inv_integral(x: f64) -> f64 {\n const B: f64 = 0.39;\n x * (1.0 - B + (B * B + 0.25 * x * x).sqrt())\n}\n\nimpl ParamCurve for QuadBez {\n #[inline]\n fn eval(&self, t: f64) -> Point {\n let mt = 1.0 - t;\n (self.p0.to_vec2() * (mt * mt)\n + (self.p1.to_vec2() * (mt * 2.0) + self.p2.to_vec2() * t) * t)\n .to_point()\n }\n\n fn subsegment(&self, range: Range) -> QuadBez {\n let (t0, t1) = (range.start, range.end);\n let p0 = self.eval(t0);\n let p2 = self.eval(t1);\n let p1 = p0 + (self.p1 - self.p0).lerp(self.p2 - self.p1, t0) * (t1 - t0);\n QuadBez { p0, p1, p2 }\n }\n\n /// Subdivide into halves, using de Casteljau.\n #[inline]\n fn subdivide(&self) -> (QuadBez, QuadBez) {\n let pm = self.eval(0.5);\n (\n QuadBez::new(self.p0, self.p0.midpoint(self.p1), pm),\n QuadBez::new(pm, self.p1.midpoint(self.p2), self.p2),\n )\n }\n\n #[inline]\n fn start(&self) -> Point {\n self.p0\n }\n\n #[inline]\n fn end(&self) -> Point {\n self.p2\n }\n}\n\nimpl ParamCurveDeriv for QuadBez {\n type DerivResult = Line;\n\n #[inline]\n fn deriv(&self) -> Line {\n Line::new(\n (2.0 * (self.p1.to_vec2() - self.p0.to_vec2())).to_point(),\n (2.0 * (self.p2.to_vec2() - self.p1.to_vec2())).to_point(),\n )\n }\n}\n\nimpl ParamCurveArclen for QuadBez {\n /// Arclength of a quadratic Bézier segment.\n ///\n /// This computation is based on an analytical formula. Since that formula suffers\n /// from numerical instability when the curve is very close to a straight line, we\n /// detect that case and fall back to Legendre-Gauss quadrature.\n ///\n /// Accuracy should be better than 1e-13 over the entire range.\n ///\n /// Adapted from \n /// with permission.\n fn arclen(&self, _accuracy: f64) -> f64 {\n let d2 = self.p0.to_vec2() - 2.0 * self.p1.to_vec2() + self.p2.to_vec2();\n let a = d2.hypot2();\n let d1 = self.p1 - self.p0;\n let c = d1.hypot2();\n if a < 5e-4 * c {\n // This case happens for nearly straight Béziers.\n //\n // Calculate arclength using Legendre-Gauss quadrature using formula from Behdad\n // in https://github.com/Pomax/BezierInfo-2/issues/77\n let v0 = (-0.492943519233745 * self.p0.to_vec2()\n + 0.430331482911935 * self.p1.to_vec2()\n + 0.0626120363218102 * self.p2.to_vec2())\n .hypot();\n let v1 = ((self.p2 - self.p0) * 0.4444444444444444).hypot();\n let v2 = (-0.0626120363218102 * self.p0.to_vec2()\n - 0.430331482911935 * self.p1.to_vec2()\n + 0.492943519233745 * self.p2.to_vec2())\n .hypot();\n return v0 + v1 + v2;\n }\n let b = 2.0 * d2.dot(d1);\n\n let sabc = (a + b + c).sqrt();\n let a2 = a.powf(-0.5);\n let a32 = a2.powi(3);\n let c2 = 2.0 * c.sqrt();\n let ba_c2 = b * a2 + c2;\n\n let v0 = 0.25 * a2 * a2 * b * (2.0 * sabc - c2) + sabc;\n // TODO: justify and fine-tune this exact constant.\n if ba_c2 < 1e-13 {\n // This case happens for Béziers with a sharp kink.\n v0\n } else {\n v0 + 0.25\n * a32\n * (4.0 * c * a - b * b)\n * (((2.0 * a + b) * a2 + 2.0 * sabc) / ba_c2).ln()\n }\n }\n}\n\nimpl ParamCurveArea for QuadBez {\n #[inline]\n fn signed_area(&self) -> f64 {\n (self.p0.x * (2.0 * self.p1.y + self.p2.y) + 2.0 * self.p1.x * (self.p2.y - self.p0.y)\n - self.p2.x * (self.p0.y + 2.0 * self.p1.y))\n * (1.0 / 6.0)\n }\n}\n\nimpl ParamCurveNearest for QuadBez {\n /// Find the nearest point, using analytical algorithm based on cubic root finding.\n fn nearest(&self, p: Point, _accuracy: f64) -> Nearest {\n fn eval_t(p: Point, t_best: &mut f64, r_best: &mut Option, t: f64, p0: Point) {\n let r = (p0 - p).hypot2();\n if r_best.map(|r_best| r < r_best).unwrap_or(true) {\n *r_best = Some(r);\n *t_best = t;\n }\n }\n fn try_t(\n q: &QuadBez,\n p: Point,\n t_best: &mut f64,\n r_best: &mut Option,\n t: f64,\n ) -> bool {\n if !(0.0..=1.0).contains(&t) {\n return true;\n }\n eval_t(p, t_best, r_best, t, q.eval(t));\n false\n }\n let d0 = self.p1 - self.p0;\n let d1 = self.p0.to_vec2() + self.p2.to_vec2() - 2.0 * self.p1.to_vec2();\n let d = self.p0 - p;\n let c0 = d.dot(d0);\n let c1 = 2.0 * d0.hypot2() + d.dot(d1);\n let c2 = 3.0 * d1.dot(d0);\n let c3 = d1.hypot2();\n let roots = solve_cubic(c0, c1, c2, c3);\n let mut r_best = None;\n let mut t_best = 0.0;\n let mut need_ends = false;\n if roots.is_empty() {\n need_ends = true;\n }\n for &t in &roots {\n need_ends |= try_t(self, p, &mut t_best, &mut r_best, t);\n }\n if need_ends {\n eval_t(p, &mut t_best, &mut r_best, 0.0, self.p0);\n eval_t(p, &mut t_best, &mut r_best, 1.0, self.p2);\n }\n\n Nearest {\n t: t_best,\n distance_sq: r_best.unwrap(),\n }\n }\n}\n\nimpl ParamCurveCurvature for QuadBez {}\n\nimpl ParamCurveExtrema for QuadBez {\n fn extrema(&self) -> ArrayVec {\n let mut result = ArrayVec::new();\n let d0 = self.p1 - self.p0;\n let d1 = self.p2 - self.p1;\n let dd = d1 - d0;\n if dd.x != 0.0 {\n let t = -d0.x / dd.x;\n if t > 0.0 && t < 1.0 {\n result.push(t);\n }\n }\n if dd.y != 0.0 {\n let t = -d0.y / dd.y;\n if t > 0.0 && t < 1.0 {\n result.push(t);\n if result.len() == 2 && result[0] > t {\n result.swap(0, 1);\n }\n }\n }\n result\n }\n}\n\nimpl Mul for Affine {\n type Output = QuadBez;\n\n #[inline]\n fn mul(self, other: QuadBez) -> QuadBez {\n QuadBez {\n p0: self * other.p0,\n p1: self * other.p1,\n p2: self * other.p2,\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{\n Affine, Nearest, ParamCurve, ParamCurveArclen, ParamCurveArea, ParamCurveDeriv,\n ParamCurveExtrema, ParamCurveNearest, Point, QuadBez,\n };\n\n fn assert_near(p0: Point, p1: Point, epsilon: f64) {\n assert!((p1 - p0).hypot() < epsilon, \"{p0:?} != {p1:?}\");\n }\n\n #[test]\n fn quadbez_deriv() {\n let q = QuadBez::new((0.0, 0.0), (0.0, 0.5), (1.0, 1.0));\n let deriv = q.deriv();\n\n let n = 10;\n for i in 0..=n {\n let t = (i as f64) * (n as f64).recip();\n let delta = 1e-6;\n let p = q.eval(t);\n let p1 = q.eval(t + delta);\n let d_approx = (p1 - p) * delta.recip();\n let d = deriv.eval(t).to_vec2();\n assert!((d - d_approx).hypot() < delta * 2.0);\n }\n }\n\n #[test]\n fn quadbez_arclen() {\n let q = QuadBez::new((0.0, 0.0), (0.0, 0.5), (1.0, 1.0));\n let true_arclen = 0.5 * 5.0f64.sqrt() + 0.25 * (2.0 + 5.0f64.sqrt()).ln();\n for i in 0..12 {\n let accuracy = 0.1f64.powi(i);\n let est = q.arclen(accuracy);\n let error = est - true_arclen;\n assert!(error.abs() < accuracy, \"{est} != {true_arclen}\");\n }\n }\n\n #[test]\n fn quadbez_arclen_pathological() {\n let q = QuadBez::new((-1.0, 0.0), (1.03, 0.0), (1.0, 0.0));\n let true_arclen = 2.0008737864167325; // A rough empirical calculation\n let accuracy = 1e-11;\n let est = q.arclen(accuracy);\n assert!(\n (est - true_arclen).abs() < accuracy,\n \"{est} != {true_arclen}\"\n );\n }\n\n #[test]\n fn quadbez_subsegment() {\n let q = QuadBez::new((3.1, 4.1), (5.9, 2.6), (5.3, 5.8));\n let t0 = 0.1;\n let t1 = 0.8;\n let qs = q.subsegment(t0..t1);\n let epsilon = 1e-12;\n let n = 10;\n for i in 0..=n {\n let t = (i as f64) * (n as f64).recip();\n let ts = t0 + t * (t1 - t0);\n assert_near(q.eval(ts), qs.eval(t), epsilon);\n }\n }\n\n #[test]\n fn quadbez_raise() {\n let q = QuadBez::new((3.1, 4.1), (5.9, 2.6), (5.3, 5.8));\n let c = q.raise();\n let qd = q.deriv();\n let cd = c.deriv();\n let epsilon = 1e-12;\n let n = 10;\n for i in 0..=n {\n let t = (i as f64) * (n as f64).recip();\n assert_near(q.eval(t), c.eval(t), epsilon);\n assert_near(qd.eval(t), cd.eval(t), epsilon);\n }\n }\n\n #[test]\n fn quadbez_signed_area() {\n // y = 1 - x^2\n let q = QuadBez::new((1.0, 0.0), (0.5, 1.0), (0.0, 1.0));\n let epsilon = 1e-12;\n assert!((q.signed_area() - 2.0 / 3.0).abs() < epsilon);\n assert!(((Affine::rotate(0.5) * q).signed_area() - 2.0 / 3.0).abs() < epsilon);\n assert!(((Affine::translate((0.0, 1.0)) * q).signed_area() - 3.5 / 3.0).abs() < epsilon);\n assert!(((Affine::translate((1.0, 0.0)) * q).signed_area() - 3.5 / 3.0).abs() < epsilon);\n }\n\n fn verify(result: Nearest, expected: f64) {\n assert!(\n (result.t - expected).abs() < 1e-6,\n \"got {result:?} expected {expected}\"\n );\n }\n\n #[test]\n fn quadbez_nearest() {\n // y = x^2\n let q = QuadBez::new((-1.0, 1.0), (0.0, -1.0), (1.0, 1.0));\n verify(q.nearest((0.0, 0.0).into(), 1e-3), 0.5);\n verify(q.nearest((0.0, 0.1).into(), 1e-3), 0.5);\n verify(q.nearest((0.0, -0.1).into(), 1e-3), 0.5);\n verify(q.nearest((0.5, 0.25).into(), 1e-3), 0.75);\n verify(q.nearest((1.0, 1.0).into(), 1e-3), 1.0);\n verify(q.nearest((1.1, 1.1).into(), 1e-3), 1.0);\n verify(q.nearest((-1.1, 1.1).into(), 1e-3), 0.0);\n let a = Affine::rotate(0.5);\n verify((a * q).nearest(a * Point::new(0.5, 0.25), 1e-3), 0.75);\n }\n\n // This test exposes a degenerate case in the solver used internally\n // by the \"nearest\" calculation - the cubic term is zero.\n #[test]\n fn quadbez_nearest_low_order() {\n let q = QuadBez::new((-1.0, 0.0), (0.0, 0.0), (1.0, 0.0));\n\n verify(q.nearest((0.0, 0.0).into(), 1e-3), 0.5);\n verify(q.nearest((0.0, 1.0).into(), 1e-3), 0.5);\n }\n\n #[test]\n fn quadbez_nearest_rounding_panic() {\n let quad = QuadBez::new(\n (-1.0394736842105263, 0.0),\n (0.8210526315789474, -1.511111111111111),\n (0.0, 1.9333333333333333),\n );\n let test = Point::new(-1.7976931348623157e308, 0.8571428571428571);\n // accuracy ignored\n let _res = quad.nearest(test, 1e-6);\n // if we got here then we didn't panic\n }\n\n #[test]\n fn quadbez_extrema() {\n // y = x^2\n let q = QuadBez::new((-1.0, 1.0), (0.0, -1.0), (1.0, 1.0));\n let extrema = q.extrema();\n assert_eq!(extrema.len(), 1);\n assert!((extrema[0] - 0.5).abs() < 1e-6);\n\n let q = QuadBez::new((0.0, 0.5), (1.0, 1.0), (0.5, 0.0));\n let extrema = q.extrema();\n assert_eq!(extrema.len(), 2);\n assert!((extrema[0] - 1.0 / 3.0).abs() < 1e-6);\n assert!((extrema[1] - 2.0 / 3.0).abs() < 1e-6);\n\n // Reverse direction\n let q = QuadBez::new((0.5, 0.0), (1.0, 1.0), (0.0, 0.5));\n let extrema = q.extrema();\n assert_eq!(extrema.len(), 2);\n assert!((extrema[0] - 1.0 / 3.0).abs() < 1e-6);\n assert!((extrema[1] - 2.0 / 3.0).abs() < 1e-6);\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/quadspline.rs", "sha256": "33051e7f381f729f89af925b6f40faa1e766b4787bba768177046fb6b699fe60", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2021 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Quadratic Bézier splines.\nuse crate::Point;\n\nuse crate::QuadBez;\nuse alloc::vec::Vec;\n\n/// A quadratic Bézier spline in [B-spline](https://en.wikipedia.org/wiki/B-spline) format.\n#[derive(Clone, Debug, PartialEq)]\npub struct QuadSpline(Vec);\n\nimpl QuadSpline {\n /// Construct a new `QuadSpline` from an array of [`Point`]s.\n #[inline]\n pub fn new(points: Vec) -> Self {\n Self(points)\n }\n\n /// Return the spline's control [`Point`]s.\n #[inline]\n pub fn points(&self) -> &[Point] {\n &self.0\n }\n\n /// Return an iterator over the implied [`QuadBez`] sequence.\n ///\n /// The returned quads are guaranteed to be G1 continuous.\n #[inline]\n pub fn to_quads(&self) -> impl Iterator + '_ {\n ToQuadBez {\n idx: 0,\n points: &self.0,\n }\n }\n}\n\nstruct ToQuadBez<'a> {\n idx: usize,\n points: &'a Vec,\n}\n\nimpl<'a> Iterator for ToQuadBez<'a> {\n type Item = QuadBez;\n\n fn next(&mut self) -> Option {\n let [mut p0, p1, mut p2]: [Point; 3] =\n self.points.get(self.idx..=self.idx + 2)?.try_into().ok()?;\n\n if self.idx != 0 {\n p0 = p0.midpoint(p1);\n }\n if self.idx + 2 < self.points.len() - 1 {\n p2 = p1.midpoint(p2);\n }\n\n self.idx += 1;\n\n Some(QuadBez { p0, p1, p2 })\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{Point, QuadBez, QuadSpline};\n\n #[test]\n pub fn no_points_no_quads() {\n assert!(QuadSpline::new(Vec::new()).to_quads().next().is_none());\n }\n\n #[test]\n pub fn one_point_no_quads() {\n assert!(QuadSpline::new(vec![Point::new(1.0, 1.0)])\n .to_quads()\n .next()\n .is_none());\n }\n\n #[test]\n pub fn two_points_no_quads() {\n assert!(\n QuadSpline::new(vec![Point::new(1.0, 1.0), Point::new(1.0, 1.0)])\n .to_quads()\n .next()\n .is_none()\n );\n }\n\n #[test]\n pub fn three_points_same_quad() {\n let p0 = Point::new(1.0, 1.0);\n let p1 = Point::new(2.0, 2.0);\n let p2 = Point::new(3.0, 3.0);\n assert_eq!(\n vec![QuadBez { p0, p1, p2 }],\n QuadSpline::new(vec![p0, p1, p2])\n .to_quads()\n .collect::>()\n );\n }\n\n #[test]\n pub fn four_points_implicit_on_curve() {\n let p0 = Point::new(1.0, 1.0);\n let p1 = Point::new(3.0, 3.0);\n let p2 = Point::new(5.0, 5.0);\n let p3 = Point::new(8.0, 8.0);\n assert_eq!(\n vec![\n QuadBez {\n p0,\n p1,\n p2: p1.midpoint(p2)\n },\n QuadBez {\n p0: p1.midpoint(p2),\n p1: p2,\n p2: p3\n }\n ],\n QuadSpline::new(vec![p0, p1, p2, p3])\n .to_quads()\n .collect::>()\n );\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/rect.rs", "sha256": "23a2fd89a0cd527c71f764840f14a152f349e8bd161ceee64ec8e038ef24e87d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A rectangle.\n\nuse core::fmt;\nuse core::ops::{Add, Sub};\n\nuse crate::{Ellipse, Insets, PathEl, Point, RoundedRect, RoundedRectRadii, Shape, Size, Vec2};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// A rectangle.\n#[derive(Clone, Copy, Default, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct Rect {\n /// The minimum x coordinate (left edge).\n pub x0: f64,\n /// The minimum y coordinate (top edge in y-down spaces).\n pub y0: f64,\n /// The maximum x coordinate (right edge).\n pub x1: f64,\n /// The maximum y coordinate (bottom edge in y-down spaces).\n pub y1: f64,\n}\n\nimpl Rect {\n /// The empty rectangle at the origin.\n pub const ZERO: Rect = Rect::new(0., 0., 0., 0.);\n\n /// A new rectangle from minimum and maximum coordinates.\n #[inline]\n pub const fn new(x0: f64, y0: f64, x1: f64, y1: f64) -> Rect {\n Rect { x0, y0, x1, y1 }\n }\n\n /// A new rectangle from two points.\n ///\n /// The result will have non-negative width and height.\n #[inline]\n pub fn from_points(p0: impl Into, p1: impl Into) -> Rect {\n let p0 = p0.into();\n let p1 = p1.into();\n Rect::new(p0.x, p0.y, p1.x, p1.y).abs()\n }\n\n /// A new rectangle from origin and size.\n ///\n /// The result will have non-negative width and height.\n #[inline]\n pub fn from_origin_size(origin: impl Into, size: impl Into) -> Rect {\n let origin = origin.into();\n Rect::from_points(origin, origin + size.into().to_vec2())\n }\n\n /// A new rectangle from center and size.\n #[inline]\n pub fn from_center_size(center: impl Into, size: impl Into) -> Rect {\n let center = center.into();\n let size = 0.5 * size.into();\n Rect::new(\n center.x - size.width,\n center.y - size.height,\n center.x + size.width,\n center.y + size.height,\n )\n }\n\n /// Create a new `Rect` with the same size as `self` and a new origin.\n #[inline]\n pub fn with_origin(self, origin: impl Into) -> Rect {\n Rect::from_origin_size(origin, self.size())\n }\n\n /// Create a new `Rect` with the same origin as `self` and a new size.\n #[inline]\n pub fn with_size(self, size: impl Into) -> Rect {\n Rect::from_origin_size(self.origin(), size)\n }\n\n /// Create a new `Rect` by applying the [`Insets`].\n ///\n /// This will not preserve negative width and height.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Rect;\n /// let inset_rect = Rect::new(0., 0., 10., 10.,).inset(2.);\n /// assert_eq!(inset_rect.width(), 14.0);\n /// assert_eq!(inset_rect.x0, -2.0);\n /// assert_eq!(inset_rect.x1, 12.0);\n /// ```\n #[inline]\n pub fn inset(self, insets: impl Into) -> Rect {\n self + insets.into()\n }\n\n /// The width of the rectangle.\n ///\n /// Note: nothing forbids negative width.\n #[inline]\n pub fn width(&self) -> f64 {\n self.x1 - self.x0\n }\n\n /// The height of the rectangle.\n ///\n /// Note: nothing forbids negative height.\n #[inline]\n pub fn height(&self) -> f64 {\n self.y1 - self.y0\n }\n\n /// Returns the minimum value for the x-coordinate of the rectangle.\n #[inline]\n pub fn min_x(&self) -> f64 {\n self.x0.min(self.x1)\n }\n\n /// Returns the maximum value for the x-coordinate of the rectangle.\n #[inline]\n pub fn max_x(&self) -> f64 {\n self.x0.max(self.x1)\n }\n\n /// Returns the minimum value for the y-coordinate of the rectangle.\n #[inline]\n pub fn min_y(&self) -> f64 {\n self.y0.min(self.y1)\n }\n\n /// Returns the maximum value for the y-coordinate of the rectangle.\n #[inline]\n pub fn max_y(&self) -> f64 {\n self.y0.max(self.y1)\n }\n\n /// The origin of the rectangle.\n ///\n /// This is the top left corner in a y-down space and with\n /// non-negative width and height.\n #[inline]\n pub fn origin(&self) -> Point {\n Point::new(self.x0, self.y0)\n }\n\n /// The size of the rectangle.\n #[inline]\n pub fn size(&self) -> Size {\n Size::new(self.width(), self.height())\n }\n\n /// The area of the rectangle.\n #[inline]\n pub fn area(&self) -> f64 {\n self.width() * self.height()\n }\n\n /// Whether this rectangle has zero area.\n ///\n /// Note: a rectangle with negative area is not considered empty.\n #[inline]\n pub fn is_empty(&self) -> bool {\n self.area() == 0.0\n }\n\n /// The center point of the rectangle.\n #[inline]\n pub fn center(&self) -> Point {\n Point::new(0.5 * (self.x0 + self.x1), 0.5 * (self.y0 + self.y1))\n }\n\n /// Returns `true` if `point` lies within `self`.\n #[inline]\n pub fn contains(&self, point: Point) -> bool {\n point.x >= self.x0 && point.x < self.x1 && point.y >= self.y0 && point.y < self.y1\n }\n\n /// Take absolute value of width and height.\n ///\n /// The resulting rect has the same extents as the original, but is\n /// guaranteed to have non-negative width and height.\n #[inline]\n pub fn abs(&self) -> Rect {\n let Rect { x0, y0, x1, y1 } = *self;\n Rect::new(x0.min(x1), y0.min(y1), x0.max(x1), y0.max(y1))\n }\n\n /// The smallest rectangle enclosing two rectangles.\n ///\n /// Results are valid only if width and height are non-negative.\n #[inline]\n pub fn union(&self, other: Rect) -> Rect {\n Rect::new(\n self.x0.min(other.x0),\n self.y0.min(other.y0),\n self.x1.max(other.x1),\n self.y1.max(other.y1),\n )\n }\n\n /// Compute the union with one point.\n ///\n /// This method includes the perimeter of zero-area rectangles.\n /// Thus, a succession of `union_pt` operations on a series of\n /// points yields their enclosing rectangle.\n ///\n /// Results are valid only if width and height are non-negative.\n pub fn union_pt(&self, pt: Point) -> Rect {\n Rect::new(\n self.x0.min(pt.x),\n self.y0.min(pt.y),\n self.x1.max(pt.x),\n self.y1.max(pt.y),\n )\n }\n\n /// The intersection of two rectangles.\n ///\n /// The result is zero-area if either input has negative width or\n /// height. The result always has non-negative width and height.\n #[inline]\n pub fn intersect(&self, other: Rect) -> Rect {\n let x0 = self.x0.max(other.x0);\n let y0 = self.y0.max(other.y0);\n let x1 = self.x1.min(other.x1);\n let y1 = self.y1.min(other.y1);\n Rect::new(x0, y0, x1.max(x0), y1.max(y0))\n }\n\n /// Expand a rectangle by a constant amount in both directions.\n ///\n /// The logic simply applies the amount in each direction. If rectangle\n /// area or added dimensions are negative, this could give odd results.\n pub fn inflate(&self, width: f64, height: f64) -> Rect {\n Rect::new(\n self.x0 - width,\n self.y0 - height,\n self.x1 + width,\n self.y1 + height,\n )\n }\n\n /// Returns a new `Rect`,\n /// with each coordinate value rounded to the nearest integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Rect;\n /// let rect = Rect::new(3.3, 3.6, 3.0, -3.1).round();\n /// assert_eq!(rect.x0, 3.0);\n /// assert_eq!(rect.y0, 4.0);\n /// assert_eq!(rect.x1, 3.0);\n /// assert_eq!(rect.y1, -3.0);\n /// ```\n #[inline]\n pub fn round(self) -> Rect {\n Rect::new(\n self.x0.round(),\n self.y0.round(),\n self.x1.round(),\n self.y1.round(),\n )\n }\n\n /// Returns a new `Rect`,\n /// with each coordinate value rounded up to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Rect;\n /// let rect = Rect::new(3.3, 3.6, 3.0, -3.1).ceil();\n /// assert_eq!(rect.x0, 4.0);\n /// assert_eq!(rect.y0, 4.0);\n /// assert_eq!(rect.x1, 3.0);\n /// assert_eq!(rect.y1, -3.0);\n /// ```\n #[inline]\n pub fn ceil(self) -> Rect {\n Rect::new(\n self.x0.ceil(),\n self.y0.ceil(),\n self.x1.ceil(),\n self.y1.ceil(),\n )\n }\n\n /// Returns a new `Rect`,\n /// with each coordinate value rounded down to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Rect;\n /// let rect = Rect::new(3.3, 3.6, 3.0, -3.1).floor();\n /// assert_eq!(rect.x0, 3.0);\n /// assert_eq!(rect.y0, 3.0);\n /// assert_eq!(rect.x1, 3.0);\n /// assert_eq!(rect.y1, -4.0);\n /// ```\n #[inline]\n pub fn floor(self) -> Rect {\n Rect::new(\n self.x0.floor(),\n self.y0.floor(),\n self.x1.floor(),\n self.y1.floor(),\n )\n }\n\n /// Returns a new `Rect`,\n /// with each coordinate value rounded away from the center of the `Rect`\n /// to the nearest integer, unless they are already an integer.\n /// That is to say this function will return the smallest possible `Rect`\n /// with integer coordinates that is a superset of `self`.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Rect;\n ///\n /// // In positive space\n /// let rect = Rect::new(3.3, 3.6, 5.6, 4.1).expand();\n /// assert_eq!(rect.x0, 3.0);\n /// assert_eq!(rect.y0, 3.0);\n /// assert_eq!(rect.x1, 6.0);\n /// assert_eq!(rect.y1, 5.0);\n ///\n /// // In both positive and negative space\n /// let rect = Rect::new(-3.3, -3.6, 5.6, 4.1).expand();\n /// assert_eq!(rect.x0, -4.0);\n /// assert_eq!(rect.y0, -4.0);\n /// assert_eq!(rect.x1, 6.0);\n /// assert_eq!(rect.y1, 5.0);\n ///\n /// // In negative space\n /// let rect = Rect::new(-5.6, -4.1, -3.3, -3.6).expand();\n /// assert_eq!(rect.x0, -6.0);\n /// assert_eq!(rect.y0, -5.0);\n /// assert_eq!(rect.x1, -3.0);\n /// assert_eq!(rect.y1, -3.0);\n ///\n /// // Inverse orientation\n /// let rect = Rect::new(5.6, -3.6, 3.3, -4.1).expand();\n /// assert_eq!(rect.x0, 6.0);\n /// assert_eq!(rect.y0, -3.0);\n /// assert_eq!(rect.x1, 3.0);\n /// assert_eq!(rect.y1, -5.0);\n /// ```\n #[inline]\n pub fn expand(self) -> Rect {\n // The compiler optimizer will remove the if branching.\n let (x0, x1) = if self.x0 < self.x1 {\n (self.x0.floor(), self.x1.ceil())\n } else {\n (self.x0.ceil(), self.x1.floor())\n };\n let (y0, y1) = if self.y0 < self.y1 {\n (self.y0.floor(), self.y1.ceil())\n } else {\n (self.y0.ceil(), self.y1.floor())\n };\n Rect::new(x0, y0, x1, y1)\n }\n\n /// Returns a new `Rect`,\n /// with each coordinate value rounded towards the center of the `Rect`\n /// to the nearest integer, unless they are already an integer.\n /// That is to say this function will return the biggest possible `Rect`\n /// with integer coordinates that is a subset of `self`.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Rect;\n ///\n /// // In positive space\n /// let rect = Rect::new(3.3, 3.6, 5.6, 4.1).trunc();\n /// assert_eq!(rect.x0, 4.0);\n /// assert_eq!(rect.y0, 4.0);\n /// assert_eq!(rect.x1, 5.0);\n /// assert_eq!(rect.y1, 4.0);\n ///\n /// // In both positive and negative space\n /// let rect = Rect::new(-3.3, -3.6, 5.6, 4.1).trunc();\n /// assert_eq!(rect.x0, -3.0);\n /// assert_eq!(rect.y0, -3.0);\n /// assert_eq!(rect.x1, 5.0);\n /// assert_eq!(rect.y1, 4.0);\n ///\n /// // In negative space\n /// let rect = Rect::new(-5.6, -4.1, -3.3, -3.6).trunc();\n /// assert_eq!(rect.x0, -5.0);\n /// assert_eq!(rect.y0, -4.0);\n /// assert_eq!(rect.x1, -4.0);\n /// assert_eq!(rect.y1, -4.0);\n ///\n /// // Inverse orientation\n /// let rect = Rect::new(5.6, -3.6, 3.3, -4.1).trunc();\n /// assert_eq!(rect.x0, 5.0);\n /// assert_eq!(rect.y0, -4.0);\n /// assert_eq!(rect.x1, 4.0);\n /// assert_eq!(rect.y1, -4.0);\n /// ```\n #[inline]\n pub fn trunc(self) -> Rect {\n // The compiler optimizer will remove the if branching.\n let (x0, x1) = if self.x0 < self.x1 {\n (self.x0.ceil(), self.x1.floor())\n } else {\n (self.x0.floor(), self.x1.ceil())\n };\n let (y0, y1) = if self.y0 < self.y1 {\n (self.y0.ceil(), self.y1.floor())\n } else {\n (self.y0.floor(), self.y1.ceil())\n };\n Rect::new(x0, y0, x1, y1)\n }\n\n /// Scales the `Rect` by `factor` with respect to the origin (the point `(0, 0)`).\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Rect;\n ///\n /// let rect = Rect::new(2., 2., 4., 6.).scale_from_origin(2.);\n /// assert_eq!(rect.x0, 4.);\n /// assert_eq!(rect.x1, 8.);\n /// ```\n #[inline]\n pub fn scale_from_origin(self, factor: f64) -> Rect {\n Rect {\n x0: self.x0 * factor,\n y0: self.y0 * factor,\n x1: self.x1 * factor,\n y1: self.y1 * factor,\n }\n }\n\n /// Creates a new [`RoundedRect`] from this `Rect` and the provided\n /// corner radius.\n #[inline]\n pub fn to_rounded_rect(self, radii: impl Into) -> RoundedRect {\n RoundedRect::from_rect(self, radii)\n }\n\n /// Returns the [`Ellipse`] that is bounded by this `Rect`.\n #[inline]\n pub fn to_ellipse(self) -> Ellipse {\n Ellipse::from_rect(self)\n }\n\n /// The aspect ratio of the `Rect`.\n ///\n /// This is defined as the height divided by the width. It measures the\n /// \"squareness\" of the rectangle (a value of `1` is square).\n ///\n /// If the width is `0` the output will be `sign(y1 - y0) * infinity`.\n ///\n /// If The width and height are `0`, the result will be `NaN`.\n #[inline]\n pub fn aspect_ratio(&self) -> f64 {\n self.size().aspect_ratio()\n }\n\n /// Returns the largest possible `Rect` that is fully contained in `self`\n /// with the given `aspect_ratio`.\n ///\n /// The aspect ratio is specified fractionally, as `height / width`.\n ///\n /// The resulting rectangle will be centered if it is smaller than the\n /// input rectangle.\n ///\n /// For the special case where the aspect ratio is `1.0`, the resulting\n /// `Rect` will be square.\n ///\n /// # Examples\n ///\n /// ```\n /// # use kurbo::Rect;\n /// let outer = Rect::new(0.0, 0.0, 10.0, 20.0);\n /// let inner = outer.contained_rect_with_aspect_ratio(1.0);\n /// // The new `Rect` is a square centered at the center of `outer`.\n /// assert_eq!(inner, Rect::new(0.0, 5.0, 10.0, 15.0));\n /// ```\n ///\n pub fn contained_rect_with_aspect_ratio(&self, aspect_ratio: f64) -> Rect {\n let (width, height) = (self.width(), self.height());\n let self_aspect = height / width;\n\n // TODO the parameter `1e-9` was chosen quickly and may not be optimal.\n if (self_aspect - aspect_ratio).abs() < 1e-9 {\n // short circuit\n *self\n } else if self_aspect.abs() < aspect_ratio.abs() {\n // shrink x to fit\n let new_width = height * aspect_ratio.recip();\n let gap = (width - new_width) * 0.5;\n let x0 = self.x0 + gap;\n let x1 = self.x1 - gap;\n Rect::new(x0, self.y0, x1, self.y1)\n } else {\n // shrink y to fit\n let new_height = width * aspect_ratio;\n let gap = (height - new_height) * 0.5;\n let y0 = self.y0 + gap;\n let y1 = self.y1 - gap;\n Rect::new(self.x0, y0, self.x1, y1)\n }\n }\n\n /// Is this rectangle finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n self.x0.is_finite() && self.x1.is_finite() && self.y0.is_finite() && self.y1.is_finite()\n }\n\n /// Is this rectangle NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n self.x0.is_nan() || self.y0.is_nan() || self.x1.is_nan() || self.y1.is_nan()\n }\n}\n\nimpl From<(Point, Point)> for Rect {\n fn from(points: (Point, Point)) -> Rect {\n Rect::from_points(points.0, points.1)\n }\n}\n\nimpl From<(Point, Size)> for Rect {\n fn from(params: (Point, Size)) -> Rect {\n Rect::from_origin_size(params.0, params.1)\n }\n}\n\nimpl Add for Rect {\n type Output = Rect;\n\n #[inline]\n fn add(self, v: Vec2) -> Rect {\n Rect::new(self.x0 + v.x, self.y0 + v.y, self.x1 + v.x, self.y1 + v.y)\n }\n}\n\nimpl Sub for Rect {\n type Output = Rect;\n\n #[inline]\n fn sub(self, v: Vec2) -> Rect {\n Rect::new(self.x0 - v.x, self.y0 - v.y, self.x1 - v.x, self.y1 - v.y)\n }\n}\n\nimpl Sub for Rect {\n type Output = Insets;\n\n #[inline]\n fn sub(self, other: Rect) -> Insets {\n let x0 = other.x0 - self.x0;\n let y0 = other.y0 - self.y0;\n let x1 = self.x1 - other.x1;\n let y1 = self.y1 - other.y1;\n Insets { x0, y0, x1, y1 }\n }\n}\n\n#[doc(hidden)]\npub struct RectPathIter {\n rect: Rect,\n ix: usize,\n}\n\nimpl Shape for Rect {\n type PathElementsIter<'iter> = RectPathIter;\n\n fn path_elements(&self, _tolerance: f64) -> RectPathIter {\n RectPathIter { rect: *self, ix: 0 }\n }\n\n // It's a bit of duplication having both this and the impl method, but\n // removing that would require using the trait. We'll leave it for now.\n #[inline]\n fn area(&self) -> f64 {\n Rect::area(self)\n }\n\n #[inline]\n fn perimeter(&self, _accuracy: f64) -> f64 {\n 2.0 * (self.width().abs() + self.height().abs())\n }\n\n /// Note: this function is carefully designed so that if the plane is\n /// tiled with rectangles, the winding number will be nonzero for exactly\n /// one of them.\n #[inline]\n fn winding(&self, pt: Point) -> i32 {\n let xmin = self.x0.min(self.x1);\n let xmax = self.x0.max(self.x1);\n let ymin = self.y0.min(self.y1);\n let ymax = self.y0.max(self.y1);\n if pt.x >= xmin && pt.x < xmax && pt.y >= ymin && pt.y < ymax {\n if (self.x1 > self.x0) ^ (self.y1 > self.y0) {\n -1\n } else {\n 1\n }\n } else {\n 0\n }\n }\n\n #[inline]\n fn bounding_box(&self) -> Rect {\n self.abs()\n }\n\n #[inline]\n fn as_rect(&self) -> Option {\n Some(*self)\n }\n\n #[inline]\n fn contains(&self, pt: Point) -> bool {\n self.contains(pt)\n }\n}\n\n// This is clockwise in a y-down coordinate system for positive area.\nimpl Iterator for RectPathIter {\n type Item = PathEl;\n\n fn next(&mut self) -> Option {\n self.ix += 1;\n match self.ix {\n 1 => Some(PathEl::MoveTo(Point::new(self.rect.x0, self.rect.y0))),\n 2 => Some(PathEl::LineTo(Point::new(self.rect.x1, self.rect.y0))),\n 3 => Some(PathEl::LineTo(Point::new(self.rect.x1, self.rect.y1))),\n 4 => Some(PathEl::LineTo(Point::new(self.rect.x0, self.rect.y1))),\n 5 => Some(PathEl::ClosePath),\n _ => None,\n }\n }\n}\n\nimpl fmt::Debug for Rect {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n if f.alternate() {\n write!(\n f,\n \"Rect {{ origin: {:?}, size: {:?} }}\",\n self.origin(),\n self.size()\n )\n } else {\n write!(\n f,\n \"Rect {{ x0: {:?}, y0: {:?}, x1: {:?}, y1: {:?} }}\",\n self.x0, self.y0, self.x1, self.y1\n )\n }\n }\n}\n\nimpl fmt::Display for Rect {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n write!(f, \"Rect {{ \")?;\n fmt::Display::fmt(&self.origin(), f)?;\n write!(f, \" \")?;\n fmt::Display::fmt(&self.size(), f)?;\n write!(f, \" }}\")\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{Point, Rect, Shape};\n\n fn assert_approx_eq(x: f64, y: f64) {\n assert!((x - y).abs() < 1e-7);\n }\n\n #[test]\n fn area_sign() {\n let r = Rect::new(0.0, 0.0, 10.0, 10.0);\n let center = r.center();\n assert_approx_eq(r.area(), 100.0);\n\n assert_eq!(r.winding(center), 1);\n\n let p = r.to_path(1e-9);\n assert_approx_eq(r.area(), p.area());\n assert_eq!(r.winding(center), p.winding(center));\n\n let r_flip = Rect::new(0.0, 10.0, 10.0, 0.0);\n assert_approx_eq(r_flip.area(), -100.0);\n\n assert_eq!(r_flip.winding(Point::new(5.0, 5.0)), -1);\n let p_flip = r_flip.to_path(1e-9);\n assert_approx_eq(r_flip.area(), p_flip.area());\n assert_eq!(r_flip.winding(center), p_flip.winding(center));\n }\n\n #[test]\n fn display() {\n let r = Rect::from_origin_size((10., 12.23214), (22.222222222, 23.1));\n assert_eq!(\n format!(\"{r}\"),\n \"Rect { (10, 12.23214) (22.222222222×23.1) }\"\n );\n assert_eq!(format!(\"{r:.2}\"), \"Rect { (10.00, 12.23) (22.22×23.10) }\");\n }\n\n /* TODO uncomment when a (possibly approximate) equality test has been decided on\n #[test]\n fn rect_from_center_size() {\n assert_eq!(\n Rect::from_center_size(Point::new(3.0, 2.0), Size::new(2.0, 4.0)),\n Rect::new(2.0, 0.0, 4.0, 4.0)\n );\n }\n */\n\n #[test]\n fn contained_rect_with_aspect_ratio() {\n fn case(outer: [f64; 4], aspect_ratio: f64, expected: [f64; 4]) {\n let outer = Rect::new(outer[0], outer[1], outer[2], outer[3]);\n let expected = Rect::new(expected[0], expected[1], expected[2], expected[3]);\n assert_eq!(\n outer.contained_rect_with_aspect_ratio(aspect_ratio),\n expected\n );\n }\n // squares (different point orderings)\n case([0.0, 0.0, 10.0, 20.0], 1.0, [0.0, 5.0, 10.0, 15.0]);\n case([0.0, 20.0, 10.0, 0.0], 1.0, [0.0, 5.0, 10.0, 15.0]);\n case([10.0, 0.0, 0.0, 20.0], 1.0, [10.0, 15.0, 0.0, 5.0]);\n case([10.0, 20.0, 0.0, 0.0], 1.0, [10.0, 15.0, 0.0, 5.0]);\n // non-square\n case([0.0, 0.0, 10.0, 20.0], 0.5, [0.0, 7.5, 10.0, 12.5]);\n // same aspect ratio\n case([0.0, 0.0, 10.0, 20.0], 2.0, [0.0, 0.0, 10.0, 20.0]);\n // negative aspect ratio\n case([0.0, 0.0, 10.0, 20.0], -1.0, [0.0, 15.0, 10.0, 5.0]);\n // infinite aspect ratio\n case([0.0, 0.0, 10.0, 20.0], f64::INFINITY, [5.0, 0.0, 5.0, 20.0]);\n // zero aspect ratio\n case([0.0, 0.0, 10.0, 20.0], 0.0, [0.0, 10.0, 10.0, 10.0]);\n // zero width rect\n case([0.0, 0.0, 0.0, 20.0], 1.0, [0.0, 10.0, 0.0, 10.0]);\n // many zeros\n case([0.0, 0.0, 0.0, 20.0], 0.0, [0.0, 10.0, 0.0, 10.0]);\n // everything zero\n case([0.0, 0.0, 0.0, 0.0], 0.0, [0.0, 0.0, 0.0, 0.0]);\n }\n\n #[test]\n fn aspect_ratio() {\n let test = Rect::new(0.0, 0.0, 1.0, 1.0);\n assert!((test.aspect_ratio() - 1.0).abs() < 1e-6);\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/rounded_rect.rs", "sha256": "33f2a8ff36d011546680eb25d1b596de53afb01d9ff81a17966d225f0cdf71da", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A rectangle with rounded corners.\n\nuse core::f64::consts::{FRAC_PI_2, FRAC_PI_4};\nuse core::ops::{Add, Sub};\n\nuse crate::{arc::ArcAppendIter, Arc, PathEl, Point, Rect, RoundedRectRadii, Shape, Size, Vec2};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// A rectangle with equally rounded corners.\n///\n/// By construction the rounded rectangle will have\n/// non-negative dimensions and radii clamped to half size of the rect.\n///\n/// The easiest way to create a `RoundedRect` is often to create a [`Rect`],\n/// and then call [`to_rounded_rect`].\n///\n/// ```\n/// use kurbo::{RoundedRect, RoundedRectRadii};\n///\n/// // Create a rounded rectangle with a single radius for all corners:\n/// RoundedRect::new(0.0, 0.0, 10.0, 10.0, 5.0);\n///\n/// // Or, specify different radii for each corner, clockwise from the top-left:\n/// RoundedRect::new(0.0, 0.0, 10.0, 10.0, (1.0, 2.0, 3.0, 4.0));\n/// ```\n///\n/// [`to_rounded_rect`]: Rect::to_rounded_rect\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct RoundedRect {\n /// Coordinates of the rectangle.\n rect: Rect,\n /// Radius of all four corners.\n radii: RoundedRectRadii,\n}\n\nimpl RoundedRect {\n /// A new rectangle from minimum and maximum coordinates.\n ///\n /// The result will have non-negative width, height and radii.\n #[inline]\n pub fn new(\n x0: f64,\n y0: f64,\n x1: f64,\n y1: f64,\n radii: impl Into,\n ) -> RoundedRect {\n RoundedRect::from_rect(Rect::new(x0, y0, x1, y1), radii)\n }\n\n /// A new rounded rectangle from a rectangle and corner radii.\n ///\n /// The result will have non-negative width, height and radii.\n ///\n /// See also [`Rect::to_rounded_rect`], which offers the same utility.\n #[inline]\n pub fn from_rect(rect: Rect, radii: impl Into) -> RoundedRect {\n let rect = rect.abs();\n let shortest_side_length = (rect.width()).min(rect.height());\n let radii = radii.into().abs().clamp(shortest_side_length / 2.0);\n\n RoundedRect { rect, radii }\n }\n\n /// A new rectangle from two [`Point`]s.\n ///\n /// The result will have non-negative width, height and radius.\n #[inline]\n pub fn from_points(\n p0: impl Into,\n p1: impl Into,\n radii: impl Into,\n ) -> RoundedRect {\n Rect::from_points(p0, p1).to_rounded_rect(radii)\n }\n\n /// A new rectangle from origin and size.\n ///\n /// The result will have non-negative width, height and radius.\n #[inline]\n pub fn from_origin_size(\n origin: impl Into,\n size: impl Into,\n radii: impl Into,\n ) -> RoundedRect {\n Rect::from_origin_size(origin, size).to_rounded_rect(radii)\n }\n\n /// The width of the rectangle.\n #[inline]\n pub fn width(&self) -> f64 {\n self.rect.width()\n }\n\n /// The height of the rectangle.\n #[inline]\n pub fn height(&self) -> f64 {\n self.rect.height()\n }\n\n /// Radii of the rounded corners.\n #[inline]\n pub fn radii(&self) -> RoundedRectRadii {\n self.radii\n }\n\n /// The (non-rounded) rectangle.\n pub fn rect(&self) -> Rect {\n self.rect\n }\n\n /// The origin of the rectangle.\n ///\n /// This is the top left corner in a y-down space.\n #[inline]\n pub fn origin(&self) -> Point {\n self.rect.origin()\n }\n\n /// The center point of the rectangle.\n #[inline]\n pub fn center(&self) -> Point {\n self.rect.center()\n }\n\n /// Is this rounded rectangle finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n self.rect.is_finite() && self.radii.is_finite()\n }\n\n /// Is this rounded rectangle NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n self.rect.is_nan() || self.radii.is_nan()\n }\n}\n\n#[doc(hidden)]\npub struct RoundedRectPathIter {\n idx: usize,\n rect: RectPathIter,\n arcs: [ArcAppendIter; 4],\n}\n\nimpl Shape for RoundedRect {\n type PathElementsIter<'iter> = RoundedRectPathIter;\n\n fn path_elements(&self, tolerance: f64) -> RoundedRectPathIter {\n let radii = self.radii();\n\n let build_arc_iter = |i, center, ellipse_radii| {\n let arc = Arc {\n center,\n radii: ellipse_radii,\n start_angle: FRAC_PI_2 * i as f64,\n sweep_angle: FRAC_PI_2,\n x_rotation: 0.0,\n };\n arc.append_iter(tolerance)\n };\n\n // Note: order follows the rectangle path iterator.\n let arcs = [\n build_arc_iter(\n 2,\n Point {\n x: self.rect.x0 + radii.top_left,\n y: self.rect.y0 + radii.top_left,\n },\n Vec2 {\n x: radii.top_left,\n y: radii.top_left,\n },\n ),\n build_arc_iter(\n 3,\n Point {\n x: self.rect.x1 - radii.top_right,\n y: self.rect.y0 + radii.top_right,\n },\n Vec2 {\n x: radii.top_right,\n y: radii.top_right,\n },\n ),\n build_arc_iter(\n 0,\n Point {\n x: self.rect.x1 - radii.bottom_right,\n y: self.rect.y1 - radii.bottom_right,\n },\n Vec2 {\n x: radii.bottom_right,\n y: radii.bottom_right,\n },\n ),\n build_arc_iter(\n 1,\n Point {\n x: self.rect.x0 + radii.bottom_left,\n y: self.rect.y1 - radii.bottom_left,\n },\n Vec2 {\n x: radii.bottom_left,\n y: radii.bottom_left,\n },\n ),\n ];\n\n let rect = RectPathIter {\n rect: self.rect,\n ix: 0,\n radii,\n };\n\n RoundedRectPathIter { idx: 0, rect, arcs }\n }\n\n #[inline]\n fn area(&self) -> f64 {\n // A corner is a quarter-circle, i.e.\n // .............#\n // . ######\n // . #########\n // . ###########\n // . ############\n // .#############\n // ##############\n // |-----r------|\n // For each corner, we need to subtract the square that bounds this\n // quarter-circle, and add back in the area of quarter circle.\n\n let radii = self.radii();\n\n // Start with the area of the bounding rectangle. For each corner,\n // subtract the area of the corner under the quarter-circle, and add\n // back the area of the quarter-circle.\n self.rect.area()\n + [\n radii.top_left,\n radii.top_right,\n radii.bottom_right,\n radii.bottom_left,\n ]\n .iter()\n .map(|radius| (FRAC_PI_4 - 1.0) * radius * radius)\n .sum::()\n }\n\n #[inline]\n fn perimeter(&self, _accuracy: f64) -> f64 {\n // A corner is a quarter-circle, i.e.\n // .............#\n // . #\n // . #\n // . #\n // . #\n // .#\n // #\n // |-----r------|\n // If we start with the bounding rectangle, then subtract 2r (the\n // straight edge outside the circle) and add 1/4 * pi * (2r) (the\n // perimeter of the quarter-circle) for each corner with radius r, we\n // get the perimeter of the shape.\n\n let radii = self.radii();\n\n // Start with the full perimeter. For each corner, subtract the\n // border surrounding the rounded corner and add the quarter-circle\n // perimeter.\n self.rect.perimeter(1.0)\n + ([\n radii.top_left,\n radii.top_right,\n radii.bottom_right,\n radii.bottom_left,\n ])\n .iter()\n .map(|radius| (-2.0 + FRAC_PI_2) * radius)\n .sum::()\n }\n\n #[inline]\n fn winding(&self, mut pt: Point) -> i32 {\n let center = self.center();\n\n // 1. Translate the point relative to the center of the rectangle.\n pt.x -= center.x;\n pt.y -= center.y;\n\n // 2. Pick a radius value to use based on which quadrant the point is\n // in.\n let radii = self.radii();\n let radius = match pt {\n pt if pt.x < 0.0 && pt.y < 0.0 => radii.top_left,\n pt if pt.x >= 0.0 && pt.y < 0.0 => radii.top_right,\n pt if pt.x >= 0.0 && pt.y >= 0.0 => radii.bottom_right,\n pt if pt.x < 0.0 && pt.y >= 0.0 => radii.bottom_left,\n _ => 0.0,\n };\n\n // 3. This is the width and height of a rectangle with one corner at\n // the center of the rounded rectangle, and another corner at the\n // center of the relevant corner circle.\n let inside_half_width = (self.width() / 2.0 - radius).max(0.0);\n let inside_half_height = (self.height() / 2.0 - radius).max(0.0);\n\n // 4. Three things are happening here.\n //\n // First, the x- and y-values are being reflected into the positive\n // (bottom-right quadrant). The radius has already been determined,\n // so it doesn't matter what quadrant is used.\n //\n // After reflecting, the points are clamped so that their x- and y-\n // values can't be lower than the x- and y- values of the center of\n // the corner circle, and the coordinate system is transformed\n // again, putting (0, 0) at the center of the corner circle.\n let px = (pt.x.abs() - inside_half_width).max(0.0);\n let py = (pt.y.abs() - inside_half_height).max(0.0);\n\n // 5. The transforms above clamp all input points such that they will\n // be inside the rounded rectangle if the corresponding output point\n // (px, py) is inside a circle centered around the origin with the\n // given radius.\n let inside = px * px + py * py <= radius * radius;\n if inside {\n 1\n } else {\n 0\n }\n }\n\n #[inline]\n fn bounding_box(&self) -> Rect {\n self.rect.bounding_box()\n }\n\n #[inline]\n fn as_rounded_rect(&self) -> Option {\n Some(*self)\n }\n}\n\nstruct RectPathIter {\n rect: Rect,\n radii: RoundedRectRadii,\n ix: usize,\n}\n\n// This is clockwise in a y-down coordinate system for positive area.\nimpl Iterator for RectPathIter {\n type Item = PathEl;\n\n fn next(&mut self) -> Option {\n self.ix += 1;\n match self.ix {\n 1 => Some(PathEl::MoveTo(Point::new(\n self.rect.x0,\n self.rect.y0 + self.radii.top_left,\n ))),\n 2 => Some(PathEl::LineTo(Point::new(\n self.rect.x1 - self.radii.top_right,\n self.rect.y0,\n ))),\n 3 => Some(PathEl::LineTo(Point::new(\n self.rect.x1,\n self.rect.y1 - self.radii.bottom_right,\n ))),\n 4 => Some(PathEl::LineTo(Point::new(\n self.rect.x0 + self.radii.bottom_left,\n self.rect.y1,\n ))),\n 5 => Some(PathEl::ClosePath),\n _ => None,\n }\n }\n}\n\n// This is clockwise in a y-down coordinate system for positive area.\nimpl Iterator for RoundedRectPathIter {\n type Item = PathEl;\n\n fn next(&mut self) -> Option {\n if self.idx > 4 {\n return None;\n }\n\n // Iterate between rectangle and arc iterators.\n // Rect iterator will start and end the path.\n\n // Initial point set by the rect iterator\n if self.idx == 0 {\n self.idx += 1;\n return self.rect.next();\n }\n\n // Generate the arc curve elements.\n // If we reached the end of the arc, add a line towards next arc (rect iterator).\n match self.arcs[self.idx - 1].next() {\n Some(elem) => Some(elem),\n None => {\n self.idx += 1;\n self.rect.next()\n }\n }\n }\n}\n\nimpl Add for RoundedRect {\n type Output = RoundedRect;\n\n #[inline]\n fn add(self, v: Vec2) -> RoundedRect {\n RoundedRect::from_rect(self.rect + v, self.radii)\n }\n}\n\nimpl Sub for RoundedRect {\n type Output = RoundedRect;\n\n #[inline]\n fn sub(self, v: Vec2) -> RoundedRect {\n RoundedRect::from_rect(self.rect - v, self.radii)\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{Circle, Point, Rect, RoundedRect, Shape};\n\n #[test]\n fn area() {\n let epsilon = 1e-9;\n\n // Extremum: 0.0 radius corner -> rectangle\n let rect = Rect::new(0.0, 0.0, 100.0, 100.0);\n let rounded_rect = RoundedRect::new(0.0, 0.0, 100.0, 100.0, 0.0);\n assert!((rect.area() - rounded_rect.area()).abs() < epsilon);\n\n // Extremum: half-size radius corner -> circle\n let circle = Circle::new((0.0, 0.0), 50.0);\n let rounded_rect = RoundedRect::new(0.0, 0.0, 100.0, 100.0, 50.0);\n assert!((circle.area() - rounded_rect.area()).abs() < epsilon);\n }\n\n #[test]\n fn winding() {\n let rect = RoundedRect::new(-5.0, -5.0, 10.0, 20.0, (5.0, 5.0, 5.0, 0.0));\n assert_eq!(rect.winding(Point::new(0.0, 0.0)), 1);\n assert_eq!(rect.winding(Point::new(-5.0, 0.0)), 1); // left edge\n assert_eq!(rect.winding(Point::new(0.0, 20.0)), 1); // bottom edge\n assert_eq!(rect.winding(Point::new(10.0, 20.0)), 0); // bottom-right corner\n assert_eq!(rect.winding(Point::new(-5.0, 20.0)), 1); // bottom-left corner (has a radius of 0)\n assert_eq!(rect.winding(Point::new(-10.0, 0.0)), 0);\n\n let rect = RoundedRect::new(-10.0, -20.0, 10.0, 20.0, 0.0); // rectangle\n assert_eq!(rect.winding(Point::new(10.0, 20.0)), 1); // bottom-right corner\n }\n\n #[test]\n fn bez_conversion() {\n let rect = RoundedRect::new(-5.0, -5.0, 10.0, 20.0, 5.0);\n let p = rect.to_path(1e-9);\n // Note: could be more systematic about tolerance tightness.\n let epsilon = 1e-7;\n assert!((rect.area() - p.area()).abs() < epsilon);\n assert_eq!(p.winding(Point::new(0.0, 0.0)), 1);\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/rounded_rect_radii.rs", "sha256": "649cd2f42d454e8c159626285730210ddc6a457b6ea78dcb4d299fbd8d897830", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2021 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A description of the radii for each corner of a rounded rectangle.\n\nuse core::convert::From;\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// Radii for each corner of a rounded rectangle.\n///\n/// The use of `top` as in `top_left` assumes a y-down coordinate space. Piet\n/// (and Druid by extension) uses a y-down coordinate space, but Kurbo also\n/// supports a y-up coordinate space, in which case `top_left` would actually\n/// refer to the bottom-left corner, and vice versa. Top may not always\n/// actually be the top, but `top` corners will always have a smaller y-value\n/// than `bottom` corners.\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct RoundedRectRadii {\n /// The radius of the top-left corner.\n pub top_left: f64,\n /// The radius of the top-right corner.\n pub top_right: f64,\n /// The radius of the bottom-right corner.\n pub bottom_right: f64,\n /// The radius of the bottom-left corner.\n pub bottom_left: f64,\n}\n\nimpl RoundedRectRadii {\n /// Create a new `RoundedRectRadii`. This function takes radius values for\n /// the four corners. The argument order is `top_left`, `top_right`,\n /// `bottom_right`, `bottom_left`, or clockwise starting from `top_left`.\n pub const fn new(top_left: f64, top_right: f64, bottom_right: f64, bottom_left: f64) -> Self {\n RoundedRectRadii {\n top_left,\n top_right,\n bottom_right,\n bottom_left,\n }\n }\n\n /// Create a new `RoundedRectRadii` from a single radius. The `radius`\n /// argument will be set as the radius for all four corners.\n pub const fn from_single_radius(radius: f64) -> Self {\n RoundedRectRadii {\n top_left: radius,\n top_right: radius,\n bottom_right: radius,\n bottom_left: radius,\n }\n }\n\n /// Takes the absolute value of all corner radii.\n pub fn abs(&self) -> Self {\n RoundedRectRadii::new(\n self.top_left.abs(),\n self.top_right.abs(),\n self.bottom_right.abs(),\n self.bottom_left.abs(),\n )\n }\n\n /// For each corner, takes the min of that corner's radius and `max`.\n pub fn clamp(&self, max: f64) -> Self {\n RoundedRectRadii::new(\n self.top_left.min(max),\n self.top_right.min(max),\n self.bottom_right.min(max),\n self.bottom_left.min(max),\n )\n }\n\n /// Returns `true` if all radius values are finite.\n pub fn is_finite(&self) -> bool {\n self.top_left.is_finite()\n && self.top_right.is_finite()\n && self.bottom_right.is_finite()\n && self.bottom_left.is_finite()\n }\n\n /// Returns `true` if any corner radius value is NaN.\n pub fn is_nan(&self) -> bool {\n self.top_left.is_nan()\n || self.top_right.is_nan()\n || self.bottom_right.is_nan()\n || self.bottom_left.is_nan()\n }\n\n /// If all radii are equal, returns the value of the radii. Otherwise,\n /// returns `None`.\n pub fn as_single_radius(&self) -> Option {\n let epsilon = 1e-9;\n\n if (self.top_left - self.top_right).abs() < epsilon\n && (self.top_right - self.bottom_right).abs() < epsilon\n && (self.bottom_right - self.bottom_left).abs() < epsilon\n {\n Some(self.top_left)\n } else {\n None\n }\n }\n}\n\nimpl From for RoundedRectRadii {\n fn from(radius: f64) -> Self {\n RoundedRectRadii::from_single_radius(radius)\n }\n}\n\nimpl From<(f64, f64, f64, f64)> for RoundedRectRadii {\n fn from(radii: (f64, f64, f64, f64)) -> Self {\n RoundedRectRadii::new(radii.0, radii.1, radii.2, radii.3)\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/shape.rs", "sha256": "c94af4bbeea5a3df2f54198754919b211b512773e8c5bdcf0003c80036dc5d9e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A generic trait for shapes.\n\nuse crate::{segments, BezPath, Circle, Line, PathEl, Point, Rect, RoundedRect, Segments};\n\n/// A generic trait for open and closed shapes.\n///\n/// This trait provides conversion from shapes to [`BezPath`]s, as well as\n/// general geometry functionality like computing [`area`], [`bounding_box`]es,\n/// and [`winding`] number.\n///\n/// [`area`]: Shape::area\n/// [`bounding_box`]: Shape::bounding_box\n/// [`winding`]: Shape::winding\npub trait Shape: Sized {\n /// The iterator returned by the [`path_elements`] method.\n ///\n /// [`path_elements`]: Shape::path_elements\n type PathElementsIter<'iter>: Iterator + 'iter\n where\n Self: 'iter;\n\n /// Returns an iterator over this shape expressed as [`PathEl`]s;\n /// that is, as Bézier path _elements_.\n ///\n /// All shapes can be represented as Béziers, but in many situations\n /// (such as when interfacing with a platform drawing API) there are more\n /// efficient native types for specific concrete shapes. In this case,\n /// the user should exhaust the `as_` methods ([`as_rect`], [`as_line`], etc)\n /// before converting to a [`BezPath`], as those are likely to be more\n /// efficient.\n ///\n /// In many cases, shapes are able to iterate their elements without\n /// allocating; however creating a [`BezPath`] object always allocates.\n /// If you need an owned [`BezPath`] you can use [`to_path`] instead.\n ///\n /// # Tolerance\n ///\n /// The `tolerance` parameter controls the accuracy of\n /// conversion of geometric primitives to Bézier curves, as\n /// curves such as circles cannot be represented exactly but\n /// only approximated. For drawing as in UI elements, a value\n /// of 0.1 is appropriate, as it is unlikely to be visible to\n /// the eye. For scientific applications, a smaller value\n /// might be appropriate. Note that in general the number of\n /// cubic Bézier segments scales as `tolerance ^ (-1/6)`.\n ///\n /// [`as_rect`]: Shape::as_rect\n /// [`as_line`]: Shape::as_line\n /// [`to_path`]: Shape::to_path\n fn path_elements(&self, tolerance: f64) -> Self::PathElementsIter<'_>;\n\n /// Convert to a Bézier path.\n ///\n /// This always allocates. It is appropriate when both the source\n /// shape and the resulting path are to be retained.\n ///\n /// If you only need to iterate the elements (such as to convert them to\n /// drawing commands for a given 2D graphics API) you should prefer\n /// [`path_elements`], which can avoid allocating where possible.\n ///\n /// The `tolerance` parameter is the same as for [`path_elements`].\n ///\n /// [`path_elements`]: Shape::path_elements\n fn to_path(&self, tolerance: f64) -> BezPath {\n self.path_elements(tolerance).collect()\n }\n\n #[deprecated(since = \"0.7.0\", note = \"Use path_elements instead\")]\n #[doc(hidden)]\n fn to_bez_path(&self, tolerance: f64) -> Self::PathElementsIter<'_> {\n self.path_elements(tolerance)\n }\n /// Convert into a Bézier path.\n ///\n /// This allocates in the general case, but is zero-cost if the\n /// shape is already a [`BezPath`].\n ///\n /// The `tolerance` parameter is the same as for [`path_elements()`].\n ///\n /// [`path_elements()`]: Shape::path_elements\n fn into_path(self, tolerance: f64) -> BezPath {\n self.to_path(tolerance)\n }\n\n #[deprecated(since = \"0.7.0\", note = \"Use into_path instead\")]\n #[doc(hidden)]\n fn into_bez_path(self, tolerance: f64) -> BezPath {\n self.into_path(tolerance)\n }\n\n /// Returns an iterator over this shape expressed as Bézier path\n /// _segments_ ([`PathSeg`]s).\n ///\n /// The allocation behaviour and `tolerance` parameter are the\n /// same as for [`path_elements()`]\n ///\n /// [`PathSeg`]: crate::PathSeg\n /// [`path_elements()`]: Shape::path_elements\n fn path_segments(&self, tolerance: f64) -> Segments> {\n segments(self.path_elements(tolerance))\n }\n\n /// Signed area.\n ///\n /// This method only produces meaningful results with closed shapes.\n ///\n /// The convention for positive area is that y increases when x is\n /// positive. Thus, it is clockwise when down is increasing y (the\n /// usual convention for graphics), and anticlockwise when\n /// up is increasing y (the usual convention for math).\n fn area(&self) -> f64;\n\n /// Total length of perimeter.\n //FIXME: document the accuracy param\n fn perimeter(&self, accuracy: f64) -> f64;\n\n /// The [winding number] of a point.\n ///\n /// This method only produces meaningful results with closed shapes.\n ///\n /// The sign of the winding number is consistent with that of [`area`],\n /// meaning it is +1 when the point is inside a positive area shape\n /// and -1 when it is inside a negative area shape. Of course, greater\n /// magnitude values are also possible when the shape is more complex.\n ///\n /// [`area`]: Shape::area\n /// [winding number]: https://mathworld.wolfram.com/ContourWindingNumber.html\n fn winding(&self, pt: Point) -> i32;\n\n /// Returns `true` if the [`Point`] is inside this shape.\n ///\n /// This is only meaningful for closed shapes.\n fn contains(&self, pt: Point) -> bool {\n self.winding(pt) != 0\n }\n\n /// The smallest rectangle that encloses the shape.\n fn bounding_box(&self) -> Rect;\n\n /// If the shape is a line, make it available.\n fn as_line(&self) -> Option {\n None\n }\n\n /// If the shape is a rectangle, make it available.\n fn as_rect(&self) -> Option {\n None\n }\n\n /// If the shape is a rounded rectangle, make it available.\n fn as_rounded_rect(&self) -> Option {\n None\n }\n\n /// If the shape is a circle, make it available.\n fn as_circle(&self) -> Option {\n None\n }\n\n /// If the shape is stored as a slice of path elements, make\n /// that available.\n ///\n /// Note: when GAT's land, a method like `path_elements` would be\n /// able to iterate through the slice with no extra allocation,\n /// without making any assumption that storage is contiguous.\n fn as_path_slice(&self) -> Option<&[PathEl]> {\n None\n }\n}\n\n/// Blanket implementation so `impl Shape` will accept owned or reference.\nimpl<'a, T: Shape> Shape for &'a T {\n type PathElementsIter<'iter>\n\n = T::PathElementsIter<'iter> where T: 'iter, 'a: 'iter;\n\n fn path_elements(&self, tolerance: f64) -> Self::PathElementsIter<'_> {\n (*self).path_elements(tolerance)\n }\n\n fn to_path(&self, tolerance: f64) -> BezPath {\n (*self).to_path(tolerance)\n }\n\n fn path_segments(&self, tolerance: f64) -> Segments> {\n (*self).path_segments(tolerance)\n }\n\n fn area(&self) -> f64 {\n (*self).area()\n }\n\n fn perimeter(&self, accuracy: f64) -> f64 {\n (*self).perimeter(accuracy)\n }\n\n fn winding(&self, pt: Point) -> i32 {\n (*self).winding(pt)\n }\n\n fn bounding_box(&self) -> Rect {\n (*self).bounding_box()\n }\n\n fn as_line(&self) -> Option {\n (*self).as_line()\n }\n\n fn as_rect(&self) -> Option {\n (*self).as_rect()\n }\n\n fn as_rounded_rect(&self) -> Option {\n (*self).as_rounded_rect()\n }\n\n fn as_circle(&self) -> Option {\n (*self).as_circle()\n }\n\n fn as_path_slice(&self) -> Option<&[PathEl]> {\n (*self).as_path_slice()\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/simplify.rs", "sha256": "0de5659f1102dd22ff312e7ba1cefa5e1e9fc030df71ff619ca5c4ec94282cd1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2022 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Simplification of a Bézier path.\n//!\n//! This module is currently experimental.\n//!\n//! The methods in this module create a `SimplifyBezPath` object, which can then\n//! be fed to [`fit_to_bezpath`] or [`fit_to_bezpath_opt`] depending on the degree\n//! of optimization desired.\n//!\n//! The implementation uses a number of techniques to achieve high performance and\n//! accuracy. The parameter (generally written `t`) evenly divides the curve segments\n//! in the original, so sampling can be done in constant time. The derivatives are\n//! computed analytically, as that is straightforward with Béziers.\n//!\n//! The areas and moments are computed analytically (using Green's theorem), and\n//! the prefix sum is stored. Thus, it is possible to analytically compute the area\n//! and moment of any subdivision of the curve, also in constant time, by taking\n//! the difference of two stored prefix sum values, then fixing up the subsegments.\n//!\n//! A current limitation (hoped to be addressed in the future) is that non-regular\n//! cubic segments may have tangents computed incorrectly. This can easily happen,\n//! for example when setting a control point equal to an endpoint.\n//!\n//! In addition, this method does not report corners (adjoining segments where the\n//! tangents are not continuous). It is not clear whether it's best to handle such\n//! cases here, or in client code.\n//!\n//! [`fit_to_bezpath`]: crate::fit_to_bezpath\n//! [`fit_to_bezpath_opt`]: crate::fit_to_bezpath_opt\n\nuse alloc::vec::Vec;\n\nuse core::ops::Range;\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\nuse crate::{\n fit_to_bezpath, fit_to_bezpath_opt, BezPath, CubicBez, CurveFitSample, Line, ParamCurve,\n ParamCurveDeriv, ParamCurveFit, PathEl, PathSeg, Point, QuadBez, Vec2,\n};\n\n/// A Bézier path which has been prepared for simplification.\n///\n/// See the [module-level documentation] for a bit more discussion of the approach,\n/// and how this struct is to be used.\n///\n/// [module-level documentation]: crate::simplify\npub struct SimplifyBezPath(Vec);\n\nstruct SimplifyCubic {\n c: CubicBez,\n // The inclusive prefix sum of the moment integrals\n moments: (f64, f64, f64),\n}\n\n/// Options for simplifying paths.\npub struct SimplifyOptions {\n /// The tangent of the minimum angle below which the path is considered smooth.\n angle_thresh: f64,\n opt_level: SimplifyOptLevel,\n}\n\n/// Optimization level for simplification.\npub enum SimplifyOptLevel {\n /// Subdivide; faster but not as optimized results.\n Subdivide,\n /// Optimize subdivision points.\n Optimize,\n}\n\nimpl Default for SimplifyOptions {\n fn default() -> Self {\n let opt_level = SimplifyOptLevel::Subdivide;\n SimplifyOptions {\n angle_thresh: 1e-3,\n opt_level,\n }\n }\n}\n\n#[doc(hidden)]\n/// Compute moment integrals.\n///\n/// This is exposed for testing purposes but is an internal detail. We can\n/// add to the public, documented interface if there is a use case.\npub fn moment_integrals(c: CubicBez) -> (f64, f64, f64) {\n let (x0, y0) = (c.p0.x, c.p0.y);\n let (x1, y1) = (c.p1.x - x0, c.p1.y - y0);\n let (x2, y2) = (c.p2.x - x0, c.p2.y - y0);\n let (x3, y3) = (c.p3.x - x0, c.p3.y - y0);\n\n let r0 = 3. * x1;\n let r1 = 3. * y1;\n let r2 = x2 * y3;\n let r3 = x3 * y2;\n let r4 = x3 * y3;\n let r5 = 27. * y1;\n let r6 = x1 * x2;\n let r7 = 27. * y2;\n let r8 = 45. * r2;\n let r9 = 18. * x3;\n let r10 = x1 * y1;\n let r11 = 30. * x1;\n let r12 = 45. * x3;\n let r13 = x2 * y1;\n let r14 = 45. * r3;\n let r15 = x1.powi(2);\n let r16 = 18. * y3;\n let r17 = x2.powi(2);\n let r18 = 45. * y3;\n let r19 = x3.powi(2);\n let r20 = 30. * y1;\n let r21 = y2.powi(2);\n let r22 = y3.powi(2);\n let r23 = y1.powi(2);\n let a = -r0 * y2 - r0 * y3 + r1 * x2 + r1 * x3 - 6. * r2 + 6. * r3 + 10. * r4;\n\n // Scale and add chord\n let lift = x3 * y0;\n let area = a * 0.05 + lift;\n let x = r10 * r9 - r11 * r4 + r12 * r13 + r14 * x2 - r15 * r16 - r15 * r7 - r17 * r18\n + r17 * r5\n + r19 * r20\n + 105. * r19 * y2\n + 280. * r19 * y3\n - 105. * r2 * x3\n + r5 * r6\n - r6 * r7\n - r8 * x1;\n let y = -r10 * r16 - r10 * r7 - r11 * r22 + r12 * r21 + r13 * r7 + r14 * y1 - r18 * x1 * y2\n + r20 * r4\n - 27. * r21 * x1\n - 105. * r22 * x2\n + 140. * r22 * x3\n + r23 * r9\n + 27. * r23 * x2\n + 105. * r3 * y3\n - r8 * y2;\n\n let mx = x * (1. / 840.) + x0 * area + 0.5 * x3 * lift;\n let my = y * (1. / 420.) + y0 * a * 0.1 + y0 * lift;\n\n (area, mx, my)\n}\n\nimpl SimplifyBezPath {\n /// Set up a new Bézier path for simplification.\n ///\n /// Currently this is not dealing with discontinuities at all, but it\n /// could be extended to do so.\n pub fn new(path: impl IntoIterator) -> Self {\n let (mut a, mut x, mut y) = (0.0, 0.0, 0.0);\n let els = crate::segments(path)\n .map(|seg| {\n let c = seg.to_cubic();\n let (ai, xi, yi) = moment_integrals(c);\n a += ai;\n x += xi;\n y += yi;\n SimplifyCubic {\n c,\n moments: (a, x, y),\n }\n })\n .collect();\n SimplifyBezPath(els)\n }\n\n /// Resolve a `t` value to a cubic.\n ///\n /// Also return the resulting `t` value for the selected cubic.\n fn scale(&self, t: f64) -> (usize, f64) {\n let t_scale = t * self.0.len() as f64;\n let t_floor = t_scale.floor();\n (t_floor as usize, t_scale - t_floor)\n }\n\n fn moment_integrals(&self, i: usize, range: Range) -> (f64, f64, f64) {\n if range.end == range.start {\n (0.0, 0.0, 0.0)\n } else {\n moment_integrals(self.0[i].c.subsegment(range))\n }\n }\n}\n\nimpl ParamCurveFit for SimplifyBezPath {\n fn sample_pt_deriv(&self, t: f64) -> (Point, Vec2) {\n let (mut i, mut t0) = self.scale(t);\n let n = self.0.len();\n if i == n {\n i -= 1;\n t0 = 1.0;\n }\n let c = self.0[i].c;\n (c.eval(t0), c.deriv().eval(t0).to_vec2() * n as f64)\n }\n\n fn sample_pt_tangent(&self, t: f64, _: f64) -> CurveFitSample {\n let (mut i, mut t0) = self.scale(t);\n if i == self.0.len() {\n i -= 1;\n t0 = 1.0;\n }\n let c = self.0[i].c;\n let p = c.eval(t0);\n let tangent = c.deriv().eval(t0).to_vec2();\n CurveFitSample { p, tangent }\n }\n\n // We could use the default implementation, but provide our own, mostly\n // because it is possible to efficiently provide an analytically accurate\n // answer.\n fn moment_integrals(&self, range: Range) -> (f64, f64, f64) {\n let (i0, t0) = self.scale(range.start);\n let (i1, t1) = self.scale(range.end);\n if i0 == i1 {\n self.moment_integrals(i0, t0..t1)\n } else {\n let (a0, x0, y0) = self.moment_integrals(i0, t0..1.0);\n let (a1, x1, y1) = self.moment_integrals(i1, 0.0..t1);\n let (mut a, mut x, mut y) = (a0 + a1, x0 + x1, y0 + y1);\n if i1 > i0 + 1 {\n let (a2, x2, y2) = self.0[i0].moments;\n let (a3, x3, y3) = self.0[i1 - 1].moments;\n a += a3 - a2;\n x += x3 - x2;\n y += y3 - y2;\n }\n (a, x, y)\n }\n }\n\n fn break_cusp(&self, _: Range) -> Option {\n None\n }\n}\n\n#[derive(Default)]\nstruct SimplifyState {\n queue: BezPath,\n result: BezPath,\n needs_moveto: bool,\n}\n\nimpl SimplifyState {\n fn add_seg(&mut self, seg: PathSeg) {\n if self.queue.is_empty() {\n self.queue.move_to(seg.start());\n }\n match seg {\n PathSeg::Quad(q) => self.queue.quad_to(q.p1, q.p2),\n PathSeg::Cubic(c) => self.queue.curve_to(c.p1, c.p2, c.p3),\n _ => unreachable!(),\n }\n }\n\n fn flush(&mut self, accuracy: f64, options: &SimplifyOptions) {\n if self.queue.is_empty() {\n return;\n }\n // TODO: if queue is one segment, just output that\n let s = SimplifyBezPath::new(&self.queue);\n let b = match options.opt_level {\n SimplifyOptLevel::Subdivide => fit_to_bezpath(&s, accuracy),\n SimplifyOptLevel::Optimize => fit_to_bezpath_opt(&s, accuracy),\n };\n self.result\n .extend(b.iter().skip(!self.needs_moveto as usize));\n self.needs_moveto = false;\n self.queue.truncate(0);\n }\n}\n\n/// Simplify a Bézier path.\n///\n/// This function simplifies an arbitrary Bézier path; it is designed to handle\n/// multiple subpaths and also corners.\npub fn simplify_bezpath(\n path: impl IntoIterator,\n accuracy: f64,\n options: &SimplifyOptions,\n) -> BezPath {\n let mut last_pt = None;\n let mut last_seg: Option = None;\n let mut state = SimplifyState::default();\n for el in path {\n let mut this_seg = None;\n match el {\n PathEl::MoveTo(p) => {\n state.flush(accuracy, options);\n state.needs_moveto = true;\n last_pt = Some(p);\n }\n PathEl::LineTo(p) => {\n let last = last_pt.unwrap();\n if last == p {\n continue;\n }\n state.flush(accuracy, options);\n this_seg = Some(PathSeg::Line(Line::new(last, p)));\n }\n PathEl::QuadTo(p1, p2) => {\n let last = last_pt.unwrap();\n if last == p1 && last == p2 {\n continue;\n }\n this_seg = Some(PathSeg::Quad(QuadBez::new(last, p1, p2)));\n }\n PathEl::CurveTo(p1, p2, p3) => {\n let last = last_pt.unwrap();\n if last == p1 && last == p2 && last == p3 {\n continue;\n }\n this_seg = Some(PathSeg::Cubic(CubicBez::new(last, p1, p2, p3)));\n }\n PathEl::ClosePath => {\n state.flush(accuracy, options);\n state.result.close_path();\n state.needs_moveto = true;\n last_seg = None;\n continue;\n }\n }\n if let Some(seg) = this_seg {\n if let Some(last) = last_seg {\n let last_tan = last.tangents().1;\n let this_tan = seg.tangents().0;\n if last_tan.cross(this_tan).abs()\n > last_tan.dot(this_tan).abs() * options.angle_thresh\n {\n state.flush(accuracy, options);\n }\n }\n last_pt = Some(seg.end());\n state.add_seg(seg);\n }\n last_seg = this_seg;\n }\n state.flush(accuracy, options);\n state.result\n}\n\nimpl SimplifyOptions {\n /// Set optimization level.\n pub fn opt_level(mut self, level: SimplifyOptLevel) -> Self {\n self.opt_level = level;\n self\n }\n\n /// Set angle threshold.\n ///\n /// The tangent of the angle below which joins are considered smooth and\n /// not corners. The default is approximately 1 milliradian.\n pub fn angle_thresh(mut self, thresh: f64) -> Self {\n self.angle_thresh = thresh;\n self\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/size.rs", "sha256": "f4a47d98e75782f200e40e968a176406b4438f67d87ae3c9256a511a0500b3e0", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A 2D size.\n\nuse core::fmt;\nuse core::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Sub, SubAssign};\n\nuse crate::common::FloatExt;\nuse crate::{Rect, RoundedRect, RoundedRectRadii, Vec2};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// A 2D size.\n#[derive(Clone, Copy, Default, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct Size {\n /// The width.\n pub width: f64,\n /// The height.\n pub height: f64,\n}\n\nimpl Size {\n /// A size with zero width or height.\n pub const ZERO: Size = Size::new(0., 0.);\n\n /// Create a new `Size` with the provided `width` and `height`.\n #[inline]\n pub const fn new(width: f64, height: f64) -> Self {\n Size { width, height }\n }\n\n /// Returns the max of `width` and `height`.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Size;\n /// let size = Size::new(-10.5, 42.0);\n /// assert_eq!(size.max_side(), 42.0);\n /// ```\n pub fn max_side(self) -> f64 {\n self.width.max(self.height)\n }\n\n /// Returns the min of `width` and `height`.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Size;\n /// let size = Size::new(-10.5, 42.0);\n /// assert_eq!(size.min_side(), -10.5);\n /// ```\n pub fn min_side(self) -> f64 {\n self.width.min(self.height)\n }\n\n /// The area covered by this size.\n #[inline]\n pub fn area(self) -> f64 {\n self.width * self.height\n }\n\n /// Whether this size has zero area.\n ///\n /// Note: a size with negative area is not considered empty.\n #[inline]\n pub fn is_empty(self) -> bool {\n self.area() == 0.0\n }\n\n /// Returns a new size bounded by `min` and `max.`\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Size;\n ///\n /// let this = Size::new(0., 100.);\n /// let min = Size::new(10., 10.,);\n /// let max = Size::new(50., 50.);\n /// assert_eq!(this.clamp(min, max), Size::new(10., 50.))\n /// ```\n pub fn clamp(self, min: Size, max: Size) -> Self {\n let width = self.width.max(min.width).min(max.width);\n let height = self.height.max(min.height).min(max.height);\n Size { width, height }\n }\n\n /// Convert this size into a [`Vec2`], with `width` mapped to `x` and `height`\n /// mapped to `y`.\n #[inline]\n pub const fn to_vec2(self) -> Vec2 {\n Vec2::new(self.width, self.height)\n }\n\n /// Returns a new `Size`,\n /// with `width` and `height` rounded to the nearest integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Size;\n /// let size_pos = Size::new(3.3, 3.6).round();\n /// assert_eq!(size_pos.width, 3.0);\n /// assert_eq!(size_pos.height, 4.0);\n /// let size_neg = Size::new(-3.3, -3.6).round();\n /// assert_eq!(size_neg.width, -3.0);\n /// assert_eq!(size_neg.height, -4.0);\n /// ```\n #[inline]\n pub fn round(self) -> Size {\n Size::new(self.width.round(), self.height.round())\n }\n\n /// Returns a new `Size`,\n /// with `width` and `height` rounded up to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Size;\n /// let size_pos = Size::new(3.3, 3.6).ceil();\n /// assert_eq!(size_pos.width, 4.0);\n /// assert_eq!(size_pos.height, 4.0);\n /// let size_neg = Size::new(-3.3, -3.6).ceil();\n /// assert_eq!(size_neg.width, -3.0);\n /// assert_eq!(size_neg.height, -3.0);\n /// ```\n #[inline]\n pub fn ceil(self) -> Size {\n Size::new(self.width.ceil(), self.height.ceil())\n }\n\n /// Returns a new `Size`,\n /// with `width` and `height` rounded down to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Size;\n /// let size_pos = Size::new(3.3, 3.6).floor();\n /// assert_eq!(size_pos.width, 3.0);\n /// assert_eq!(size_pos.height, 3.0);\n /// let size_neg = Size::new(-3.3, -3.6).floor();\n /// assert_eq!(size_neg.width, -4.0);\n /// assert_eq!(size_neg.height, -4.0);\n /// ```\n #[inline]\n pub fn floor(self) -> Size {\n Size::new(self.width.floor(), self.height.floor())\n }\n\n /// Returns a new `Size`,\n /// with `width` and `height` rounded away from zero to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Size;\n /// let size_pos = Size::new(3.3, 3.6).expand();\n /// assert_eq!(size_pos.width, 4.0);\n /// assert_eq!(size_pos.height, 4.0);\n /// let size_neg = Size::new(-3.3, -3.6).expand();\n /// assert_eq!(size_neg.width, -4.0);\n /// assert_eq!(size_neg.height, -4.0);\n /// ```\n #[inline]\n pub fn expand(self) -> Size {\n Size::new(self.width.expand(), self.height.expand())\n }\n\n /// Returns a new `Size`,\n /// with `width` and `height` rounded down towards zero the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Size;\n /// let size_pos = Size::new(3.3, 3.6).trunc();\n /// assert_eq!(size_pos.width, 3.0);\n /// assert_eq!(size_pos.height, 3.0);\n /// let size_neg = Size::new(-3.3, -3.6).trunc();\n /// assert_eq!(size_neg.width, -3.0);\n /// assert_eq!(size_neg.height, -3.0);\n /// ```\n #[inline]\n pub fn trunc(self) -> Size {\n Size::new(self.width.trunc(), self.height.trunc())\n }\n\n /// Returns the aspect ratio of a rectangle with the given size.\n ///\n /// If the width is `0`, the output will be `sign(self.height) * infinity`. If The width and\n /// height are `0`, then the output will be `NaN`.\n pub fn aspect_ratio(self) -> f64 {\n self.height / self.width\n }\n\n /// Convert this `Size` into a [`Rect`] with origin `(0.0, 0.0)`.\n #[inline]\n pub const fn to_rect(self) -> Rect {\n Rect::new(0., 0., self.width, self.height)\n }\n\n /// Convert this `Size` into a [`RoundedRect`] with origin `(0.0, 0.0)` and\n /// the provided corner radius.\n #[inline]\n pub fn to_rounded_rect(self, radii: impl Into) -> RoundedRect {\n self.to_rect().to_rounded_rect(radii)\n }\n\n /// Is this size finite?\n #[inline]\n pub fn is_finite(self) -> bool {\n self.width.is_finite() && self.height.is_finite()\n }\n\n /// Is this size NaN?\n #[inline]\n pub fn is_nan(self) -> bool {\n self.width.is_nan() || self.height.is_nan()\n }\n}\n\nimpl fmt::Debug for Size {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n write!(f, \"{:?}W×{:?}H\", self.width, self.height)\n }\n}\n\nimpl fmt::Display for Size {\n fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n write!(formatter, \"(\")?;\n fmt::Display::fmt(&self.width, formatter)?;\n write!(formatter, \"×\")?;\n fmt::Display::fmt(&self.height, formatter)?;\n write!(formatter, \")\")\n }\n}\n\nimpl MulAssign for Size {\n #[inline]\n fn mul_assign(&mut self, other: f64) {\n *self = Size {\n width: self.width * other,\n height: self.height * other,\n };\n }\n}\n\nimpl Mul for f64 {\n type Output = Size;\n\n #[inline]\n fn mul(self, other: Size) -> Size {\n other * self\n }\n}\n\nimpl Mul for Size {\n type Output = Size;\n\n #[inline]\n fn mul(self, other: f64) -> Size {\n Size {\n width: self.width * other,\n height: self.height * other,\n }\n }\n}\n\nimpl DivAssign for Size {\n #[inline]\n fn div_assign(&mut self, other: f64) {\n *self = Size {\n width: self.width / other,\n height: self.height / other,\n };\n }\n}\n\nimpl Div for Size {\n type Output = Size;\n\n #[inline]\n fn div(self, other: f64) -> Size {\n Size {\n width: self.width / other,\n height: self.height / other,\n }\n }\n}\n\nimpl Add for Size {\n type Output = Size;\n #[inline]\n fn add(self, other: Size) -> Size {\n Size {\n width: self.width + other.width,\n height: self.height + other.height,\n }\n }\n}\n\nimpl AddAssign for Size {\n #[inline]\n fn add_assign(&mut self, other: Size) {\n *self = *self + other;\n }\n}\n\nimpl Sub for Size {\n type Output = Size;\n #[inline]\n fn sub(self, other: Size) -> Size {\n Size {\n width: self.width - other.width,\n height: self.height - other.height,\n }\n }\n}\n\nimpl SubAssign for Size {\n #[inline]\n fn sub_assign(&mut self, other: Size) {\n *self = *self - other;\n }\n}\n\nimpl From<(f64, f64)> for Size {\n #[inline]\n fn from(v: (f64, f64)) -> Size {\n Size {\n width: v.0,\n height: v.1,\n }\n }\n}\n\nimpl From for (f64, f64) {\n #[inline]\n fn from(v: Size) -> (f64, f64) {\n (v.width, v.height)\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn display() {\n let s = Size::new(-0.12345, 9.87654);\n assert_eq!(format!(\"{s}\"), \"(-0.12345×9.87654)\");\n\n let s = Size::new(-0.12345, 9.87654);\n assert_eq!(format!(\"{s:+6.2}\"), \"( -0.12× +9.88)\");\n }\n\n #[test]\n fn aspect_ratio() {\n let s = Size::new(1.0, 1.0);\n assert!((s.aspect_ratio() - 1.0).abs() < 1e-6);\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/stroke.rs", "sha256": "90c99b353f122b5f745f62d54cd77b9904d8ab7c26b8545d734de05b664c4576", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2023 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse core::{borrow::Borrow, f64::consts::PI};\n\nuse alloc::vec::Vec;\n\nuse smallvec::SmallVec;\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\nuse crate::{\n common::solve_quadratic, fit_to_bezpath, fit_to_bezpath_opt, offset::CubicOffset, Affine, Arc,\n BezPath, CubicBez, Line, ParamCurve, ParamCurveArclen, PathEl, PathSeg, Point, QuadBez, Vec2,\n};\n\n/// Defines the connection between two segments of a stroke.\n#[derive(Copy, Clone, PartialEq, Eq, Debug)]\npub enum Join {\n /// A straight line connecting the segments.\n Bevel,\n /// The segments are extended to their natural intersection point.\n Miter,\n /// An arc between the segments.\n Round,\n}\n\n/// Defines the shape to be drawn at the ends of a stroke.\n#[derive(Copy, Clone, PartialEq, Eq, Debug)]\npub enum Cap {\n /// Flat cap.\n Butt,\n /// Square cap with dimensions equal to half the stroke width.\n Square,\n /// Rounded cap with radius equal to half the stroke width.\n Round,\n}\n\n/// Describes the visual style of a stroke.\n#[derive(Clone, Debug)]\npub struct Stroke {\n /// Width of the stroke.\n pub width: f64,\n /// Style for connecting segments of the stroke.\n pub join: Join,\n /// Limit for miter joins.\n pub miter_limit: f64,\n /// Style for capping the beginning of an open subpath.\n pub start_cap: Cap,\n /// Style for capping the end of an open subpath.\n pub end_cap: Cap,\n /// Lengths of dashes in alternating on/off order.\n pub dash_pattern: Dashes,\n /// Offset of the first dash.\n pub dash_offset: f64,\n}\n\n/// Options for path stroking.\npub struct StrokeOpts {\n opt_level: StrokeOptLevel,\n}\n\n/// Optimization level for computing\npub enum StrokeOptLevel {\n /// Adaptively subdivide segments in half.\n Subdivide,\n /// Compute optimized subdivision points to minimize error.\n Optimized,\n}\n\nimpl Default for StrokeOpts {\n fn default() -> Self {\n let opt_level = StrokeOptLevel::Subdivide;\n StrokeOpts { opt_level }\n }\n}\n\nimpl Default for Stroke {\n fn default() -> Self {\n Self {\n width: 1.0,\n join: Join::Round,\n miter_limit: 4.0,\n start_cap: Cap::Round,\n end_cap: Cap::Round,\n dash_pattern: Default::default(),\n dash_offset: 0.0,\n }\n }\n}\n\nimpl Stroke {\n /// Creates a new stroke with the specified width.\n pub fn new(width: f64) -> Self {\n Self {\n width,\n ..Default::default()\n }\n }\n\n /// Builder method for setting the join style.\n pub fn with_join(mut self, join: Join) -> Self {\n self.join = join;\n self\n }\n\n /// Builder method for setting the limit for miter joins.\n pub fn with_miter_limit(mut self, limit: f64) -> Self {\n self.miter_limit = limit;\n self\n }\n\n /// Builder method for setting the cap style for the start of the stroke.\n pub fn with_start_cap(mut self, cap: Cap) -> Self {\n self.start_cap = cap;\n self\n }\n\n /// Builder method for setting the cap style for the end of the stroke.\n pub fn with_end_cap(mut self, cap: Cap) -> Self {\n self.end_cap = cap;\n self\n }\n\n /// Builder method for setting the cap style.\n pub fn with_caps(mut self, cap: Cap) -> Self {\n self.start_cap = cap;\n self.end_cap = cap;\n self\n }\n\n /// Builder method for setting the dashing parameters.\n pub fn with_dashes

(mut self, offset: f64, pattern: P) -> Self\n where\n P: IntoIterator,\n P::Item: Borrow,\n {\n self.dash_offset = offset;\n self.dash_pattern.clear();\n self.dash_pattern\n .extend(pattern.into_iter().map(|dash| *dash.borrow()));\n self\n }\n}\n\nimpl StrokeOpts {\n /// Set optimization level for computing stroke outlines.\n pub fn opt_level(mut self, opt_level: StrokeOptLevel) -> Self {\n self.opt_level = opt_level;\n self\n }\n}\n\n/// Collection of values representing lengths in a dash pattern.\npub type Dashes = SmallVec<[f64; 4]>;\n\n/// Internal structure used for creating strokes.\n#[derive(Default)]\nstruct StrokeCtx {\n // As a possible future optimization, we might not need separate storage\n // for forward and backward paths, we can add forward to the output in-place.\n // However, this structure is clearer and the cost fairly modest.\n output: BezPath,\n forward_path: BezPath,\n backward_path: BezPath,\n start_pt: Point,\n start_norm: Vec2,\n start_tan: Vec2,\n last_pt: Point,\n last_tan: Vec2,\n // Precomputation of the join threshold, to optimize per-join logic.\n // If hypot < (hypot + dot) * join_thresh, omit join altogether.\n join_thresh: f64,\n}\n\n/// Expand a stroke into a fill.\n///\n/// The `tolerance` parameter controls the accuracy of the result. In general,\n/// the number of subdivisions in the output scales to the -1/6 power of the\n/// parameter, for example making it 1/64 as big generates twice as many\n/// segments. The appropriate value depends on the application; if the result\n/// of the stroke will be scaled up, a smaller value is needed.\n///\n/// This method attempts a fairly high degree of correctness, but ultimately\n/// is based on computing parallel curves and adding joins and caps, rather than\n/// computing the rigorously correct parallel sweep (which requires evolutes in\n/// the general case). See [Nehab 2020] for more discussion.\n///\n/// [Nehab 2020]: https://dl.acm.org/doi/10.1145/3386569.3392392\npub fn stroke(\n path: impl IntoIterator,\n style: &Stroke,\n opts: &StrokeOpts,\n tolerance: f64,\n) -> BezPath {\n if style.dash_pattern.is_empty() {\n stroke_undashed(path, style, tolerance, opts)\n } else {\n let dashed = dash(path.into_iter(), style.dash_offset, &style.dash_pattern);\n stroke_undashed(dashed, style, tolerance, opts)\n }\n}\n\n/// Version of stroke expansion for styles with no dashes.\nfn stroke_undashed(\n path: impl IntoIterator,\n style: &Stroke,\n tolerance: f64,\n opts: &StrokeOpts,\n) -> BezPath {\n let mut ctx = StrokeCtx {\n join_thresh: 2.0 * tolerance / style.width,\n ..Default::default()\n };\n for el in path {\n let p0 = ctx.last_pt;\n match el {\n PathEl::MoveTo(p) => {\n ctx.finish(style);\n ctx.start_pt = p;\n ctx.last_pt = p;\n }\n PathEl::LineTo(p1) => {\n if p1 != p0 {\n let tangent = p1 - p0;\n ctx.do_join(style, tangent);\n ctx.last_tan = tangent;\n ctx.do_line(style, tangent, p1);\n }\n }\n PathEl::QuadTo(p1, p2) => {\n if p1 != p0 || p2 != p0 {\n let q = QuadBez::new(p0, p1, p2);\n let (tan0, tan1) = PathSeg::Quad(q).tangents();\n ctx.do_join(style, tan0);\n ctx.do_cubic(style, q.raise(), tolerance, opts);\n ctx.last_tan = tan1;\n }\n }\n PathEl::CurveTo(p1, p2, p3) => {\n if p1 != p0 || p2 != p0 || p3 != p0 {\n let c = CubicBez::new(p0, p1, p2, p3);\n let (tan0, tan1) = PathSeg::Cubic(c).tangents();\n ctx.do_join(style, tan0);\n ctx.do_cubic(style, c, tolerance, opts);\n ctx.last_tan = tan1;\n }\n }\n PathEl::ClosePath => {\n if p0 != ctx.start_pt {\n let tangent = ctx.start_pt - p0;\n ctx.do_join(style, tangent);\n ctx.last_tan = tangent;\n ctx.do_line(style, tangent, ctx.start_pt);\n }\n ctx.finish_closed(style);\n }\n }\n }\n ctx.finish(style);\n ctx.output\n}\n\nfn round_cap(out: &mut BezPath, tolerance: f64, center: Point, norm: Vec2) {\n round_join(out, tolerance, center, norm, PI);\n}\n\nfn round_join(out: &mut BezPath, tolerance: f64, center: Point, norm: Vec2, angle: f64) {\n let a = Affine::new([norm.x, norm.y, -norm.y, norm.x, center.x, center.y]);\n let arc = Arc::new(Point::ORIGIN, (1.0, 1.0), PI - angle, angle, 0.0);\n arc.to_cubic_beziers(tolerance, |p1, p2, p3| out.curve_to(a * p1, a * p2, a * p3));\n}\n\nfn round_join_rev(out: &mut BezPath, tolerance: f64, center: Point, norm: Vec2, angle: f64) {\n let a = Affine::new([norm.x, norm.y, norm.y, -norm.x, center.x, center.y]);\n let arc = Arc::new(Point::ORIGIN, (1.0, 1.0), PI - angle, angle, 0.0);\n arc.to_cubic_beziers(tolerance, |p1, p2, p3| out.curve_to(a * p1, a * p2, a * p3));\n}\n\nfn square_cap(out: &mut BezPath, close: bool, center: Point, norm: Vec2) {\n let a = Affine::new([norm.x, norm.y, -norm.y, norm.x, center.x, center.y]);\n out.line_to(a * Point::new(1.0, 1.0));\n out.line_to(a * Point::new(-1.0, 1.0));\n if close {\n out.close_path();\n } else {\n out.line_to(a * Point::new(-1.0, 0.0));\n }\n}\n\nfn extend_reversed(out: &mut BezPath, elements: &[PathEl]) {\n for i in (1..elements.len()).rev() {\n let end = elements[i - 1].end_point().unwrap();\n match elements[i] {\n PathEl::LineTo(_) => out.line_to(end),\n PathEl::QuadTo(p1, _) => out.quad_to(p1, end),\n PathEl::CurveTo(p1, p2, _) => out.curve_to(p2, p1, end),\n _ => unreachable!(),\n }\n }\n}\n\nfn fit_with_opts(co: &CubicOffset, tolerance: f64, opts: &StrokeOpts) -> BezPath {\n match opts.opt_level {\n StrokeOptLevel::Subdivide => fit_to_bezpath(co, tolerance),\n StrokeOptLevel::Optimized => fit_to_bezpath_opt(co, tolerance),\n }\n}\n\nimpl StrokeCtx {\n /// Append forward and backward paths to output.\n fn finish(&mut self, style: &Stroke) {\n // TODO: scale\n let tolerance = 1e-3;\n if self.forward_path.is_empty() {\n return;\n }\n self.output.extend(&self.forward_path);\n let back_els = self.backward_path.elements();\n let return_p = back_els[back_els.len() - 1].end_point().unwrap();\n let d = self.last_pt - return_p;\n match style.end_cap {\n Cap::Butt => self.output.line_to(return_p),\n Cap::Round => round_cap(&mut self.output, tolerance, self.last_pt, d),\n Cap::Square => square_cap(&mut self.output, false, self.last_pt, d),\n }\n extend_reversed(&mut self.output, back_els);\n match style.start_cap {\n Cap::Butt => self.output.close_path(),\n Cap::Round => round_cap(&mut self.output, tolerance, self.start_pt, self.start_norm),\n Cap::Square => square_cap(&mut self.output, true, self.start_pt, self.start_norm),\n }\n\n self.forward_path.truncate(0);\n self.backward_path.truncate(0);\n }\n\n /// Finish a closed path\n fn finish_closed(&mut self, style: &Stroke) {\n self.do_join(style, self.start_tan);\n self.output.extend(&self.forward_path);\n self.output.close_path();\n let back_els = self.backward_path.elements();\n let last_pt = back_els[back_els.len() - 1].end_point().unwrap();\n self.output.move_to(last_pt);\n extend_reversed(&mut self.output, back_els);\n self.output.close_path();\n self.forward_path.truncate(0);\n self.backward_path.truncate(0);\n }\n\n fn do_join(&mut self, style: &Stroke, tan0: Vec2) {\n // TODO: scale\n let tolerance = 1e-3;\n let scale = 0.5 * style.width / tan0.hypot();\n let norm = scale * Vec2::new(-tan0.y, tan0.x);\n let p0 = self.last_pt;\n if self.forward_path.elements().is_empty() {\n self.forward_path.move_to(p0 - norm);\n self.backward_path.move_to(p0 + norm);\n self.start_tan = tan0;\n self.start_norm = norm;\n } else {\n let ab = self.last_tan;\n let cd = tan0;\n let cross = ab.cross(cd);\n let dot = ab.dot(cd);\n let hypot = cross.hypot(dot);\n // possible TODO: a minor speedup could be squaring both sides\n if dot <= 0.0 || cross.abs() >= hypot * self.join_thresh {\n match style.join {\n Join::Bevel => {\n self.forward_path.line_to(p0 - norm);\n self.backward_path.line_to(p0 + norm);\n }\n Join::Miter => {\n if 2.0 * hypot < (hypot + dot) * style.miter_limit.powi(2) {\n // TODO: maybe better to store last_norm or derive from path?\n let last_scale = 0.5 * style.width / ab.hypot();\n let last_norm = last_scale * Vec2::new(-ab.y, ab.x);\n if cross > 0.0 {\n let fp_last = p0 - last_norm;\n let fp_this = p0 - norm;\n let h = ab.cross(fp_this - fp_last) / cross;\n let miter_pt = fp_this - cd * h;\n self.forward_path.line_to(miter_pt);\n } else if cross < 0.0 {\n let fp_last = p0 + last_norm;\n let fp_this = p0 + norm;\n let h = ab.cross(fp_this - fp_last) / cross;\n let miter_pt = fp_this - cd * h;\n self.backward_path.line_to(miter_pt);\n }\n }\n self.forward_path.line_to(p0 - norm);\n self.backward_path.line_to(p0 + norm);\n }\n Join::Round => {\n let angle = cross.atan2(dot);\n if angle > 0.0 {\n self.backward_path.line_to(p0 + norm);\n round_join(&mut self.forward_path, tolerance, p0, norm, angle);\n } else {\n self.forward_path.line_to(p0 - norm);\n round_join_rev(&mut self.backward_path, tolerance, p0, -norm, -angle);\n }\n }\n }\n }\n }\n }\n\n fn do_line(&mut self, style: &Stroke, tangent: Vec2, p1: Point) {\n let scale = 0.5 * style.width / tangent.hypot();\n let norm = scale * Vec2::new(-tangent.y, tangent.x);\n self.forward_path.line_to(p1 - norm);\n self.backward_path.line_to(p1 + norm);\n self.last_pt = p1;\n }\n\n fn do_cubic(&mut self, style: &Stroke, c: CubicBez, tolerance: f64, opts: &StrokeOpts) {\n // First, detect degenerate linear case\n\n // Ordinarily, this is the direction of the chord, but if the chord is very\n // short, we take the longer control arm.\n let chord = c.p3 - c.p0;\n let mut chord_ref = chord;\n let mut chord_ref_hypot2 = chord_ref.hypot2();\n let d01 = c.p1 - c.p0;\n if d01.hypot2() > chord_ref_hypot2 {\n chord_ref = d01;\n chord_ref_hypot2 = chord_ref.hypot2();\n }\n let d23 = c.p3 - c.p2;\n if d23.hypot2() > chord_ref_hypot2 {\n chord_ref = d23;\n chord_ref_hypot2 = chord_ref.hypot2();\n }\n // Project Bézier onto chord\n let p0 = c.p0.to_vec2().dot(chord_ref);\n let p1 = c.p1.to_vec2().dot(chord_ref);\n let p2 = c.p2.to_vec2().dot(chord_ref);\n let p3 = c.p3.to_vec2().dot(chord_ref);\n const ENDPOINT_D: f64 = 0.01;\n if p3 <= p0\n || p1 > p2\n || p1 < p0 + ENDPOINT_D * (p3 - p0)\n || p2 > p3 - ENDPOINT_D * (p3 - p0)\n {\n // potentially a cusp inside\n let x01 = d01.cross(chord_ref);\n let x23 = d23.cross(chord_ref);\n let x03 = chord.cross(chord_ref);\n let thresh = tolerance.powi(2) * chord_ref_hypot2;\n if x01 * x01 < thresh && x23 * x23 < thresh && x03 * x03 < thresh {\n // control points are nearly co-linear\n let midpoint = c.p0.midpoint(c.p3);\n // Mapping back from projection of reference chord\n let ref_vec = chord_ref / chord_ref_hypot2;\n let ref_pt = midpoint - 0.5 * (p0 + p3) * ref_vec;\n self.do_linear(style, c, [p0, p1, p2, p3], ref_pt, ref_vec);\n return;\n }\n }\n\n // A tuning parameter for regularization. A value too large may distort the curve,\n // while a value too small may fail to generate smooth curves. This is a somewhat\n // arbitrary value, and should be revisited.\n const DIM_TUNE: f64 = 0.25;\n let dimension = tolerance * DIM_TUNE;\n let co = CubicOffset::new_regularized(c, -0.5 * style.width, dimension);\n let forward = fit_with_opts(&co, tolerance, opts);\n self.forward_path.extend(forward.into_iter().skip(1));\n let co = CubicOffset::new_regularized(c, 0.5 * style.width, dimension);\n let backward = fit_with_opts(&co, tolerance, opts);\n self.backward_path.extend(backward.into_iter().skip(1));\n self.last_pt = c.p3;\n }\n\n /// Do a cubic which is actually linear.\n ///\n /// The `p` argument is the control points projected to the reference chord.\n /// The ref arguments are the inverse map of a projection back to the client\n /// coordinate space.\n fn do_linear(\n &mut self,\n style: &Stroke,\n c: CubicBez,\n p: [f64; 4],\n ref_pt: Point,\n ref_vec: Vec2,\n ) {\n // Always do round join, to model cusp as limit of finite curvature (see Nehab).\n let style = Stroke::new(style.width).with_join(Join::Round);\n // Tangents of endpoints (for connecting to joins)\n let (tan0, tan1) = PathSeg::Cubic(c).tangents();\n self.last_tan = tan0;\n // find cusps\n let c0 = p[1] - p[0];\n let c1 = 2.0 * p[2] - 4.0 * p[1] + 2.0 * p[0];\n let c2 = p[3] - 3.0 * p[2] + 3.0 * p[1] - p[0];\n let roots = solve_quadratic(c0, c1, c2);\n // discard cusps right at endpoints\n const EPSILON: f64 = 1e-12;\n for t in roots {\n if t > EPSILON && t < 1.0 - EPSILON {\n let mt = 1.0 - t;\n let z = mt * (mt * mt * p[0] + 3.0 * t * (mt * p[1] + t * p[2])) + t * t * t * p[3];\n let p = ref_pt + z * ref_vec;\n let tan = p - self.last_pt;\n self.do_join(&style, tan);\n self.do_line(&style, tan, p);\n self.last_tan = tan;\n }\n }\n let tan = c.p3 - self.last_pt;\n self.do_join(&style, tan);\n self.do_line(&style, tan, c.p3);\n self.last_tan = tan;\n self.do_join(&style, tan1);\n }\n}\n\n/// An implementation of dashing as an iterator-to-iterator transformation.\n#[doc(hidden)]\npub struct DashIterator<'a, T> {\n inner: T,\n input_done: bool,\n closepath_pending: bool,\n dashes: &'a [f64],\n dash_ix: usize,\n init_dash_ix: usize,\n init_dash_remaining: f64,\n init_is_active: bool,\n is_active: bool,\n state: DashState,\n current_seg: PathSeg,\n t: f64,\n dash_remaining: f64,\n seg_remaining: f64,\n start_pt: Point,\n last_pt: Point,\n stash: Vec,\n stash_ix: usize,\n}\n\n#[derive(PartialEq, Eq)]\nenum DashState {\n NeedInput,\n ToStash,\n Working,\n FromStash,\n}\n\nimpl<'a, T: Iterator> Iterator for DashIterator<'a, T> {\n type Item = PathEl;\n\n fn next(&mut self) -> Option {\n loop {\n match self.state {\n DashState::NeedInput => {\n if self.input_done {\n return None;\n }\n self.get_input();\n if self.input_done {\n return None;\n }\n self.state = DashState::ToStash;\n }\n DashState::ToStash => {\n if let Some(el) = self.step() {\n self.stash.push(el);\n }\n }\n DashState::Working => {\n if let Some(el) = self.step() {\n return Some(el);\n }\n }\n DashState::FromStash => {\n if let Some(el) = self.stash.get(self.stash_ix) {\n self.stash_ix += 1;\n return Some(*el);\n } else {\n self.stash.clear();\n self.stash_ix = 0;\n if self.input_done {\n return None;\n }\n if self.closepath_pending {\n self.closepath_pending = false;\n self.state = DashState::NeedInput;\n } else {\n self.state = DashState::ToStash;\n }\n }\n }\n }\n }\n }\n}\n\nfn seg_to_el(el: &PathSeg) -> PathEl {\n match el {\n PathSeg::Line(l) => PathEl::LineTo(l.p1),\n PathSeg::Quad(q) => PathEl::QuadTo(q.p1, q.p2),\n PathSeg::Cubic(c) => PathEl::CurveTo(c.p1, c.p2, c.p3),\n }\n}\n\nconst DASH_ACCURACY: f64 = 1e-6;\n\n/// Create a new dashing iterator.\n///\n/// Handling of dashes is fairly orthogonal to stroke expansion. This iterator\n/// is an internal detail of the stroke expansion logic, but is also available\n/// separately, and is expected to be useful when doing stroke expansion on\n/// GPU.\n///\n/// It is implemented as an iterator-to-iterator transform. Because it consumes\n/// the input sequentially and produces consistent output with correct joins,\n/// it requires internal state and may allocate.\n///\n/// Accuracy is currently hard-coded to 1e-6. This is better than generally\n/// expected, and care is taken to get cusps correct, among other things.\npub fn dash<'a>(\n inner: impl Iterator + 'a,\n dash_offset: f64,\n dashes: &'a [f64],\n) -> impl Iterator + 'a {\n dash_impl(inner, dash_offset, dashes)\n}\n\n// This is only a separate function to make `DashIterator::new()` typecheck.\nfn dash_impl>(\n inner: T,\n dash_offset: f64,\n dashes: &[f64],\n) -> DashIterator {\n let mut dash_ix = 0;\n let mut dash_remaining = dash_offset;\n let mut is_active = true;\n // Find place in dashes array for initial offset.\n while dash_remaining > 0.0 {\n let dash_len = dashes[dash_ix];\n if dash_remaining < dash_len {\n break;\n }\n dash_remaining -= dash_len;\n dash_ix = (dash_ix + 1) % dashes.len();\n is_active = !is_active;\n }\n DashIterator {\n inner,\n input_done: false,\n closepath_pending: false,\n dashes,\n dash_ix,\n init_dash_ix: dash_ix,\n init_dash_remaining: dash_remaining,\n init_is_active: is_active,\n is_active,\n state: DashState::NeedInput,\n current_seg: PathSeg::Line(Line::new(Point::ORIGIN, Point::ORIGIN)),\n t: 0.0,\n dash_remaining,\n seg_remaining: 0.0,\n start_pt: Point::ORIGIN,\n last_pt: Point::ORIGIN,\n stash: Vec::new(),\n stash_ix: 0,\n }\n}\n\nimpl<'a, T: Iterator> DashIterator<'a, T> {\n #[doc(hidden)]\n #[deprecated(since = \"0.10.4\", note = \"use dash() instead\")]\n pub fn new(inner: T, dash_offset: f64, dashes: &'a [f64]) -> Self {\n dash_impl(inner, dash_offset, dashes)\n }\n\n fn get_input(&mut self) {\n loop {\n if self.closepath_pending {\n self.handle_closepath();\n break;\n }\n let Some(next_el) = self.inner.next() else {\n self.input_done = true;\n self.state = DashState::FromStash;\n return;\n };\n let p0 = self.last_pt;\n match next_el {\n PathEl::MoveTo(p) => {\n if !self.stash.is_empty() {\n self.state = DashState::FromStash;\n }\n self.start_pt = p;\n self.last_pt = p;\n self.reset_phase();\n continue;\n }\n PathEl::LineTo(p1) => {\n let l = Line::new(p0, p1);\n self.seg_remaining = l.arclen(DASH_ACCURACY);\n self.current_seg = PathSeg::Line(l);\n self.last_pt = p1;\n }\n PathEl::QuadTo(p1, p2) => {\n let q = QuadBez::new(p0, p1, p2);\n self.seg_remaining = q.arclen(DASH_ACCURACY);\n self.current_seg = PathSeg::Quad(q);\n self.last_pt = p2;\n }\n PathEl::CurveTo(p1, p2, p3) => {\n let c = CubicBez::new(p0, p1, p2, p3);\n self.seg_remaining = c.arclen(DASH_ACCURACY);\n self.current_seg = PathSeg::Cubic(c);\n self.last_pt = p3;\n }\n PathEl::ClosePath => {\n self.closepath_pending = true;\n if p0 != self.start_pt {\n let l = Line::new(p0, self.start_pt);\n self.seg_remaining = l.arclen(DASH_ACCURACY);\n self.current_seg = PathSeg::Line(l);\n self.last_pt = self.start_pt;\n } else {\n self.handle_closepath();\n }\n }\n }\n break;\n }\n self.t = 0.0;\n }\n\n /// Move arc length forward to next event.\n fn step(&mut self) -> Option {\n let mut result = None;\n if self.state == DashState::ToStash && self.stash.is_empty() {\n if self.is_active {\n result = Some(PathEl::MoveTo(self.current_seg.start()));\n } else {\n self.state = DashState::Working;\n }\n } else if self.dash_remaining < self.seg_remaining {\n // next transition is a dash transition\n let seg = self.current_seg.subsegment(self.t..1.0);\n let t1 = seg.inv_arclen(self.dash_remaining, DASH_ACCURACY);\n if self.is_active {\n let subseg = seg.subsegment(0.0..t1);\n result = Some(seg_to_el(&subseg));\n self.state = DashState::Working;\n } else {\n let p = seg.eval(t1);\n result = Some(PathEl::MoveTo(p));\n }\n self.is_active = !self.is_active;\n self.t += t1 * (1.0 - self.t);\n self.seg_remaining -= self.dash_remaining;\n self.dash_ix += 1;\n if self.dash_ix == self.dashes.len() {\n self.dash_ix = 0;\n }\n self.dash_remaining = self.dashes[self.dash_ix];\n } else {\n if self.is_active {\n let seg = self.current_seg.subsegment(self.t..1.0);\n result = Some(seg_to_el(&seg));\n }\n self.dash_remaining -= self.seg_remaining;\n self.get_input();\n }\n result\n }\n\n fn handle_closepath(&mut self) {\n if self.state == DashState::ToStash {\n // Have looped back without breaking a dash, just play it back\n self.stash.push(PathEl::ClosePath);\n } else if self.is_active {\n // connect with path in stash, skip MoveTo.\n self.stash_ix = 1;\n }\n self.state = DashState::FromStash;\n self.reset_phase();\n }\n\n fn reset_phase(&mut self) {\n self.dash_ix = self.init_dash_ix;\n self.dash_remaining = self.init_dash_remaining;\n self.is_active = self.init_is_active;\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{stroke, Cap::Butt, CubicBez, Join::Miter, Shape, Stroke};\n\n // A degenerate stroke with a cusp at the endpoint.\n #[test]\n fn pathological_stroke() {\n let curve = CubicBez::new(\n (602.469, 286.585),\n (641.975, 286.585),\n (562.963, 286.585),\n (562.963, 286.585),\n );\n let path = curve.into_path(0.1);\n let stroke_style = Stroke::new(1.);\n let stroked = stroke(path, &stroke_style, &Default::default(), 0.001);\n assert!(stroked.is_finite());\n }\n\n // Test cases adapted from https://github.com/linebender/vello/pull/388\n #[test]\n fn broken_strokes() {\n let broken_cubics = [\n [(0., -0.01), (128., 128.001), (128., -0.01), (0., 128.001)], // Near-cusp\n [(0., 0.), (0., -10.), (0., -10.), (0., 10.)], // Flat line with 180\n [(10., 0.), (0., 0.), (20., 0.), (10., 0.)], // Flat line with 2 180s\n [(39., -39.), (40., -40.), (40., -40.), (0., 0.)], // Flat diagonal with 180\n [(40., 40.), (0., 0.), (200., 200.), (0., 0.)], // Diag w/ an internal 180\n [(0., 0.), (1e-2, 0.), (-1e-2, 0.), (0., 0.)], // Circle\n // Flat line with no turns:\n [\n (400.75, 100.05),\n (400.75, 100.05),\n (100.05, 300.95),\n (100.05, 300.95),\n ],\n [(0.5, 0.), (0., 0.), (20., 0.), (10., 0.)], // Flat line with 2 180s\n [(10., 0.), (0., 0.), (10., 0.), (10., 0.)], // Flat line with a 180\n ];\n let stroke_style = Stroke::new(30.).with_caps(Butt).with_join(Miter);\n for cubic in &broken_cubics {\n let path = CubicBez::new(cubic[0], cubic[1], cubic[2], cubic[3]).into_path(0.1);\n let stroked = stroke(path, &stroke_style, &Default::default(), 0.001);\n assert!(stroked.is_finite());\n }\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/svg.rs", "sha256": "1fa0588849bbfe1c3919e5c1c4e046b66ab5a01ea7a5032154c11ba07d07363a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! SVG path representation.\n\nuse std::error::Error;\nuse std::f64::consts::PI;\nuse std::fmt::{self, Display, Formatter};\nuse std::io::{self, Write};\n\nuse crate::{Arc, BezPath, ParamCurve, PathEl, PathSeg, Point, Vec2};\n\n// Note: the SVG arc logic is heavily adapted from https://github.com/nical/lyon\n\n/// A single SVG arc segment.\n#[derive(Clone, Copy, Debug)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct SvgArc {\n /// The arc's start point.\n pub from: Point,\n /// The arc's end point.\n pub to: Point,\n /// The arc's radii, where the vector's x-component is the radius in the\n /// positive x direction after applying `x_rotation`.\n pub radii: Vec2,\n /// How much the arc is rotated, in radians.\n pub x_rotation: f64,\n /// Does this arc sweep through more than π radians?\n pub large_arc: bool,\n /// Determines if the arc should begin moving at positive angles.\n pub sweep: bool,\n}\n\nimpl BezPath {\n /// Create a `BezPath` with segments corresponding to the sequence of\n /// `PathSeg`s\n pub fn from_path_segments(segments: impl Iterator) -> BezPath {\n let mut path_elements = Vec::new();\n let mut current_pos = None;\n\n for segment in segments {\n let start = segment.start();\n if Some(start) != current_pos {\n path_elements.push(PathEl::MoveTo(start));\n };\n path_elements.push(match segment {\n PathSeg::Line(l) => PathEl::LineTo(l.p1),\n PathSeg::Quad(q) => PathEl::QuadTo(q.p1, q.p2),\n PathSeg::Cubic(c) => PathEl::CurveTo(c.p1, c.p2, c.p3),\n });\n\n current_pos = Some(segment.end());\n }\n\n BezPath::from_vec(path_elements)\n }\n\n /// Convert the path to an SVG path string representation.\n ///\n /// The current implementation doesn't take any special care to produce a\n /// short string (reducing precision, using relative movement).\n pub fn to_svg(&self) -> String {\n let mut buffer = Vec::new();\n self.write_to(&mut buffer).unwrap();\n String::from_utf8(buffer).unwrap()\n }\n\n /// Write the SVG representation of this path to the provided buffer.\n pub fn write_to(&self, mut writer: W) -> io::Result<()> {\n for (i, el) in self.elements().iter().enumerate() {\n if i > 0 {\n write!(writer, \" \")?;\n }\n match *el {\n PathEl::MoveTo(p) => write!(writer, \"M{},{}\", p.x, p.y)?,\n PathEl::LineTo(p) => write!(writer, \"L{},{}\", p.x, p.y)?,\n PathEl::QuadTo(p1, p2) => write!(writer, \"Q{},{} {},{}\", p1.x, p1.y, p2.x, p2.y)?,\n PathEl::CurveTo(p1, p2, p3) => write!(\n writer,\n \"C{},{} {},{} {},{}\",\n p1.x, p1.y, p2.x, p2.y, p3.x, p3.y\n )?,\n PathEl::ClosePath => write!(writer, \"Z\")?,\n }\n }\n\n Ok(())\n }\n\n /// Try to parse a bezier path from an SVG path element.\n ///\n /// This is implemented on a best-effort basis, intended for cases where the\n /// user controls the source of paths, and is not intended as a replacement\n /// for a general, robust SVG parser.\n pub fn from_svg(data: &str) -> Result {\n let mut lexer = SvgLexer::new(data);\n let mut path = BezPath::new();\n let mut last_cmd = 0;\n let mut last_ctrl = None;\n let mut first_pt = Point::ORIGIN;\n let mut implicit_moveto = None;\n while let Some(c) = lexer.get_cmd(last_cmd) {\n if c != b'm' && c != b'M' {\n if let Some(pt) = implicit_moveto.take() {\n path.move_to(pt);\n }\n }\n match c {\n b'm' | b'M' => {\n implicit_moveto = None;\n let pt = lexer.get_maybe_relative(c)?;\n path.move_to(pt);\n lexer.last_pt = pt;\n first_pt = pt;\n last_ctrl = Some(pt);\n last_cmd = c - (b'M' - b'L');\n }\n b'l' | b'L' => {\n let pt = lexer.get_maybe_relative(c)?;\n path.line_to(pt);\n lexer.last_pt = pt;\n last_ctrl = Some(pt);\n last_cmd = c;\n }\n b'h' | b'H' => {\n let mut x = lexer.get_number()?;\n lexer.opt_comma();\n if c == b'h' {\n x += lexer.last_pt.x;\n }\n let pt = Point::new(x, lexer.last_pt.y);\n path.line_to(pt);\n lexer.last_pt = pt;\n last_ctrl = Some(pt);\n last_cmd = c;\n }\n b'v' | b'V' => {\n let mut y = lexer.get_number()?;\n lexer.opt_comma();\n if c == b'v' {\n y += lexer.last_pt.y;\n }\n let pt = Point::new(lexer.last_pt.x, y);\n path.line_to(pt);\n lexer.last_pt = pt;\n last_ctrl = Some(pt);\n last_cmd = c;\n }\n b'q' | b'Q' => {\n let p1 = lexer.get_maybe_relative(c)?;\n let p2 = lexer.get_maybe_relative(c)?;\n path.quad_to(p1, p2);\n last_ctrl = Some(p1);\n lexer.last_pt = p2;\n last_cmd = c;\n }\n b't' | b'T' => {\n let p1 = match last_ctrl {\n Some(ctrl) => (2.0 * lexer.last_pt.to_vec2() - ctrl.to_vec2()).to_point(),\n None => lexer.last_pt,\n };\n let p2 = lexer.get_maybe_relative(c)?;\n path.quad_to(p1, p2);\n last_ctrl = Some(p1);\n lexer.last_pt = p2;\n last_cmd = c;\n }\n b'c' | b'C' => {\n let p1 = lexer.get_maybe_relative(c)?;\n let p2 = lexer.get_maybe_relative(c)?;\n let p3 = lexer.get_maybe_relative(c)?;\n path.curve_to(p1, p2, p3);\n last_ctrl = Some(p2);\n lexer.last_pt = p3;\n last_cmd = c;\n }\n b's' | b'S' => {\n let p1 = match last_ctrl {\n Some(ctrl) => (2.0 * lexer.last_pt.to_vec2() - ctrl.to_vec2()).to_point(),\n None => lexer.last_pt,\n };\n let p2 = lexer.get_maybe_relative(c)?;\n let p3 = lexer.get_maybe_relative(c)?;\n path.curve_to(p1, p2, p3);\n last_ctrl = Some(p2);\n lexer.last_pt = p3;\n last_cmd = c;\n }\n b'a' | b'A' => {\n let radii = lexer.get_number_pair()?;\n let x_rotation = lexer.get_number()?.to_radians();\n lexer.opt_comma();\n let large_arc = lexer.get_flag()?;\n lexer.opt_comma();\n let sweep = lexer.get_flag()?;\n lexer.opt_comma();\n let p = lexer.get_maybe_relative(c)?;\n let svg_arc = SvgArc {\n from: lexer.last_pt,\n to: p,\n radii: radii.to_vec2(),\n x_rotation,\n large_arc,\n sweep,\n };\n\n match Arc::from_svg_arc(&svg_arc) {\n Some(arc) => {\n // TODO: consider making tolerance configurable\n arc.to_cubic_beziers(0.1, |p1, p2, p3| {\n path.curve_to(p1, p2, p3);\n });\n }\n None => {\n path.line_to(p);\n }\n }\n\n last_ctrl = Some(p);\n lexer.last_pt = p;\n last_cmd = c;\n }\n b'z' | b'Z' => {\n path.close_path();\n lexer.last_pt = first_pt;\n implicit_moveto = Some(first_pt);\n }\n _ => return Err(SvgParseError::UnknownCommand(c as char)),\n }\n }\n Ok(path)\n }\n}\n\n/// An error which can be returned when parsing an SVG.\n#[derive(Debug)]\npub enum SvgParseError {\n /// A number was expected.\n Wrong,\n /// The input string ended while still expecting input.\n UnexpectedEof,\n /// Encountered an unknown command letter.\n UnknownCommand(char),\n}\n\nimpl Display for SvgParseError {\n fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n match self {\n SvgParseError::Wrong => write!(f, \"Unable to parse a number\"),\n SvgParseError::UnexpectedEof => write!(f, \"Unexpected EOF\"),\n SvgParseError::UnknownCommand(letter) => write!(f, \"Unknown command, \\\"{letter}\\\"\"),\n }\n }\n}\n\nimpl Error for SvgParseError {}\n\nstruct SvgLexer<'a> {\n data: &'a str,\n ix: usize,\n pub last_pt: Point,\n}\n\nimpl<'a> SvgLexer<'a> {\n fn new(data: &str) -> SvgLexer {\n SvgLexer {\n data,\n ix: 0,\n last_pt: Point::ORIGIN,\n }\n }\n\n fn skip_ws(&mut self) {\n while let Some(&c) = self.data.as_bytes().get(self.ix) {\n if !(c == b' ' || c == 9 || c == 10 || c == 12 || c == 13) {\n break;\n }\n self.ix += 1;\n }\n }\n\n fn get_cmd(&mut self, last_cmd: u8) -> Option {\n self.skip_ws();\n if let Some(c) = self.get_byte() {\n if c.is_ascii_lowercase() || c.is_ascii_uppercase() {\n return Some(c);\n } else if last_cmd != 0 && (c == b'-' || c == b'.' || c.is_ascii_digit()) {\n // Plausible number start\n self.unget();\n return Some(last_cmd);\n } else {\n self.unget();\n }\n }\n None\n }\n\n fn get_byte(&mut self) -> Option {\n self.data.as_bytes().get(self.ix).map(|&c| {\n self.ix += 1;\n c\n })\n }\n\n fn unget(&mut self) {\n self.ix -= 1;\n }\n\n fn get_number(&mut self) -> Result {\n self.skip_ws();\n let start = self.ix;\n let c = self.get_byte().ok_or(SvgParseError::UnexpectedEof)?;\n if !(c == b'-' || c == b'+') {\n self.unget();\n }\n let mut digit_count = 0;\n let mut seen_period = false;\n while let Some(c) = self.get_byte() {\n if c.is_ascii_digit() {\n digit_count += 1;\n } else if c == b'.' && !seen_period {\n seen_period = true;\n } else {\n self.unget();\n break;\n }\n }\n if let Some(c) = self.get_byte() {\n if c == b'e' || c == b'E' {\n let mut c = self.get_byte().ok_or(SvgParseError::Wrong)?;\n if c == b'-' || c == b'+' {\n c = self.get_byte().ok_or(SvgParseError::Wrong)?\n }\n if c.is_ascii_digit() {\n return Err(SvgParseError::Wrong);\n }\n while let Some(c) = self.get_byte() {\n if c.is_ascii_digit() {\n self.unget();\n break;\n }\n }\n } else {\n self.unget();\n }\n }\n if digit_count > 0 {\n self.data[start..self.ix]\n .parse()\n .map_err(|_| SvgParseError::Wrong)\n } else {\n Err(SvgParseError::Wrong)\n }\n }\n\n fn get_flag(&mut self) -> Result {\n self.skip_ws();\n match self.get_byte().ok_or(SvgParseError::UnexpectedEof)? {\n b'0' => Ok(false),\n b'1' => Ok(true),\n _ => Err(SvgParseError::Wrong),\n }\n }\n\n fn get_number_pair(&mut self) -> Result {\n let x = self.get_number()?;\n self.opt_comma();\n let y = self.get_number()?;\n self.opt_comma();\n Ok(Point::new(x, y))\n }\n\n fn get_maybe_relative(&mut self, cmd: u8) -> Result {\n let pt = self.get_number_pair()?;\n if cmd.is_ascii_lowercase() {\n Ok(self.last_pt + pt.to_vec2())\n } else {\n Ok(pt)\n }\n }\n\n fn opt_comma(&mut self) {\n self.skip_ws();\n if let Some(c) = self.get_byte() {\n if c != b',' {\n self.unget();\n }\n }\n }\n}\n\nimpl SvgArc {\n /// Checks that arc is actually a straight line.\n ///\n /// In this case, it can be replaced with a `LineTo`.\n pub fn is_straight_line(&self) -> bool {\n self.radii.x.abs() <= 1e-5 || self.radii.y.abs() <= 1e-5 || self.from == self.to\n }\n}\n\nimpl Arc {\n /// Creates an `Arc` from a `SvgArc`.\n ///\n /// Returns `None` if `arc` is actually a straight line.\n pub fn from_svg_arc(arc: &SvgArc) -> Option {\n // Have to check this first, otherwise `sum_of_sq` will be 0.\n if arc.is_straight_line() {\n return None;\n }\n\n let mut rx = arc.radii.x.abs();\n let mut ry = arc.radii.y.abs();\n\n let xr = arc.x_rotation % (2.0 * PI);\n let (sin_phi, cos_phi) = xr.sin_cos();\n let hd_x = (arc.from.x - arc.to.x) * 0.5;\n let hd_y = (arc.from.y - arc.to.y) * 0.5;\n let hs_x = (arc.from.x + arc.to.x) * 0.5;\n let hs_y = (arc.from.y + arc.to.y) * 0.5;\n\n // F6.5.1\n let p = Vec2::new(\n cos_phi * hd_x + sin_phi * hd_y,\n -sin_phi * hd_x + cos_phi * hd_y,\n );\n\n // Sanitize the radii.\n // If rf > 1 it means the radii are too small for the arc to\n // possibly connect the end points. In this situation we scale\n // them up according to the formula provided by the SVG spec.\n\n // F6.6.2\n let rf = p.x * p.x / (rx * rx) + p.y * p.y / (ry * ry);\n if rf > 1.0 {\n let scale = rf.sqrt();\n rx *= scale;\n ry *= scale;\n }\n\n let rxry = rx * ry;\n let rxpy = rx * p.y;\n let rypx = ry * p.x;\n let sum_of_sq = rxpy * rxpy + rypx * rypx;\n\n debug_assert!(sum_of_sq != 0.0);\n\n // F6.5.2\n let sign_coe = if arc.large_arc == arc.sweep {\n -1.0\n } else {\n 1.0\n };\n let coe = sign_coe * ((rxry * rxry - sum_of_sq) / sum_of_sq).abs().sqrt();\n let transformed_cx = coe * rxpy / ry;\n let transformed_cy = -coe * rypx / rx;\n\n // F6.5.3\n let center = Point::new(\n cos_phi * transformed_cx - sin_phi * transformed_cy + hs_x,\n sin_phi * transformed_cx + cos_phi * transformed_cy + hs_y,\n );\n\n let start_v = Vec2::new((p.x - transformed_cx) / rx, (p.y - transformed_cy) / ry);\n let end_v = Vec2::new((-p.x - transformed_cx) / rx, (-p.y - transformed_cy) / ry);\n\n let start_angle = start_v.atan2();\n\n let mut sweep_angle = (end_v.atan2() - start_angle) % (2.0 * PI);\n\n if arc.sweep && sweep_angle < 0.0 {\n sweep_angle += 2.0 * PI;\n } else if !arc.sweep && sweep_angle > 0.0 {\n sweep_angle -= 2.0 * PI;\n }\n\n Some(Arc {\n center,\n radii: Vec2::new(rx, ry),\n start_angle,\n sweep_angle,\n x_rotation: arc.x_rotation,\n })\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{BezPath, CubicBez, Line, ParamCurve, PathSeg, Point, QuadBez, Shape};\n\n #[test]\n fn test_parse_svg() {\n let path = BezPath::from_svg(\"m10 10 100 0 0 100 -100 0z\").unwrap();\n assert_eq!(path.segments().count(), 4);\n }\n\n #[test]\n fn test_parse_svg2() {\n let path =\n BezPath::from_svg(\"M3.5 8a.5.5 0 01.5-.5h8a.5.5 0 010 1H4a.5.5 0 01-.5-.5z\").unwrap();\n assert_eq!(path.segments().count(), 6);\n }\n\n #[test]\n fn test_parse_svg_arc() {\n let path = BezPath::from_svg(\"M 100 100 A 25 25 0 1 0 -25 25 z\").unwrap();\n assert_eq!(path.segments().count(), 3);\n }\n\n // Regression test for #51\n #[test]\n #[allow(clippy::float_cmp)]\n fn test_parse_svg_arc_pie() {\n let path = BezPath::from_svg(\"M 100 100 h 25 a 25 25 0 1 0 -25 25 z\").unwrap();\n // Approximate figures, but useful for regression testing\n assert_eq!(path.area().round(), -1473.0);\n assert_eq!(path.perimeter(1e-6).round(), 168.0);\n }\n\n #[test]\n fn test_write_svg_single() {\n let segments = [CubicBez::new(\n Point::new(10., 10.),\n Point::new(20., 20.),\n Point::new(30., 30.),\n Point::new(40., 40.),\n )\n .into()];\n let path = BezPath::from_path_segments(segments.iter().cloned());\n\n assert_eq!(path.to_svg(), \"M10,10 C20,20 30,30 40,40\");\n }\n\n #[test]\n fn test_write_svg_two_nomove() {\n let segments = [\n CubicBez::new(\n Point::new(10., 10.),\n Point::new(20., 20.),\n Point::new(30., 30.),\n Point::new(40., 40.),\n )\n .into(),\n CubicBez::new(\n Point::new(40., 40.),\n Point::new(30., 30.),\n Point::new(20., 20.),\n Point::new(10., 10.),\n )\n .into(),\n ];\n let path = BezPath::from_path_segments(segments.iter().cloned());\n\n assert_eq!(\n path.to_svg(),\n \"M10,10 C20,20 30,30 40,40 C30,30 20,20 10,10\"\n );\n }\n\n #[test]\n fn test_write_svg_two_move() {\n let segments = [\n CubicBez::new(\n Point::new(10., 10.),\n Point::new(20., 20.),\n Point::new(30., 30.),\n Point::new(40., 40.),\n )\n .into(),\n CubicBez::new(\n Point::new(50., 50.),\n Point::new(30., 30.),\n Point::new(20., 20.),\n Point::new(10., 10.),\n )\n .into(),\n ];\n let path = BezPath::from_path_segments(segments.iter().cloned());\n\n assert_eq!(\n path.to_svg(),\n \"M10,10 C20,20 30,30 40,40 M50,50 C30,30 20,20 10,10\"\n );\n }\n\n use rand::prelude::*;\n\n fn gen_random_path_sequence(rng: &mut impl Rng) -> Vec {\n const MAX_LENGTH: u32 = 10;\n\n let mut elements = vec![];\n let mut position = None;\n\n let length = rng.gen_range(0..MAX_LENGTH);\n for _ in 0..length {\n let should_follow: bool = rand::random();\n let kind = rng.gen_range(0..3);\n\n let first = position\n .filter(|_| should_follow)\n .unwrap_or_else(|| Point::new(rng.gen(), rng.gen()));\n\n let element: PathSeg = match kind {\n 0 => Line::new(first, Point::new(rng.gen(), rng.gen())).into(),\n\n 1 => QuadBez::new(\n first,\n Point::new(rng.gen(), rng.gen()),\n Point::new(rng.gen(), rng.gen()),\n )\n .into(),\n\n 2 => CubicBez::new(\n first,\n Point::new(rng.gen(), rng.gen()),\n Point::new(rng.gen(), rng.gen()),\n Point::new(rng.gen(), rng.gen()),\n )\n .into(),\n\n _ => unreachable!(),\n };\n\n position = Some(element.end());\n elements.push(element);\n }\n\n elements\n }\n\n #[test]\n fn test_serialize_deserialize() {\n const N_TESTS: u32 = 100;\n let mut rng = rand::thread_rng();\n\n for _ in 0..N_TESTS {\n let vec = gen_random_path_sequence(&mut rng);\n let ser = BezPath::from_path_segments(vec.iter().cloned()).to_svg();\n let deser = BezPath::from_svg(&ser).expect(\"failed deserialization\");\n\n let deser_vec = deser.segments().collect::>();\n\n assert_eq!(vec, deser_vec);\n }\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/translate_scale.rs", "sha256": "8ac6ee46f99ff83656fe185c8182834c34c65544b045788e7004bc64c28a19f1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A transformation that includes both scale and translation.\n\nuse core::ops::{Add, AddAssign, Mul, MulAssign, Sub, SubAssign};\n\nuse crate::{\n Affine, Circle, CubicBez, Line, Point, QuadBez, Rect, RoundedRect, RoundedRectRadii, Vec2,\n};\n\n/// A transformation consisting of a uniform scaling followed by a translation.\n///\n/// If the translation is `(x, y)` and the scale is `s`, then this\n/// transformation represents this augmented matrix:\n///\n/// ```text\n/// | s 0 x |\n/// | 0 s y |\n/// | 0 0 1 |\n/// ```\n///\n/// See [`Affine`] for more details about the\n/// equivalence with augmented matrices.\n///\n/// Various multiplication ops are defined, and these are all defined\n/// to be consistent with matrix multiplication. Therefore,\n/// `TranslateScale * Point` is defined but not the other way around.\n///\n/// Also note that multiplication is not commutative. Thus,\n/// `TranslateScale::scale(2.0) * TranslateScale::translate(Vec2::new(1.0, 0.0))`\n/// has a translation of (2, 0), while\n/// `TranslateScale::translate(Vec2::new(1.0, 0.0)) * TranslateScale::scale(2.0)`\n/// has a translation of (1, 0). (Both have a scale of 2; also note that\n/// the first case can be written\n/// `2.0 * TranslateScale::translate(Vec2::new(1.0, 0.0))` as this case\n/// has an implicit conversion).\n///\n/// This transformation is less powerful than `Affine`, but can be applied\n/// to more primitives, especially including [`Rect`].\n#[derive(Clone, Copy, Debug)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct TranslateScale {\n /// The translation component of this transformation\n pub translation: Vec2,\n /// The scale component of this transformation\n pub scale: f64,\n}\n\nimpl TranslateScale {\n /// Create a new transformation from translation and scale.\n #[inline]\n pub const fn new(translation: Vec2, scale: f64) -> TranslateScale {\n TranslateScale { translation, scale }\n }\n\n /// Create a new transformation with scale only.\n #[inline]\n pub const fn scale(s: f64) -> TranslateScale {\n TranslateScale::new(Vec2::ZERO, s)\n }\n\n /// Create a new transformation with translation only.\n #[inline]\n pub fn translate(translation: impl Into) -> TranslateScale {\n TranslateScale::new(translation.into(), 1.0)\n }\n\n /// Decompose transformation into translation and scale.\n #[deprecated(note = \"use the struct fields directly\")]\n #[inline]\n pub const fn as_tuple(self) -> (Vec2, f64) {\n (self.translation, self.scale)\n }\n\n /// Create a transform that scales about a point other than the origin.\n ///\n /// # Examples\n ///\n /// ```\n /// # use kurbo::{Point, TranslateScale};\n /// # fn assert_near(p0: Point, p1: Point) {\n /// # assert!((p1 - p0).hypot() < 1e-9, \"{p0:?} != {p1:?}\");\n /// # }\n /// let center = Point::new(1., 1.);\n /// let ts = TranslateScale::from_scale_about(2., center);\n /// // Should keep the point (1., 1.) stationary\n /// assert_near(ts * center, center);\n /// // (2., 2.) -> (3., 3.)\n /// assert_near(ts * Point::new(2., 2.), Point::new(3., 3.));\n /// ```\n #[inline]\n pub fn from_scale_about(scale: f64, focus: Point) -> Self {\n // We need to create a transform that is equivalent to translating `focus`\n // to the origin, followed by a normal scale, followed by reversing the translation.\n // We need to find the (translation ∘ scale) that matches this.\n let focus = focus.to_vec2();\n let translation = focus - focus * scale;\n Self::new(translation, scale)\n }\n\n /// Compute the inverse transform.\n ///\n /// Multiplying a transform with its inverse (either on the\n /// left or right) results in the identity transform\n /// (modulo floating point rounding errors).\n ///\n /// Produces NaN values when scale is zero.\n #[inline]\n pub fn inverse(self) -> TranslateScale {\n let scale_recip = self.scale.recip();\n TranslateScale {\n translation: self.translation * -scale_recip,\n scale: scale_recip,\n }\n }\n\n /// Is this translate/scale finite?\n #[inline]\n pub fn is_finite(&self) -> bool {\n self.translation.is_finite() && self.scale.is_finite()\n }\n\n /// Is this translate/scale NaN?\n #[inline]\n pub fn is_nan(&self) -> bool {\n self.translation.is_nan() || self.scale.is_nan()\n }\n}\n\nimpl Default for TranslateScale {\n #[inline]\n fn default() -> TranslateScale {\n TranslateScale::new(Vec2::ZERO, 1.0)\n }\n}\n\nimpl From for Affine {\n fn from(ts: TranslateScale) -> Affine {\n let TranslateScale { translation, scale } = ts;\n Affine::new([scale, 0.0, 0.0, scale, translation.x, translation.y])\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = Point;\n\n #[inline]\n fn mul(self, other: Point) -> Point {\n (self.scale * other.to_vec2()).to_point() + self.translation\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = TranslateScale;\n\n #[inline]\n fn mul(self, other: TranslateScale) -> TranslateScale {\n TranslateScale {\n translation: self.translation + self.scale * other.translation,\n scale: self.scale * other.scale,\n }\n }\n}\n\nimpl MulAssign for TranslateScale {\n #[inline]\n fn mul_assign(&mut self, other: TranslateScale) {\n *self = self.mul(other);\n }\n}\n\nimpl Mul for f64 {\n type Output = TranslateScale;\n\n #[inline]\n fn mul(self, other: TranslateScale) -> TranslateScale {\n TranslateScale {\n translation: other.translation * self,\n scale: other.scale * self,\n }\n }\n}\n\nimpl Add for TranslateScale {\n type Output = TranslateScale;\n\n #[inline]\n fn add(self, other: Vec2) -> TranslateScale {\n TranslateScale {\n translation: self.translation + other,\n scale: self.scale,\n }\n }\n}\n\nimpl Add for Vec2 {\n type Output = TranslateScale;\n\n #[inline]\n fn add(self, other: TranslateScale) -> TranslateScale {\n other + self\n }\n}\n\nimpl AddAssign for TranslateScale {\n #[inline]\n fn add_assign(&mut self, other: Vec2) {\n *self = self.add(other);\n }\n}\n\nimpl Sub for TranslateScale {\n type Output = TranslateScale;\n\n #[inline]\n fn sub(self, other: Vec2) -> TranslateScale {\n TranslateScale {\n translation: self.translation - other,\n scale: self.scale,\n }\n }\n}\n\nimpl SubAssign for TranslateScale {\n #[inline]\n fn sub_assign(&mut self, other: Vec2) {\n *self = self.sub(other);\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = Circle;\n\n #[inline]\n fn mul(self, other: Circle) -> Circle {\n Circle::new(self * other.center, self.scale * other.radius)\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = Line;\n\n #[inline]\n fn mul(self, other: Line) -> Line {\n Line::new(self * other.p0, self * other.p1)\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = Rect;\n\n #[inline]\n fn mul(self, other: Rect) -> Rect {\n let pt0 = self * Point::new(other.x0, other.y0);\n let pt1 = self * Point::new(other.x1, other.y1);\n (pt0, pt1).into()\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = RoundedRect;\n\n #[inline]\n fn mul(self, other: RoundedRect) -> RoundedRect {\n RoundedRect::from_rect(self * other.rect(), self * other.radii())\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = RoundedRectRadii;\n\n #[inline]\n fn mul(self, other: RoundedRectRadii) -> RoundedRectRadii {\n RoundedRectRadii::new(\n self.scale * other.top_left,\n self.scale * other.top_right,\n self.scale * other.bottom_right,\n self.scale * other.bottom_left,\n )\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = QuadBez;\n\n #[inline]\n fn mul(self, other: QuadBez) -> QuadBez {\n QuadBez::new(self * other.p0, self * other.p1, self * other.p2)\n }\n}\n\nimpl Mul for TranslateScale {\n type Output = CubicBez;\n\n #[inline]\n fn mul(self, other: CubicBez) -> CubicBez {\n CubicBez::new(\n self * other.p0,\n self * other.p1,\n self * other.p2,\n self * other.p3,\n )\n }\n}\n\n#[cfg(test)]\nmod tests {\n use crate::{Affine, Point, TranslateScale, Vec2};\n\n fn assert_near(p0: Point, p1: Point) {\n assert!((p1 - p0).hypot() < 1e-9, \"{p0:?} != {p1:?}\");\n }\n\n #[test]\n fn translate_scale() {\n let p = Point::new(3.0, 4.0);\n let ts = TranslateScale::new(Vec2::new(5.0, 6.0), 2.0);\n\n assert_near(ts * p, Point::new(11.0, 14.0));\n }\n\n #[test]\n fn conversions() {\n let p = Point::new(3.0, 4.0);\n let s = 2.0;\n let t = Vec2::new(5.0, 6.0);\n let ts = TranslateScale::new(t, s);\n\n // Test that conversion to affine is consistent.\n let a: Affine = ts.into();\n assert_near(ts * p, a * p);\n\n assert_near((s * p.to_vec2()).to_point(), TranslateScale::scale(s) * p);\n assert_near(p + t, TranslateScale::translate(t) * p);\n }\n\n #[test]\n fn inverse() {\n let p = Point::new(3.0, 4.0);\n let ts = TranslateScale::new(Vec2::new(5.0, 6.0), 2.0);\n\n assert_near(p, (ts * ts.inverse()) * p);\n assert_near(p, (ts.inverse() * ts) * p);\n }\n}\n" }, { "package": "kurbo@0.10.4", "sourcePath": "src/vec2.rs", "sha256": "bff561911d2a24fc4326156b0a1ede05bc5738acb39e57c8e472292b950a22e9", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 the Kurbo Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! A simple 2D vector.\n\nuse core::fmt;\nuse core::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Neg, Sub, SubAssign};\n\nuse crate::common::FloatExt;\nuse crate::{Point, Size};\n\n#[cfg(not(feature = \"std\"))]\nuse crate::common::FloatFuncs;\n\n/// A 2D vector.\n///\n/// This is intended primarily for a vector in the mathematical sense,\n/// but it can be interpreted as a translation, and converted to and\n/// from a point (vector relative to the origin) and size.\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\n#[cfg_attr(feature = \"schemars\", derive(schemars::JsonSchema))]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct Vec2 {\n /// The x-coordinate.\n pub x: f64,\n /// The y-coordinate.\n pub y: f64,\n}\n\nimpl Vec2 {\n /// The vector (0, 0).\n pub const ZERO: Vec2 = Vec2::new(0., 0.);\n\n /// Create a new vector.\n #[inline]\n pub const fn new(x: f64, y: f64) -> Vec2 {\n Vec2 { x, y }\n }\n\n /// Convert this vector into a `Point`.\n #[inline]\n pub const fn to_point(self) -> Point {\n Point::new(self.x, self.y)\n }\n\n /// Convert this vector into a `Size`.\n #[inline]\n pub const fn to_size(self) -> Size {\n Size::new(self.x, self.y)\n }\n\n /// Create a `Vec2` with the same value for x and y\n pub(crate) const fn splat(v: f64) -> Self {\n Vec2 { x: v, y: v }\n }\n\n /// Dot product of two vectors.\n #[inline]\n pub fn dot(self, other: Vec2) -> f64 {\n self.x * other.x + self.y * other.y\n }\n\n /// Cross product of two vectors.\n ///\n /// This is signed so that (0, 1) × (1, 0) = 1.\n #[inline]\n pub fn cross(self, other: Vec2) -> f64 {\n self.x * other.y - self.y * other.x\n }\n\n /// Magnitude of vector.\n ///\n /// This is similar to `self.hypot2().sqrt()` but defers to the platform `hypot` method, which\n /// in general will handle the case where `self.hypot2() > f64::MAX`.\n #[inline]\n pub fn hypot(self) -> f64 {\n self.x.hypot(self.y)\n }\n\n /// Magnitude of vector.\n ///\n /// This is an alias for [`Vec2::hypot`].\n #[inline]\n pub fn length(self) -> f64 {\n self.hypot()\n }\n\n /// Magnitude squared of vector.\n #[inline]\n pub fn hypot2(self) -> f64 {\n self.dot(self)\n }\n\n /// Magnitude squared of vector.\n ///\n /// This is an alias for [`Vec2::hypot2`].\n #[inline]\n pub fn length_squared(self) -> f64 {\n self.hypot2()\n }\n\n /// Find the angle in radians between this vector and the vector `Vec2 { x: 1.0, y: 0.0 }`\n /// in the positive `y` direction.\n ///\n /// If the vector is interpreted as a complex number, this is the argument.\n /// The angle is expressed in radians.\n #[inline]\n pub fn atan2(self) -> f64 {\n self.y.atan2(self.x)\n }\n\n /// Find the angle in radians between this vector and the vector `Vec2 { x: 1.0, y: 0.0 }`\n /// in the positive `y` direction.\n ///\n /// This is an alias for [`Vec2::atan2`].\n #[inline]\n pub fn angle(self) -> f64 {\n self.atan2()\n }\n\n /// A unit vector of the given angle.\n ///\n /// With `th` at zero, the result is the positive X unit vector, and\n /// at π/2, it is the positive Y unit vector. The angle is expressed\n /// in radians.\n ///\n /// Thus, in a Y-down coordinate system (as is common for graphics),\n /// it is a clockwise rotation, and in Y-up (traditional for math), it\n /// is anti-clockwise. This convention is consistent with\n /// [`Affine::rotate`].\n ///\n /// [`Affine::rotate`]: crate::Affine::rotate\n #[inline]\n pub fn from_angle(th: f64) -> Vec2 {\n let (th_sin, th_cos) = th.sin_cos();\n Vec2 {\n x: th_cos,\n y: th_sin,\n }\n }\n\n /// Linearly interpolate between two vectors.\n #[inline]\n pub fn lerp(self, other: Vec2, t: f64) -> Vec2 {\n self + t * (other - self)\n }\n\n /// Returns a vector of magnitude 1.0 with the same angle as `self`; i.e.\n /// a unit/direction vector.\n ///\n /// This produces `NaN` values when the magnitude is `0`.\n #[inline]\n pub fn normalize(self) -> Vec2 {\n self / self.hypot()\n }\n\n /// Returns a new `Vec2`,\n /// with `x` and `y` rounded to the nearest integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Vec2;\n /// let a = Vec2::new(3.3, 3.6).round();\n /// let b = Vec2::new(3.0, -3.1).round();\n /// assert_eq!(a.x, 3.0);\n /// assert_eq!(a.y, 4.0);\n /// assert_eq!(b.x, 3.0);\n /// assert_eq!(b.y, -3.0);\n /// ```\n #[inline]\n pub fn round(self) -> Vec2 {\n Vec2::new(self.x.round(), self.y.round())\n }\n\n /// Returns a new `Vec2`,\n /// with `x` and `y` rounded up to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Vec2;\n /// let a = Vec2::new(3.3, 3.6).ceil();\n /// let b = Vec2::new(3.0, -3.1).ceil();\n /// assert_eq!(a.x, 4.0);\n /// assert_eq!(a.y, 4.0);\n /// assert_eq!(b.x, 3.0);\n /// assert_eq!(b.y, -3.0);\n /// ```\n #[inline]\n pub fn ceil(self) -> Vec2 {\n Vec2::new(self.x.ceil(), self.y.ceil())\n }\n\n /// Returns a new `Vec2`,\n /// with `x` and `y` rounded down to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Vec2;\n /// let a = Vec2::new(3.3, 3.6).floor();\n /// let b = Vec2::new(3.0, -3.1).floor();\n /// assert_eq!(a.x, 3.0);\n /// assert_eq!(a.y, 3.0);\n /// assert_eq!(b.x, 3.0);\n /// assert_eq!(b.y, -4.0);\n /// ```\n #[inline]\n pub fn floor(self) -> Vec2 {\n Vec2::new(self.x.floor(), self.y.floor())\n }\n\n /// Returns a new `Vec2`,\n /// with `x` and `y` rounded away from zero to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Vec2;\n /// let a = Vec2::new(3.3, 3.6).expand();\n /// let b = Vec2::new(3.0, -3.1).expand();\n /// assert_eq!(a.x, 4.0);\n /// assert_eq!(a.y, 4.0);\n /// assert_eq!(b.x, 3.0);\n /// assert_eq!(b.y, -4.0);\n /// ```\n #[inline]\n pub fn expand(self) -> Vec2 {\n Vec2::new(self.x.expand(), self.y.expand())\n }\n\n /// Returns a new `Vec2`,\n /// with `x` and `y` rounded towards zero to the nearest integer,\n /// unless they are already an integer.\n ///\n /// # Examples\n ///\n /// ```\n /// use kurbo::Vec2;\n /// let a = Vec2::new(3.3, 3.6).trunc();\n /// let b = Vec2::new(3.0, -3.1).trunc();\n /// assert_eq!(a.x, 3.0);\n /// assert_eq!(a.y, 3.0);\n /// assert_eq!(b.x, 3.0);\n /// assert_eq!(b.y, -3.0);\n /// ```\n #[inline]\n pub fn trunc(self) -> Vec2 {\n Vec2::new(self.x.trunc(), self.y.trunc())\n }\n\n /// Is this Vec2 finite?\n #[inline]\n pub fn is_finite(self) -> bool {\n self.x.is_finite() && self.y.is_finite()\n }\n\n /// Is this Vec2 NaN?\n #[inline]\n pub fn is_nan(self) -> bool {\n self.x.is_nan() || self.y.is_nan()\n }\n\n /// Divides this Vec2 by a scalar.\n ///\n /// Unlike the division by scalar operator, which multiplies by the\n /// reciprocal for performance, this performs the division\n /// per-component for consistent rounding behavior.\n pub(crate) fn div_exact(self, divisor: f64) -> Vec2 {\n Vec2 {\n x: self.x / divisor,\n y: self.y / divisor,\n }\n }\n}\n\nimpl From<(f64, f64)> for Vec2 {\n #[inline]\n fn from(v: (f64, f64)) -> Vec2 {\n Vec2 { x: v.0, y: v.1 }\n }\n}\n\nimpl From for (f64, f64) {\n #[inline]\n fn from(v: Vec2) -> (f64, f64) {\n (v.x, v.y)\n }\n}\n\nimpl Add for Vec2 {\n type Output = Vec2;\n\n #[inline]\n fn add(self, other: Vec2) -> Vec2 {\n Vec2 {\n x: self.x + other.x,\n y: self.y + other.y,\n }\n }\n}\n\nimpl AddAssign for Vec2 {\n #[inline]\n fn add_assign(&mut self, other: Vec2) {\n *self = Vec2 {\n x: self.x + other.x,\n y: self.y + other.y,\n }\n }\n}\n\nimpl Sub for Vec2 {\n type Output = Vec2;\n\n #[inline]\n fn sub(self, other: Vec2) -> Vec2 {\n Vec2 {\n x: self.x - other.x,\n y: self.y - other.y,\n }\n }\n}\n\nimpl SubAssign for Vec2 {\n #[inline]\n fn sub_assign(&mut self, other: Vec2) {\n *self = Vec2 {\n x: self.x - other.x,\n y: self.y - other.y,\n }\n }\n}\n\nimpl Mul for Vec2 {\n type Output = Vec2;\n\n #[inline]\n fn mul(self, other: f64) -> Vec2 {\n Vec2 {\n x: self.x * other,\n y: self.y * other,\n }\n }\n}\n\nimpl MulAssign for Vec2 {\n #[inline]\n fn mul_assign(&mut self, other: f64) {\n *self = Vec2 {\n x: self.x * other,\n y: self.y * other,\n };\n }\n}\n\nimpl Mul for f64 {\n type Output = Vec2;\n\n #[inline]\n fn mul(self, other: Vec2) -> Vec2 {\n other * self\n }\n}\n\nimpl Div for Vec2 {\n type Output = Vec2;\n\n /// Note: division by a scalar is implemented by multiplying by the reciprocal.\n ///\n /// This is more efficient but has different roundoff behavior than division.\n #[inline]\n #[allow(clippy::suspicious_arithmetic_impl)]\n fn div(self, other: f64) -> Vec2 {\n self * other.recip()\n }\n}\n\nimpl DivAssign for Vec2 {\n #[inline]\n fn div_assign(&mut self, other: f64) {\n self.mul_assign(other.recip());\n }\n}\n\nimpl Neg for Vec2 {\n type Output = Vec2;\n\n #[inline]\n fn neg(self) -> Vec2 {\n Vec2 {\n x: -self.x,\n y: -self.y,\n }\n }\n}\n\nimpl fmt::Display for Vec2 {\n fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n write!(formatter, \"𝐯=(\")?;\n fmt::Display::fmt(&self.x, formatter)?;\n write!(formatter, \", \")?;\n fmt::Display::fmt(&self.y, formatter)?;\n write!(formatter, \")\")\n }\n}\n\n// Conversions to and from mint\n#[cfg(feature = \"mint\")]\nimpl From for mint::Vector2 {\n #[inline]\n fn from(p: Vec2) -> mint::Vector2 {\n mint::Vector2 { x: p.x, y: p.y }\n }\n}\n\n#[cfg(feature = \"mint\")]\nimpl From> for Vec2 {\n #[inline]\n fn from(p: mint::Vector2) -> Vec2 {\n Vec2 { x: p.x, y: p.y }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n #[test]\n fn display() {\n let v = Vec2::new(1.2332421, 532.10721213123);\n let s = format!(\"{v:.2}\");\n assert_eq!(s.as_str(), \"𝐯=(1.23, 532.11)\");\n }\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "LICENSE.txt", "sha256": "a08554fab028af3af047e8588d32d019662c1b1bf07959a195a2f75fe340726b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Copyright 2019 Octavian Oncescu\n\nPermission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE." }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/include/mimalloc-new-delete.h", "sha256": "dd4f25cae53209d45d73f8e6a2b9c219e8fc7434d97f20eba9af1a8b850030fd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2020 Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_NEW_DELETE_H\n#define MIMALLOC_NEW_DELETE_H\n\n// ----------------------------------------------------------------------------\n// This header provides convenient overrides for the new and\n// delete operations in C++.\n//\n// This header should be included in only one source file!\n//\n// On Windows, or when linking dynamically with mimalloc, these\n// can be more performant than the standard new-delete operations.\n// See \n// ---------------------------------------------------------------------------\n#if defined(__cplusplus)\n #include \n #include \n\n #if defined(_MSC_VER) && defined(_Ret_notnull_) && defined(_Post_writable_byte_size_)\n // stay consistent with VCRT definitions\n #define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict _Ret_notnull_ _Post_writable_byte_size_(n)\n #define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict _Ret_maybenull_ _Success_(return != NULL) _Post_writable_byte_size_(n)\n #else\n #define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict\n #define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict\n #endif\n\n void operator delete(void* p) noexcept { mi_free(p); };\n void operator delete[](void* p) noexcept { mi_free(p); };\n\n void operator delete (void* p, const std::nothrow_t&) noexcept { mi_free(p); }\n void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); }\n\n mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }\n mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }\n\n mi_decl_new_nothrow(n) void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }\n mi_decl_new_nothrow(n) void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }\n\n #if (__cplusplus >= 201402L || _MSC_VER >= 1916)\n void operator delete (void* p, std::size_t n) noexcept { mi_free_size(p,n); };\n void operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); };\n #endif\n\n #if (__cplusplus > 201402L || defined(__cpp_aligned_new))\n void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); };\n void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); };\n void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); }\n\n void* operator new (std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); }\n void* operator new[](std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); }\n void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); }\n void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); }\n #endif\n#endif\n\n#endif // MIMALLOC_NEW_DELETE_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/include/mimalloc-override.h", "sha256": "21fcf61c4443341ac6bf6ea528af31dc7267e8e3456fc64bfd07704503032175", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2020 Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_OVERRIDE_H\n#define MIMALLOC_OVERRIDE_H\n\n/* ----------------------------------------------------------------------------\nThis header can be used to statically redirect malloc/free and new/delete\nto the mimalloc variants. This can be useful if one can include this file on\neach source file in a project (but be careful when using external code to\nnot accidentally mix pointers from different allocators).\n-----------------------------------------------------------------------------*/\n\n#include \n\n// Standard C allocation\n#define malloc(n) mi_malloc(n)\n#define calloc(c,n) mi_calloc(c,n)\n#define realloc(p,n) mi_realloc(p,n)\n#define free(p) mi_free(p)\n\n#define strdup(s) mi_strdup(s)\n#define strndup(s,n) mi_strndup(s,n)\n#define realpath(f,n) mi_realpath(f,n)\n\n// Microsoft extensions\n#define _expand(p,n) mi_expand(p,n)\n#define _msize(p) mi_usable_size(p)\n#define _recalloc(p,c,n) mi_recalloc(p,c,n)\n\n#define _strdup(s) mi_strdup(s)\n#define _strndup(s,n) mi_strndup(s,n)\n#define _wcsdup(s) mi_wcsdup(s)\n#define _mbsdup(s) mi_mbsdup(s)\n#define _dupenv_s(buf,n,nm) mi_dupenv_s(buf,n,nm)\n#define _wdupenv_s(buf,n,nm) mi_wdupenv_s(buf,n,nm)\n\n// Various Posix and Unix variants\n#define reallocf(p,n) mi_reallocf(p,n)\n#define malloc_size(p) mi_usable_size(p)\n#define malloc_usable_size(p) mi_usable_size(p)\n#define malloc_good_size(n) mi_malloc_good_size(n)\n#define cfree(p) mi_free(p)\n\n#define valloc(n) mi_valloc(n)\n#define pvalloc(n) mi_pvalloc(n)\n#define reallocarray(p,c,n) mi_reallocarray(p,c,n)\n#define reallocarr(ptrp,c,n) mi_reallocarr(ptrp,c,n)\n#define memalign(a,n) mi_memalign(a,n)\n#define aligned_alloc(a,n) mi_aligned_alloc(a,n)\n#define posix_memalign(p,a,n) mi_posix_memalign(p,a,n)\n#define _posix_memalign(p,a,n) mi_posix_memalign(p,a,n)\n\n// Microsoft aligned variants\n#define _aligned_malloc(n,a) mi_malloc_aligned(n,a)\n#define _aligned_realloc(p,n,a) mi_realloc_aligned(p,n,a)\n#define _aligned_recalloc(p,c,n,a) mi_aligned_recalloc(p,c,n,a)\n#define _aligned_msize(p,a,o) mi_usable_size(p)\n#define _aligned_free(p) mi_free(p)\n#define _aligned_offset_malloc(n,a,o) mi_malloc_aligned_at(n,a,o)\n#define _aligned_offset_realloc(p,n,a,o) mi_realloc_aligned_at(p,n,a,o)\n#define _aligned_offset_recalloc(p,c,n,a,o) mi_recalloc_aligned_at(p,c,n,a,o)\n\n#endif // MIMALLOC_OVERRIDE_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/include/mimalloc-stats.h", "sha256": "95b4e031c2e8ec1e4abff7d904b90ae47f9bef8adcbafeacf911f43fa981cc94", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_STATS_H\n#define MIMALLOC_STATS_H\n\n#include \n#include \n\n#define MI_STAT_VERSION 4 // increased on every backward incompatible change\n\n// count allocation over time\ntypedef struct mi_stat_count_s {\n int64_t total; // total allocated\n int64_t peak; // peak allocation\n int64_t current; // current allocation\n} mi_stat_count_t;\n\n// counters only increase\ntypedef struct mi_stat_counter_s {\n int64_t total; // total count\n} mi_stat_counter_t;\n\n#define MI_STAT_FIELDS() \\\n MI_STAT_COUNT(pages) /* count of mimalloc pages */ \\\n MI_STAT_COUNT(reserved) /* reserved memory bytes */ \\\n MI_STAT_COUNT(committed) /* committed bytes */ \\\n MI_STAT_COUNTER(reset) /* reset bytes */ \\\n MI_STAT_COUNTER(purged) /* purged bytes */ \\\n MI_STAT_COUNT(page_committed) /* committed memory inside pages */ \\\n MI_STAT_COUNT(pages_abandoned) /* abandoned pages count */ \\\n MI_STAT_COUNT(threads) /* number of threads */ \\\n MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \\\n MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \\\n MI_STAT_COUNT(malloc_requested) /* malloc requested bytes */ \\\n \\\n MI_STAT_COUNTER(mmap_calls) \\\n MI_STAT_COUNTER(commit_calls) \\\n MI_STAT_COUNTER(reset_calls) \\\n MI_STAT_COUNTER(purge_calls) \\\n MI_STAT_COUNTER(arena_count) /* number of memory arena's */ \\\n MI_STAT_COUNTER(malloc_normal_count) /* number of blocks <= MI_LARGE_OBJ_SIZE_MAX */ \\\n MI_STAT_COUNTER(malloc_huge_count) /* number of huge bloks */ \\\n MI_STAT_COUNTER(malloc_guarded_count) /* number of allocations with guard pages */ \\\n \\\n /* internal statistics */ \\\n MI_STAT_COUNTER(arena_rollback_count) \\\n MI_STAT_COUNTER(arena_purges) \\\n MI_STAT_COUNTER(pages_extended) /* number of page extensions */ \\\n MI_STAT_COUNTER(pages_retire) /* number of pages that are retired */ \\\n MI_STAT_COUNTER(page_searches) /* total pages searched for a fresh page */ \\\n MI_STAT_COUNTER(page_searches_count) /* searched count for a fresh page */ \\\n /* only on v1 and v2 */ \\\n MI_STAT_COUNT(segments) \\\n MI_STAT_COUNT(segments_abandoned) \\\n MI_STAT_COUNT(segments_cache) \\\n MI_STAT_COUNT(_segments_reserved) \\\n /* only on v3 */ \\\n MI_STAT_COUNTER(pages_reclaim_on_alloc) \\\n MI_STAT_COUNTER(pages_reclaim_on_free) \\\n MI_STAT_COUNTER(pages_reabandon_full) \\\n MI_STAT_COUNTER(pages_unabandon_busy_wait) \\\n\n\n// Define the statistics structure\n#define MI_BIN_HUGE (73U) // see types.h\n#define MI_STAT_COUNT(stat) mi_stat_count_t stat;\n#define MI_STAT_COUNTER(stat) mi_stat_counter_t stat;\n\ntypedef struct mi_stats_s\n{\n size_t size; // size of the mi_stats_t structure \n size_t version; \n\n MI_STAT_FIELDS()\n\n // future extension\n mi_stat_count_t _stat_reserved[4];\n mi_stat_counter_t _stat_counter_reserved[4];\n\n // size segregated statistics\n mi_stat_count_t malloc_bins[MI_BIN_HUGE+1]; // allocation per size bin\n mi_stat_count_t page_bins[MI_BIN_HUGE+1]; // pages allocated per size bin\n} mi_stats_t;\n\n#undef MI_STAT_COUNT\n#undef MI_STAT_COUNTER\n\n// helper\n#define mi_stats_t_decl(name) mi_stats_t name = { 0 }; name.size = sizeof(mi_stats_t); name.version = MI_STAT_VERSION;\n\n// Exported definitions\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nmi_decl_export bool mi_stats_get( mi_stats_t* stats ) mi_attr_noexcept;\nmi_decl_export char* mi_stats_get_json( size_t buf_size, char* buf ) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif // MIMALLOC_STATS_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/include/mimalloc.h", "sha256": "ba52ce5c7708d8d54d4ffee372c5bac0921f5ac22e16445ca862818be1e7075f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2026, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_H\n#define MIMALLOC_H\n\n#define MI_MALLOC_VERSION 20302 // major + 2 digits minor + 2 digits patch\n\n// ------------------------------------------------------\n// Compiler specific attributes\n// ------------------------------------------------------\n\n#ifdef __cplusplus\n #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11\n #define mi_attr_noexcept noexcept\n #else\n #define mi_attr_noexcept throw()\n #endif\n#else\n #define mi_attr_noexcept\n#endif\n\n#if defined(__cplusplus) && (__cplusplus >= 201703)\n #define mi_decl_nodiscard [[nodiscard]]\n#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl\n #define mi_decl_nodiscard __attribute__((warn_unused_result))\n#elif defined(_HAS_NODISCARD)\n #define mi_decl_nodiscard _NODISCARD\n#elif (_MSC_VER >= 1700)\n #define mi_decl_nodiscard _Check_return_\n#else\n #define mi_decl_nodiscard\n#endif\n\n#if defined(_MSC_VER) || defined(__MINGW32__)\n #if !defined(MI_SHARED_LIB)\n #define mi_decl_export\n #elif defined(MI_SHARED_LIB_EXPORT)\n #define mi_decl_export __declspec(dllexport)\n #else\n #define mi_decl_export __declspec(dllimport)\n #endif\n #if defined(__MINGW32__)\n #define mi_decl_restrict\n #define mi_attr_malloc __attribute__((malloc))\n #else\n #if (_MSC_VER >= 1900) && !defined(__EDG__)\n #define mi_decl_restrict __declspec(allocator) __declspec(restrict)\n #else\n #define mi_decl_restrict __declspec(restrict)\n #endif\n #define mi_attr_malloc\n #endif\n #define mi_cdecl __cdecl\n #define mi_attr_alloc_size(s)\n #define mi_attr_alloc_size2(s1,s2)\n #define mi_attr_alloc_align(p)\n#elif defined(__GNUC__) // includes clang and icc\n #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT)\n #define mi_decl_export __attribute__((visibility(\"default\")))\n #else\n #define mi_decl_export\n #endif\n #define mi_cdecl // leads to warnings... __attribute__((cdecl))\n #define mi_decl_restrict\n #define mi_attr_malloc __attribute__((malloc))\n #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5)\n #define mi_attr_alloc_size(s)\n #define mi_attr_alloc_size2(s1,s2)\n #define mi_attr_alloc_align(p)\n #elif defined(__INTEL_COMPILER)\n #define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))\n #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))\n #define mi_attr_alloc_align(p)\n #else\n #define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))\n #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))\n #define mi_attr_alloc_align(p) __attribute__((alloc_align(p)))\n #endif\n#else\n #define mi_cdecl\n #define mi_decl_export\n #define mi_decl_restrict\n #define mi_attr_malloc\n #define mi_attr_alloc_size(s)\n #define mi_attr_alloc_size2(s1,s2)\n #define mi_attr_alloc_align(p)\n#endif\n\n// ------------------------------------------------------\n// Includes\n// ------------------------------------------------------\n\n#include // size_t\n#include // bool\n#include // INTPTR_MAX\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// ------------------------------------------------------\n// Standard malloc interface\n// ------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);\nmi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);\nmi_decl_export void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);\n\nmi_decl_export void mi_free(void* p) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept;\n\n// ------------------------------------------------------\n// Extended functionality\n// ------------------------------------------------------\n#define MI_SMALL_WSIZE_MAX (128)\n#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*))\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);\nmi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);\nmi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);\n\nmi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;\n\n// for compat with v3.\nmi_decl_export void mi_free_small(void* p) mi_attr_noexcept;\n\n// ------------------------------------------------------\n// Internals\n// ------------------------------------------------------\n\ntypedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);\nmi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept;\n\ntypedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg);\nmi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept;\n\ntypedef void (mi_cdecl mi_error_fun)(int err, void* arg);\nmi_decl_export void mi_register_error(mi_error_fun* fun, void* arg);\n\nmi_decl_export void mi_collect(bool force) mi_attr_noexcept;\nmi_decl_export int mi_version(void) mi_attr_noexcept;\nmi_decl_export void mi_stats_reset(void) mi_attr_noexcept;\nmi_decl_export void mi_stats_merge(void) mi_attr_noexcept;\nmi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL\nmi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;\nmi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;\nmi_decl_export void mi_options_print(void) mi_attr_noexcept;\n\nmi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs,\n size_t* current_rss, size_t* peak_rss,\n size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept;\n\n\n// Generally do not use the following as these are usually called automatically\nmi_decl_export void mi_process_init(void) mi_attr_noexcept;\nmi_decl_export void mi_cdecl mi_process_done(void) mi_attr_noexcept;\nmi_decl_export void mi_thread_init(void) mi_attr_noexcept;\nmi_decl_export void mi_thread_done(void) mi_attr_noexcept;\n\n\n// -------------------------------------------------------------------------------------\n// Aligned allocation\n// Note that `alignment` always follows `size` for consistency with unaligned\n// allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`.\n// -------------------------------------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);\nmi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);\nmi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);\n\n\n// -----------------------------------------------------------------\n// Return allocated block size (if the return value is not NULL)\n// -----------------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);\nmi_decl_nodiscard mi_decl_export void* mi_urealloc(void* p, size_t newsize, size_t* block_size_pre, size_t* block_size_post) mi_attr_noexcept mi_attr_alloc_size(2);\nmi_decl_export void mi_ufree(void* p, size_t* block_size) mi_attr_noexcept;\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\n\n\n// -------------------------------------------------------------------------------------\n// Heaps: first-class, but can only allocate from the same thread that created it.\n// -------------------------------------------------------------------------------------\n\nstruct mi_heap_s;\ntypedef struct mi_heap_s mi_heap_t;\n\nmi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void);\nmi_decl_export void mi_heap_delete(mi_heap_t* heap);\nmi_decl_export void mi_heap_destroy(mi_heap_t* heap);\nmi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap);\nmi_decl_export mi_heap_t* mi_heap_get_default(void);\nmi_decl_export mi_heap_t* mi_heap_get_backing(void);\nmi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept;\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\n\nmi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);\nmi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4);\nmi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept;\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);\nmi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);\nmi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);\n\n\n// --------------------------------------------------------------------------------\n// Zero initialized re-allocation.\n// Only valid on memory that was originally allocated with zero initialization too.\n// e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc.\n// see \n// --------------------------------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);\n\nmi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);\nmi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4);\nmi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3);\n\nmi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);\nmi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4);\n\nmi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);\nmi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);\nmi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5);\nmi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3,4);\n\n\n// ------------------------------------------------------\n// Analysis\n// ------------------------------------------------------\n\nmi_decl_export bool mi_heap_contains_block(mi_heap_t* heap, const void* p);\nmi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p);\nmi_decl_export bool mi_check_owned(const void* p);\n\n// An area of heap space contains blocks of a single size.\ntypedef struct mi_heap_area_s {\n void* blocks; // start of the area containing heap blocks\n size_t reserved; // bytes reserved for this area (virtual)\n size_t committed; // current available bytes for this area\n size_t used; // number of allocated blocks\n size_t block_size; // size in bytes of each block\n size_t full_block_size; // size in bytes of a full block including padding and metadata.\n int heap_tag; // heap tag associated with this area\n} mi_heap_area_t;\n\ntypedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);\n\nmi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);\n\n// Experimental\nmi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept;\n\nmi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept;\nmi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept;\n\nmi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;\nmi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept;\n\nmi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept;\nmi_decl_export void mi_arenas_print(void) mi_attr_noexcept;\n\n// Experimental: heaps associated with specific memory arena's\ntypedef int mi_arena_id_t;\nmi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);\nmi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;\nmi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;\nmi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;\n\n#if MI_MALLOC_VERSION >= 182\n// Create a heap that only allocates in the specified arena\nmi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);\n#endif\n\n\n// Experimental: allow sub-processes whose memory areas stay separated (and no reclamation between them)\n// Used for example for separate interpreters in one process.\ntypedef void* mi_subproc_id_t;\nmi_decl_export mi_subproc_id_t mi_subproc_main(void);\nmi_decl_export mi_subproc_id_t mi_subproc_new(void);\nmi_decl_export void mi_subproc_delete(mi_subproc_id_t subproc);\nmi_decl_export void mi_subproc_add_current_thread(mi_subproc_id_t subproc); // this should be called right after a thread is created (and no allocation has taken place yet)\n\n// Experimental: visit abandoned heap areas (that are not owned by a specific heap)\nmi_decl_export bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);\n\n// Experimental: objects followed by a guard page.\n// A sample rate of 0 disables guarded objects, while 1 uses a guard page for every object.\n// A seed of 0 uses a random start point. Only objects within the size bound are eligable for guard pages.\nmi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed);\nmi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max);\n\n// Experimental: communicate that the thread is part of a threadpool\nmi_decl_export void mi_thread_set_in_threadpool(void) mi_attr_noexcept;\n\n// Experimental: create a new heap with a specified heap tag. Set `allow_destroy` to false to allow the thread\n// to reclaim abandoned memory (with a compatible heap_tag and arena_id) but in that case `mi_heap_destroy` will\n// fall back to `mi_heap_delete`.\nmi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id);\n\n// Experimental and unsafe: assumes the page of `p` is only accessed by the calling thread\nmi_decl_nodiscard mi_decl_export bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept;\n\n// deprecated\nmi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;\nmi_decl_export void mi_collect_reduce(size_t target_thread_owned) mi_attr_noexcept;\n\n\n\n// ------------------------------------------------------\n// Convenience\n// ------------------------------------------------------\n\n#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp)))\n#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp)))\n#define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp)))\n#define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp)))\n#define mi_reallocn_tp(p,tp,n) ((tp*)mi_reallocn(p,n,sizeof(tp)))\n#define mi_recalloc_tp(p,tp,n) ((tp*)mi_recalloc(p,n,sizeof(tp)))\n\n#define mi_heap_malloc_tp(hp,tp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))\n#define mi_heap_zalloc_tp(hp,tp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))\n#define mi_heap_calloc_tp(hp,tp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp)))\n#define mi_heap_mallocn_tp(hp,tp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp)))\n#define mi_heap_reallocn_tp(hp,p,tp,n) ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp)))\n#define mi_heap_recalloc_tp(hp,p,tp,n) ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp)))\n\n\n// ------------------------------------------------------\n// Compatibility with v3\n// ------------------------------------------------------\n\ntypedef mi_heap_t mi_theap_t;\n\n#define mi_theap_get_default() mi_heap_get_default() \n#define mi_theap_set_default(hp) mi_heap_set_default(hp) \n#define mi_theap_collect(hp,force) mi_heap_collect(hp,force)\n#define mi_theap_malloc(hp,sz) mi_heap_malloc(hp,sz)\n#define mi_theap_zalloc(hp,sz) mi_heap_zalloc(hp,sz)\n#define mi_theap_calloc(hp,cnt,sz) mi_heap_calloc(hp,cnt,sz)\n#define mi_theap_malloc_small(hp,sz) mi_heap_malloc_small(hp,sz)\n#define mi_theap_malloc_aligned(hp,sz,a) mi_heap_malloc_aligned(hp,sz,a)\n#define mi_theap_realloc(hp,p,newsz) mi_heap_realloc(hp,p,newsz)\n\n\n// ------------------------------------------------------\n// Options\n// ------------------------------------------------------\n\ntypedef enum mi_option_e {\n // stable options\n mi_option_show_errors, // print error messages\n mi_option_show_stats, // print statistics on termination\n mi_option_verbose, // print verbose messages\n // advanced options\n mi_option_eager_commit, // eager commit segments? (after `eager_commit_delay` segments) (=1)\n mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2)\n mi_option_purge_decommits, // should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)\n mi_option_allow_large_os_pages, // allow use of large (2 or 4 MiB) OS pages, implies eager commit.\n mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB pages) at startup\n mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node\n mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`)\n mi_option_deprecated_segment_cache,\n mi_option_deprecated_page_reset,\n mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination\n mi_option_deprecated_segment_reset,\n mi_option_eager_commit_delay, // the first N segments per thread are not eagerly committed (but per page in the segment on demand)\n mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10)\n mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes.\n mi_option_disallow_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas)\n mi_option_os_tag, // tag used for OS logging (macOS only for now) (=100)\n mi_option_max_errors, // issue at most N error messages\n mi_option_max_warnings, // issue at most N warning messages\n mi_option_max_segment_reclaim, // max. percentage of the abandoned segments can be reclaimed per try (=10%)\n mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe\n mi_option_arena_reserve, // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`)\n mi_option_arena_purge_mult, // multiplier for `purge_delay` for the purging delay for arenas (=10)\n mi_option_purge_extend_delay,\n mi_option_abandoned_reclaim_on_free, // allow to reclaim an abandoned segment on a free (=1)\n mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's)\n mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)\n mi_option_visit_abandoned, // allow visiting heap blocks from abandoned threads (=0)\n mi_option_guarded_min, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)\n mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)\n mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)\n mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000)\n mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)\n mi_option_target_segments_per_thread, // experimental (=0)\n mi_option_generic_collect, // collect heaps every N (=10000) generic allocation calls\n mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.\n _mi_option_last,\n // legacy option names\n mi_option_large_os_pages = mi_option_allow_large_os_pages,\n mi_option_eager_region_commit = mi_option_arena_eager_commit,\n mi_option_reset_decommits = mi_option_purge_decommits,\n mi_option_reset_delay = mi_option_purge_delay,\n mi_option_abandoned_page_reset = mi_option_abandoned_page_purge,\n mi_option_limit_os_alloc = mi_option_disallow_os_alloc\n} mi_option_t;\n\n\nmi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option);\nmi_decl_export void mi_option_enable(mi_option_t option);\nmi_decl_export void mi_option_disable(mi_option_t option);\nmi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);\nmi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);\n\nmi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);\nmi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max);\nmi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option);\nmi_decl_export void mi_option_set(mi_option_t option, long value);\nmi_decl_export void mi_option_set_default(mi_option_t option, long value);\n\n\n// -------------------------------------------------------------------------------------------------------\n// \"mi\" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions.\n// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.)\n// note: we use `mi_cfree` as \"checked free\" and it checks if the pointer is in our heap before free-ing.\n// -------------------------------------------------------------------------------------------------------\n\nmi_decl_export void mi_cfree(void* p) mi_attr_noexcept;\nmi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;\n\nmi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);\n\nmi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);\nmi_decl_nodiscard mi_decl_export int mi_reallocarr(void* ptrp, size_t count, size_t size) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;\n\nmi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept;\nmi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;\nmi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept;\nmi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept;\n\n// wide characters\n#include // wchar_t\nmi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc;\n\n// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.\n// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2);\nmi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);\n\n#ifdef __cplusplus\n}\n#endif\n\n// ---------------------------------------------------------------------------------------------\n// Implement the C++ std::allocator interface for use in STL containers.\n// (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally)\n// ---------------------------------------------------------------------------------------------\n#ifdef __cplusplus\n\n#include // std::size_t\n#include // PTRDIFF_MAX\n#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11\n#include // std::true_type\n#include // std::forward\n#endif\n\ntemplate struct _mi_stl_allocator_common {\n typedef T value_type;\n typedef std::size_t size_type;\n typedef std::ptrdiff_t difference_type;\n typedef value_type& reference;\n typedef value_type const& const_reference;\n typedef value_type* pointer;\n typedef value_type const* const_pointer;\n\n #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11\n using propagate_on_container_copy_assignment = std::true_type;\n using propagate_on_container_move_assignment = std::true_type;\n using propagate_on_container_swap = std::true_type;\n template void construct(U* p, Args&& ...args) { ::new(p) U(std::forward(args)...); }\n template void destroy(U* p) mi_attr_noexcept { p->~U(); }\n #else\n void construct(pointer p, value_type const& val) { ::new(p) value_type(val); }\n void destroy(pointer p) { p->~value_type(); }\n #endif\n\n size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); }\n pointer address(reference x) const { return &x; }\n const_pointer address(const_reference x) const { return &x; }\n};\n\ntemplate struct mi_stl_allocator : public _mi_stl_allocator_common {\n using typename _mi_stl_allocator_common::size_type;\n using typename _mi_stl_allocator_common::value_type;\n using typename _mi_stl_allocator_common::pointer;\n template struct rebind { typedef mi_stl_allocator other; };\n\n mi_stl_allocator() mi_attr_noexcept = default;\n mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept = default;\n template mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept { }\n mi_stl_allocator select_on_container_copy_construction() const { return *this; }\n void deallocate(T* p, size_type) { mi_free(p); }\n\n #if (__cplusplus >= 201703L) // C++17\n mi_decl_nodiscard T* allocate(size_type count) { return static_cast(mi_new_n(count, sizeof(T))); }\n mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }\n #else\n mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast(mi_new_n(count, sizeof(value_type))); }\n #endif\n\n #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11\n using is_always_equal = std::true_type;\n #endif\n};\n\ntemplate bool operator==(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return true; }\ntemplate bool operator!=(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return false; }\n\n\n#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11\n#define MI_HAS_HEAP_STL_ALLOCATOR 1\n\n#include // std::shared_ptr\n\n// Common base class for STL allocators in a specific heap\ntemplate struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common {\n using typename _mi_stl_allocator_common::size_type;\n using typename _mi_stl_allocator_common::value_type;\n using typename _mi_stl_allocator_common::pointer;\n\n _mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp, [](mi_heap_t*) {}) {} /* will not delete nor destroy the passed in heap */\n\n #if (__cplusplus >= 201703L) // C++17\n mi_decl_nodiscard T* allocate(size_type count) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); }\n mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }\n #else\n mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); }\n #endif\n\n #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11\n using is_always_equal = std::false_type;\n #endif\n\n void collect(bool force) { mi_heap_collect(this->heap.get(), force); }\n template bool is_equal(const _mi_heap_stl_allocator_common& x) const { return (this->heap == x.heap); }\n\nprotected:\n std::shared_ptr heap;\n template friend struct _mi_heap_stl_allocator_common;\n\n _mi_heap_stl_allocator_common() {\n mi_heap_t* hp = mi_heap_new();\n this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */\n }\n _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { }\n template _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { }\n\nprivate:\n static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } }\n static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } }\n};\n\n// STL allocator allocation in a specific heap\ntemplate struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common {\n using typename _mi_heap_stl_allocator_common::size_type;\n mi_heap_stl_allocator() : _mi_heap_stl_allocator_common() { } // creates fresh heap that is deleted when the destructor is called\n mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap\n template mi_heap_stl_allocator(const mi_heap_stl_allocator& x) mi_attr_noexcept : _mi_heap_stl_allocator_common(x) { }\n\n mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; }\n void deallocate(T* p, size_type) { mi_free(p); }\n template struct rebind { typedef mi_heap_stl_allocator other; };\n};\n\ntemplate bool operator==(const mi_heap_stl_allocator& x, const mi_heap_stl_allocator& y) mi_attr_noexcept { return (x.is_equal(y)); }\ntemplate bool operator!=(const mi_heap_stl_allocator& x, const mi_heap_stl_allocator& y) mi_attr_noexcept { return (!x.is_equal(y)); }\n\n\n// STL allocator allocation in a specific heap, where `free` does nothing and\n// the heap is destroyed in one go on destruction -- use with care!\ntemplate struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common {\n using typename _mi_heap_stl_allocator_common::size_type;\n mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common() { } // creates fresh heap that is destroyed when the destructor is called\n mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap\n template mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator& x) mi_attr_noexcept : _mi_heap_stl_allocator_common(x) { }\n\n mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; }\n void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ }\n template struct rebind { typedef mi_heap_destroy_stl_allocator other; };\n};\n\ntemplate bool operator==(const mi_heap_destroy_stl_allocator& x, const mi_heap_destroy_stl_allocator& y) mi_attr_noexcept { return (x.is_equal(y)); }\ntemplate bool operator!=(const mi_heap_destroy_stl_allocator& x, const mi_heap_destroy_stl_allocator& y) mi_attr_noexcept { return (!x.is_equal(y)); }\n\n#endif // C++11\n\n#endif // __cplusplus\n\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/include/mimalloc/atomic.h", "sha256": "3a4f1df56ec523bb59e0de2bceccf8d0aa2f1465fcf1fd694000e7a4aa601287", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2024 Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_ATOMIC_H\n#define MIMALLOC_ATOMIC_H\n\n// include windows.h or pthreads.h\n#if defined(_WIN32)\n#ifndef WIN32_LEAN_AND_MEAN\n#define WIN32_LEAN_AND_MEAN\n#endif\n#include \n#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))\n#define MI_USE_PTHREADS\n#include \n#endif\n\n// --------------------------------------------------------------------------------------------\n// Atomics\n// We need to be portable between C, C++, and MSVC.\n// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode.\n// This is why we try to use only `uintptr_t` and `*` as atomic types.\n// To gain better insight in the range of used atomics, we use explicitly named memory order operations\n// instead of passing the memory order as a parameter.\n// -----------------------------------------------------------------------------------------------\n\n#if defined(__cplusplus)\n// Use C++ atomics\n#include \n#define _Atomic(tp) std::atomic\n#define mi_atomic(name) std::atomic_##name\n#define mi_memory_order(name) std::memory_order_##name\n#if (__cplusplus >= 202002L) // c++20, see issue #571\n #define MI_ATOMIC_VAR_INIT(x) x\n#elif !defined(ATOMIC_VAR_INIT)\n #define MI_ATOMIC_VAR_INIT(x) x\n#else\n #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)\n#endif\n#elif defined(_MSC_VER)\n// Use MSVC C wrapper for C11 atomics\n#define _Atomic(tp) tp\n#define MI_ATOMIC_VAR_INIT(x) x\n#define mi_atomic(name) mi_atomic_##name\n#define mi_memory_order(name) mi_memory_order_##name\n#else\n// Use C11 atomics\n#include \n#define mi_atomic(name) atomic_##name\n#define mi_memory_order(name) memory_order_##name\n#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735\n #define MI_ATOMIC_VAR_INIT(x) x\n#elif !defined(ATOMIC_VAR_INIT)\n #define MI_ATOMIC_VAR_INIT(x) x\n#else\n #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)\n#endif\n#endif\n\n// Various defines for all used memory orders in mimalloc\n#define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail) \\\n mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail)\n\n#define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail) \\\n mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail)\n\n#define mi_atomic_load_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))\n#define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))\n#define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))\n#define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))\n#define mi_atomic_exchange_relaxed(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(relaxed))\n#define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release))\n#define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel))\n#define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))\n#define mi_atomic_cas_weak_acq_rel(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))\n#define mi_atomic_cas_strong_release(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))\n#define mi_atomic_cas_strong_acq_rel(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))\n\n#define mi_atomic_add_relaxed(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed))\n#define mi_atomic_sub_relaxed(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed))\n#define mi_atomic_add_acq_rel(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel))\n#define mi_atomic_sub_acq_rel(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel))\n#define mi_atomic_and_acq_rel(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel))\n#define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel))\n\n#define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(uintptr_t)1)\n#define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(uintptr_t)1)\n#define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1)\n#define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1)\n\nstatic inline void mi_atomic_yield(void);\nstatic inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add);\nstatic inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub);\n\n\n#if defined(__cplusplus) || !defined(_MSC_VER)\n\n// In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value)\n// We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well\n#define mi_atomic_load_ptr_acquire(tp,p) mi_atomic_load_acquire(p)\n#define mi_atomic_load_ptr_relaxed(tp,p) mi_atomic_load_relaxed(p)\n\n// In C++ we need to add casts to help resolve templates if NULL is passed\n#if defined(__cplusplus)\n#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,(tp*)x)\n#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,(tp*)x)\n#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des)\n#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des)\n#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des)\n#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,(tp*)des)\n#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,(tp*)x)\n#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x)\n#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x)\n#else\n#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,x)\n#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,x)\n#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des)\n#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des)\n#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des)\n#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,des)\n#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,x)\n#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x)\n#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x)\n#endif\n\n// These are used by the statistics\nstatic inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) {\n return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));\n}\nstatic inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {\n const int64_t add = mi_atomic_load_relaxed((_Atomic(int64_t)*)padd);\n if (add != 0) {\n mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));\n }\n}\nstatic inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {\n int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p);\n while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, ¤t, x)) { /* nothing */ };\n}\n\n// Used by timers\n#define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))\n#define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))\n#define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))\n#define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))\n\n#define mi_atomic_casi64_strong_acq_rel(p,e,d) mi_atomic_cas_strong_acq_rel(p,e,d)\n#define mi_atomic_addi64_acq_rel(p,i) mi_atomic_add_acq_rel(p,i)\n\n\n#elif defined(_MSC_VER)\n\n// Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.\n// It is recommended to always compile as C++ when using MSVC.\n\n#include \n#ifdef _WIN64\ntypedef LONG64 msc_intptr_t;\n#define MI_MSC_64(f) f##64\n#define MI_MSC_XX(f) f##64\n#else\ntypedef LONG msc_intptr_t;\n#define MI_MSC_64(f) f\n#define MI_MSC_XX(f) f##32\n#endif\n\ntypedef enum mi_memory_order_e {\n mi_memory_order_relaxed,\n mi_memory_order_consume,\n mi_memory_order_acquire,\n mi_memory_order_release,\n mi_memory_order_acq_rel,\n mi_memory_order_seq_cst\n} mi_memory_order;\n\nstatic inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) {\n (void)(mo);\n return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add);\n}\nstatic inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) {\n (void)(mo);\n return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub));\n}\nstatic inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {\n (void)(mo);\n return (uintptr_t)MI_MSC_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x);\n}\nstatic inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {\n (void)(mo);\n return (uintptr_t)MI_MSC_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x);\n}\nstatic inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {\n (void)(mo1); (void)(mo2);\n const uintptr_t read = (uintptr_t)MI_MSC_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected));\n if (read == *expected) {\n return true;\n }\n else {\n *expected = read;\n return false;\n }\n}\nstatic inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {\n return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2);\n}\nstatic inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) {\n (void)(mo);\n return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);\n}\nstatic inline void mi_atomic_thread_fence(mi_memory_order mo) {\n (void)(mo);\n _Atomic(uintptr_t) x = 0;\n mi_atomic_exchange_explicit(&x, 1, mo);\n}\n\nstatic inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) {\n (void)(mo);\n // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc\n #if defined(_M_IX86) || defined(_M_X64)\n return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);\n #elif defined(_M_ARM) || defined(_M_ARM64)\n if (mo == mi_memory_order_relaxed) {\n return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);\n }\n else if (mo <= mi_memory_order_acquire) {\n return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p);\n }\n else {\n const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);\n __dmb(15); // _ARM(64)_BARRIER_SY\n return u;\n }\n #else\n #warning \"define mi_atomic_load_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)\"\n return MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);\n #endif\n}\nstatic inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {\n (void)(mo);\n // assert(mo<=mi_memory_order_release); // others are not used by mimalloc\n #if defined(_M_IX86) || defined(_M_X64)\n MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);\n #elif defined(_M_ARM) || defined(_M_ARM64)\n if (mo == mi_memory_order_relaxed) {\n MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);\n }\n else if (mo <= mi_memory_order_release) {\n MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x);\n }\n else {\n mi_atomic_exchange_explicit(p, x, mo);\n }\n #else\n mi_atomic_exchange_explicit(p, x, mo);\n #endif\n}\n\nstatic inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) {\n (void)(mo);\n // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc\n #if defined(_M_IX86) || defined(_M_X64)\n return __iso_volatile_load64((volatile const int64_t*)p);\n #elif defined(_M_ARM) || defined(_M_ARM64)\n if (mo == mi_memory_order_relaxed) {\n return __iso_volatile_load64((volatile const int64_t*)p);\n }\n #if defined(_M_ARM64)\n else if (mo <= mi_memory_order_acquire) {\n return __ldar64((volatile const uintptr_t*)p);\n }\n #endif\n else {\n const int64_t i = __iso_volatile_load64((volatile const int64_t*)p);\n __dmb(15); // _ARM(64)_BARRIER_SY\n return i;\n }\n #else\n #warning \"define mi_atomic_loadi64_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)\"\n return __iso_volatile_load64((volatile const int64_t*)p);\n #endif\n}\n\nstatic inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) {\n (void)(mo);\n // assert(mo<=mi_memory_order_release); // others are not used by mimalloc\n #if defined(_M_IX86) || defined(_M_X64)\n __iso_volatile_store64((volatile int64_t*)p,x);\n #elif defined(_M_ARM) || defined(_M_ARM64)\n if (mo == mi_memory_order_relaxed) {\n __iso_volatile_store64((volatile int64_t*)p,x);\n }\n #if defined(_M_ARM64)\n else if (mo == mi_memory_order_release) {\n __stlr64((volatile uint64_t*)p, (uint64_t)x);\n }\n #endif\n else {\n InterlockedExchange64(p, x);\n }\n #else\n InterlockedExchange64(p, x);\n #endif\n}\n\n// These are used by the statistics\nstatic inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) {\n #ifdef _WIN64\n return (int64_t)mi_atomic_addi((int64_t*)p, add);\n #elif defined(_M_ARM)\n return _InterlockedExchangeAdd64(p, add);\n #else\n // x86\n int64_t current;\n int64_t sum;\n do {\n current = __iso_volatile_load64((volatile const int64_t*)p);\n sum = current + add;\n } while (_InterlockedCompareExchange64(p, sum, current) != current);\n return current;\n #endif\n}\nstatic inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {\n const int64_t add = *padd;\n if (add != 0) {\n mi_atomic_addi64_relaxed((volatile _Atomic(int64_t)*)p, add);\n }\n}\n\nstatic inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) {\n int64_t current;\n do {\n current = *p;\n } while (current < x && _InterlockedCompareExchange64(p, x, current) != current);\n}\n\nstatic inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) {\n mi_atomic_addi64_relaxed(p, i);\n}\n\nstatic inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) {\n const int64_t read = _InterlockedCompareExchange64(p, des, *exp);\n if (read == *exp) {\n return true;\n }\n else {\n *exp = read;\n return false;\n }\n}\n\n// The pointer macros cast to `uintptr_t`.\n#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))\n#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))\n#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))\n#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))\n#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)\n#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)\n#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)\n#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)\n#define mi_atomic_exchange_ptr_relaxed(tp,p,x) (tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)\n#define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)\n#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)\n\n#define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire))\n#define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed))\n#define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release))\n#define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed))\n\n\n#endif\n\n\n// Atomically add a signed value; returns the previous value.\nstatic inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) {\n return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add);\n}\n\n// Atomically subtract a signed value; returns the previous value.\nstatic inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) {\n return (intptr_t)mi_atomic_addi(p, -sub);\n}\n\n\n// ----------------------------------------------------------------------\n// Once and Guard\n// ----------------------------------------------------------------------\n\ntypedef _Atomic(uintptr_t) mi_atomic_guard_t;\n\n// Allows only one thread to execute at a time (without blocking anyone)\n#define mi_atomic_guard(guard) \\\n uintptr_t _mi_guard_expected = 0; \\\n for(bool _mi_guard_once = true; \\\n _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \\\n (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) )\n\n\n\n// ----------------------------------------------------------------------\n// Yield\n// ----------------------------------------------------------------------\n\n#if defined(_WIN32)\nstatic inline void mi_atomic_yield(void) {\n YieldProcessor(); // see issue #1215 and #1225 why this is preferred over __yield or SwitchToThread\n}\n#elif defined(__SSE2__)\n#include \nstatic inline void mi_atomic_yield(void) {\n _mm_pause();\n}\n#elif (defined(__GNUC__) || defined(__clang__)) && \\\n (defined(__x86_64__) || defined(__i386__) || \\\n defined(__aarch64__) || defined(__arm__) || \\\n defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__))\n#if defined(__x86_64__) || defined(__i386__)\nstatic inline void mi_atomic_yield(void) {\n __asm__ volatile (\"pause\" ::: \"memory\");\n}\n#elif defined(__aarch64__)\nstatic inline void mi_atomic_yield(void) {\n __asm__ volatile(\"isb\");\n}\n#elif defined(__arm__)\n#if __ARM_ARCH >= 7\nstatic inline void mi_atomic_yield(void) {\n __asm__ volatile(\"yield\" ::: \"memory\");\n}\n#else\nstatic inline void mi_atomic_yield(void) {\n __asm__ volatile (\"nop\" ::: \"memory\");\n}\n#endif\n#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__)\n#ifdef __APPLE__\nstatic inline void mi_atomic_yield(void) {\n __asm__ volatile (\"or r27,r27,r27\" ::: \"memory\");\n}\n#else\nstatic inline void mi_atomic_yield(void) {\n __asm__ __volatile__ (\"or 27,27,27\" ::: \"memory\");\n}\n#endif\n#endif\n#elif defined(__sun)\n#include \nstatic inline void mi_atomic_yield(void) {\n smt_pause();\n}\n#elif defined(__wasi__)\n#include \nstatic inline void mi_atomic_yield(void) {\n sched_yield();\n}\n// Fallback for other archs\n#elif defined(__cplusplus)\n#include \nstatic inline void mi_atomic_yield(void) {\n std::this_thread::yield();\n}\n#else\n#include \nstatic inline void mi_atomic_yield(void) {\n sleep(0);\n}\n#endif\n\n\n// ----------------------------------------------------------------------\n// Locks\n// These should be light-weight in-process only locks.\n// Only used for reserving arena's and to maintain the abandoned list.\n// ----------------------------------------------------------------------\n#if _MSC_VER\n#pragma warning(disable:26110) // unlock with holding lock\n#endif\n\n#define mi_lock(lock) for(bool _mi_go = (mi_lock_acquire(lock),true); _mi_go; (mi_lock_release(lock), _mi_go=false) )\n#define mi_lock_maybe(lock,acquire) for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) )\n\n\n#if defined(_WIN32)\n\ntypedef struct mi_lock_s {\n SRWLOCK mutex; // slim reader-writer lock\n} mi_lock_t;\n\n#define MI_LOCK_INITIALIZER { SRWLOCK_INIT }\n\nstatic inline bool mi_lock_try_acquire(mi_lock_t* lock) {\n return TryAcquireSRWLockExclusive(&lock->mutex);\n}\nstatic inline void mi_lock_acquire(mi_lock_t* lock) {\n AcquireSRWLockExclusive(&lock->mutex);\n}\nstatic inline void mi_lock_release(mi_lock_t* lock) {\n ReleaseSRWLockExclusive(&lock->mutex);\n}\nstatic inline void mi_lock_init(mi_lock_t* lock) {\n InitializeSRWLock(&lock->mutex);\n}\nstatic inline void mi_lock_done(mi_lock_t* lock) {\n (void)(lock);\n}\n\n#elif defined(MI_USE_PTHREADS)\n\n#include // memcpy\nvoid _mi_error_message(int err, const char* fmt, ...);\n\ntypedef struct mi_lock_s {\n pthread_mutex_t mutex;\n} mi_lock_t;\n\n#define MI_LOCK_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }\n\nstatic inline bool mi_lock_try_acquire(mi_lock_t* lock) {\n return (pthread_mutex_trylock(&lock->mutex) == 0);\n}\nstatic inline void mi_lock_acquire(mi_lock_t* lock) {\n const int err = pthread_mutex_lock(&lock->mutex);\n if (err != 0) {\n _mi_error_message(err, \"internal error: lock cannot be acquired (err %i)\\n\", err);\n }\n}\nstatic inline void mi_lock_release(mi_lock_t* lock) {\n pthread_mutex_unlock(&lock->mutex);\n}\nstatic inline void mi_lock_init(mi_lock_t* lock) {\n if(lock==NULL) return;\n // use this instead of pthread_mutex_init since that can cause allocation on some platforms (and recursively initialize)\n const pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;\n memcpy(&lock->mutex,&mutex,sizeof(mutex));\n}\nstatic inline void mi_lock_done(mi_lock_t* lock) {\n pthread_mutex_destroy(&lock->mutex);\n}\n\n#elif defined(__cplusplus)\n\n#include \n#include \n#include \n\ntypedef struct mi_lock_s {\n std::mutex mutex;\n} mi_lock_t;\n\n#define MI_LOCK_INITIALIZER { }\n\nstatic inline bool mi_lock_try_acquire(mi_lock_t* lock) {\n return lock->mutex.try_lock();\n}\nstatic inline void mi_lock_acquire(mi_lock_t* lock) {\n lock->mutex.lock();\n}\nstatic inline void mi_lock_release(mi_lock_t* lock) {\n lock->mutex.unlock();\n}\nstatic inline void mi_lock_init(mi_lock_t* lock) {\n new(&lock->mutex) std::mutex();\n}\nstatic inline void mi_lock_done(mi_lock_t* lock) {\n (void)(lock);\n}\n\n#else\n\n// fall back to poor man's locks.\n// this should only be the case in a single-threaded environment (like __wasi__)\n#include \n#ifndef EFAULT\n#define EFAULT (14)\n#endif\nvoid _mi_error_message(int err, const char* fmt, ...);\n\ntypedef struct mi_lock_s {\n _Atomic(uintptr_t) mutex;\n} mi_lock_t;\n\n#define MI_LOCK_INITIALIZER { MI_ATOMIC_VAR_INIT(0) }\n\nstatic inline bool mi_lock_try_acquire(mi_lock_t* lock) {\n uintptr_t expected = 0;\n return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1);\n}\nstatic inline void mi_lock_acquire(mi_lock_t* lock) {\n for (int i = 0; i < 10000; i++) { // for at most 10000 tries?\n if (mi_lock_try_acquire(lock)) return;\n mi_atomic_yield();\n }\n _mi_error_message(EFAULT, \"internal error: lock cannot be acquired (due to lack of native lock primitives)\\n\");\n}\nstatic inline void mi_lock_release(mi_lock_t* lock) {\n mi_atomic_store_release(&lock->mutex, (uintptr_t)0);\n}\nstatic inline void mi_lock_init(mi_lock_t* lock) {\n mi_lock_release(lock);\n}\nstatic inline void mi_lock_done(mi_lock_t* lock) {\n (void)(lock);\n}\n\n#endif\n\n\ntypedef struct mi_atomic_once_s {\n _Atomic(uintptr_t) tid;\n mi_lock_t lock;\n} mi_atomic_once_t;\n\n// Returns `true` only on the first invocation, signifying we can execute an action once.\n// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.\n// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.\nbool _mi_atomic_once_enter(mi_atomic_once_t* once); // defined in `libc.c`\nvoid _mi_atomic_once_release(mi_atomic_once_t* once); // defined in `libc.c`\n\n#define mi_atomic_do_once \\\n static mi_atomic_once_t _mi_once = { MI_ATOMIC_VAR_INIT(0), MI_LOCK_INITIALIZER }; \\\n for(bool _mi_exec = _mi_atomic_once_enter(&_mi_once); _mi_exec; (_mi_atomic_once_release(&_mi_once),_mi_exec=false))\n\n\n#endif // __MIMALLOC_ATOMIC_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/include/mimalloc/internal.h", "sha256": "def781a5e62b35de55c2a9ddd667b5329bb6875b91c514ee63af55c978c1fb5e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_INTERNAL_H\n#define MIMALLOC_INTERNAL_H\n\n// --------------------------------------------------------------------------\n// This file contains the internal API's of mimalloc and various utility\n// functions and macros.\n// --------------------------------------------------------------------------\n\n#include \"types.h\"\n#include \"track.h\"\n\n\n// --------------------------------------------------------------------------\n// Compiler defines\n// --------------------------------------------------------------------------\n\n#if (MI_DEBUG>0)\n#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__)\n#else\n#define mi_trace_message(...)\n#endif\n\n#define mi_decl_cache_align mi_decl_align(64)\n\n#if defined(_MSC_VER)\n#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths)\n#pragma warning(disable:26812) // unscoped enum warning\n#define mi_decl_noinline __declspec(noinline)\n#define mi_decl_thread __declspec(thread)\n#define mi_decl_align(a) __declspec(align(a))\n#define mi_decl_noreturn __declspec(noreturn)\n#define mi_decl_weak\n#define mi_decl_hidden\n#define mi_decl_cold\n#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc\n#define mi_decl_noinline __attribute__((noinline))\n#define mi_decl_thread __thread\n#define mi_decl_align(a) __attribute__((aligned(a)))\n#define mi_decl_noreturn __attribute__((noreturn))\n#define mi_decl_weak __attribute__((weak))\n#define mi_decl_hidden __attribute__((visibility(\"hidden\")))\n#if (__GNUC__ >= 4) || defined(__clang__)\n#define mi_decl_cold __attribute__((cold))\n#else\n#define mi_decl_cold\n#endif\n#elif __cplusplus >= 201103L // c++11\n#define mi_decl_noinline\n#define mi_decl_thread thread_local\n#define mi_decl_align(a) alignas(a)\n#define mi_decl_noreturn [[noreturn]]\n#define mi_decl_weak\n#define mi_decl_hidden\n#define mi_decl_cold\n#else\n#define mi_decl_noinline\n#define mi_decl_thread __thread // hope for the best :-)\n#define mi_decl_align(a)\n#define mi_decl_noreturn\n#define mi_decl_weak\n#define mi_decl_hidden\n#define mi_decl_cold\n#endif\n\n#if defined(__GNUC__) || defined(__clang__)\n#define mi_unlikely(x) (__builtin_expect(!!(x),false))\n#define mi_likely(x) (__builtin_expect(!!(x),true))\n#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L)\n#define mi_unlikely(x) (x) [[unlikely]]\n#define mi_likely(x) (x) [[likely]]\n#else\n#define mi_unlikely(x) (x)\n#define mi_likely(x) (x)\n#endif\n\n#ifndef __has_builtin\n#define __has_builtin(x) 0\n#endif\n\n#if defined(__cplusplus)\n#define mi_decl_externc extern \"C\"\n#else\n#define mi_decl_externc\n#endif\n\n#if defined(__EMSCRIPTEN__) && !defined(__wasi__)\n#define __wasi__\n#endif\n\n\n// --------------------------------------------------------------------------\n// Internal functions\n// --------------------------------------------------------------------------\n\n// \"libc.c\"\n#include \nint _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args);\nint _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...);\nchar _mi_toupper(char c);\nint _mi_strnicmp(const char* s, const char* t, size_t n);\nvoid _mi_strlcpy(char* dest, const char* src, size_t dest_size);\nvoid _mi_strlcat(char* dest, const char* src, size_t dest_size);\nsize_t _mi_strlen(const char* s);\nsize_t _mi_strnlen(const char* s, size_t max_len);\nbool _mi_streq(const char* s, const char* t);\nbool _mi_getenv(const char* name, char* result, size_t result_size);\n\n// \"options.c\"\nvoid _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);\nvoid _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);\nvoid _mi_message(const char* fmt, ...);\nvoid _mi_warning_message(const char* fmt, ...);\nvoid _mi_verbose_message(const char* fmt, ...);\nvoid _mi_trace_message(const char* fmt, ...);\nvoid _mi_options_init(void);\nlong _mi_option_get_fast(mi_option_t option);\nvoid _mi_error_message(int err, const char* fmt, ...);\n\n// random.c\nvoid _mi_random_init(mi_random_ctx_t* ctx);\nvoid _mi_random_init_weak(mi_random_ctx_t* ctx);\nvoid _mi_random_reinit_if_weak(mi_random_ctx_t * ctx);\nvoid _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx);\nuintptr_t _mi_random_next(mi_random_ctx_t* ctx);\nuintptr_t _mi_heap_random_next(mi_heap_t* heap);\nuintptr_t _mi_os_random_weak(uintptr_t extra_seed);\nstatic inline uintptr_t _mi_random_shuffle(uintptr_t x);\n\n// init.c\nextern mi_decl_hidden mi_decl_cache_align mi_stats_t _mi_stats_main;\nextern mi_decl_hidden mi_decl_cache_align const mi_page_t _mi_page_empty;\nvoid _mi_auto_process_init(void);\nvoid mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept;\nbool _mi_is_redirected(void);\nbool _mi_allocator_init(const char** message);\nvoid _mi_allocator_done(void);\nbool _mi_is_main_thread(void);\nsize_t _mi_current_thread_count(void);\nbool _mi_preloading(void); // true while the C runtime is not initialized yet\nvoid _mi_thread_done(mi_heap_t* heap);\nvoid _mi_thread_data_collect(void);\nvoid _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap);\nmi_threadid_t _mi_thread_id(void) mi_attr_noexcept;\nmi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap\nmi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);\nvoid _mi_heap_guarded_init(mi_heap_t* heap);\n\n// os.c\nvoid _mi_os_init(void); // called from process init\nvoid* _mi_os_alloc(size_t size, mi_memid_t* memid);\nvoid* _mi_os_zalloc(size_t size, mi_memid_t* memid);\nvoid _mi_os_free(void* p, size_t size, mi_memid_t memid);\nvoid _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid);\n\nsize_t _mi_os_page_size(void);\nsize_t _mi_os_good_alloc_size(size_t size);\nbool _mi_os_has_overcommit(void);\nbool _mi_os_has_virtual_reserve(void);\n\nbool _mi_os_reset(void* addr, size_t size);\nbool _mi_os_decommit(void* addr, size_t size);\nbool _mi_os_unprotect(void* addr, size_t size);\nbool _mi_os_purge(void* p, size_t size);\nbool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size);\nvoid _mi_os_reuse(void* p, size_t size);\nmi_decl_nodiscard bool _mi_os_commit(void* p, size_t size, bool* is_zero);\nmi_decl_nodiscard bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size);\nbool _mi_os_protect(void* addr, size_t size);\n\nvoid* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid);\nvoid* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid);\n\nvoid* _mi_os_get_aligned_hint(size_t try_alignment, size_t size);\nbool _mi_os_canuse_large_page(size_t size, size_t alignment);\nsize_t _mi_os_large_page_size(void);\nvoid* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);\n\nint _mi_os_numa_node_count(void);\nint _mi_os_numa_node(void);\n\n// arena.c\nmi_arena_id_t _mi_arena_id_none(void);\nvoid _mi_arena_free(void* p, size_t size, size_t still_committed_size, mi_memid_t memid);\nvoid* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid);\nvoid* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid);\nbool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id);\nbool _mi_arena_contains(const void* p);\nvoid _mi_arenas_collect(bool force_purge);\nvoid _mi_arena_unsafe_destroy_all(void);\n\nbool _mi_arena_segment_clear_abandoned(mi_segment_t* segment);\nvoid _mi_arena_segment_mark_abandoned(mi_segment_t* segment);\n\nvoid* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid);\nvoid _mi_arena_meta_free(void* p, mi_memid_t memid, size_t size);\n\ntypedef struct mi_arena_field_cursor_s { // abstract struct\n size_t os_list_count; // max entries to visit in the OS abandoned list\n size_t start; // start arena idx (may need to be wrapped)\n size_t end; // end arena idx (exclusive, may need to be wrapped)\n size_t bitmap_idx; // current bit idx for an arena\n mi_subproc_t* subproc; // only visit blocks in this sub-process\n bool visit_all; // ensure all abandoned blocks are seen (blocking)\n bool hold_visit_lock; // if the subproc->abandoned_os_visit_lock is held\n} mi_arena_field_cursor_t;\nvoid _mi_arena_field_cursor_init(mi_heap_t* heap, mi_subproc_t* subproc, bool visit_all, mi_arena_field_cursor_t* current);\nmi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous);\nvoid _mi_arena_field_cursor_done(mi_arena_field_cursor_t* current);\n\n// \"segment-map.c\"\nvoid _mi_segment_map_allocated_at(const mi_segment_t* segment);\nvoid _mi_segment_map_freed_at(const mi_segment_t* segment);\nvoid _mi_segment_map_unsafe_destroy(void);\n\n// \"segment.c\"\nmi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld);\nvoid _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld);\nvoid _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld);\nbool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld);\nvoid _mi_segment_collect(mi_segment_t* segment, bool force);\n\n#if MI_HUGE_PAGE_ABANDON\nvoid _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block);\n#else\nvoid _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block);\n#endif\n\nuint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); // page start for any page\nvoid _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld);\nvoid _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld);\nbool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment);\nbool _mi_segment_visit_blocks(mi_segment_t* segment, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);\n\n// \"page.c\"\nvoid* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept mi_attr_malloc;\n\nvoid _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks\nvoid _mi_page_unfull(mi_page_t* page);\nvoid _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page\nvoid _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread...\nvoid _mi_page_force_abandon(mi_page_t* page);\n\nvoid _mi_heap_delayed_free_all(mi_heap_t* heap);\nbool _mi_heap_delayed_free_partial(mi_heap_t* heap);\nvoid _mi_heap_collect_retired(mi_heap_t* heap, bool force);\n\nvoid _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never);\nbool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never);\nsize_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append);\nvoid _mi_deferred_free(mi_heap_t* heap, bool force);\n\nvoid _mi_page_free_collect(mi_page_t* page,bool force);\nvoid _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments\n\nsize_t _mi_page_stats_bin(const mi_page_t* page); // for stats\nsize_t _mi_bin_size(size_t bin); // for stats\nsize_t _mi_bin(size_t size); // for stats\n\n// \"heap.c\"\nvoid _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag);\nvoid _mi_heap_destroy_pages(mi_heap_t* heap);\nvoid _mi_heap_collect_abandon(mi_heap_t* heap);\nvoid _mi_heap_set_default_direct(mi_heap_t* heap);\nbool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid);\nvoid _mi_heap_unsafe_destroy_all(mi_heap_t* heap);\nmi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag);\nvoid _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page);\nbool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg);\n\n// \"stats.c\"\nvoid _mi_stats_done(mi_stats_t* stats);\nvoid _mi_stats_merge_thread(mi_tld_t* tld);\nmi_msecs_t _mi_clock_now(void);\nmi_msecs_t _mi_clock_end(mi_msecs_t start);\nmi_msecs_t _mi_clock_start(void);\n\n// \"alloc.c\"\nvoid* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero, size_t* usable) mi_attr_noexcept; // called from `_mi_malloc_generic`\nvoid* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_heap_malloc_aligned`\nvoid* _mi_page_malloc_zeroed(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_heap_malloc_aligned`\nvoid* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept;\nvoid* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept; // called from `_mi_heap_malloc_aligned`\nvoid* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept;\nmi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p);\nbool _mi_free_delayed_block(mi_block_t* block);\nvoid _mi_free_generic(mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept; // for runtime integration\nvoid _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size);\n\n#if MI_DEBUG>1\nbool _mi_page_is_valid(mi_page_t* page);\n#endif\n\n\n/* -----------------------------------------------------------\n Error codes passed to `_mi_fatal_error`\n All are recoverable but EFAULT is a serious error and aborts by default in secure mode.\n For portability define undefined error codes using common Unix codes:\n \n----------------------------------------------------------- */\n#include \n#ifndef EAGAIN // double free\n#define EAGAIN (11)\n#endif\n#ifndef ENOMEM // out of memory\n#define ENOMEM (12)\n#endif\n#ifndef EFAULT // corrupted free-list or meta-data\n#define EFAULT (14)\n#endif\n#ifndef EINVAL // trying to free an invalid pointer\n#define EINVAL (22)\n#endif\n#ifndef EOVERFLOW // count*size overflow\n#define EOVERFLOW (75)\n#endif\n\n\n// ------------------------------------------------------\n// Assertions\n// ------------------------------------------------------\n\n#if (MI_DEBUG)\n// use our own assertion to print without memory allocation\nmi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_attr_noexcept;\n#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__))\n#else\n#define mi_assert(x)\n#endif\n\n#if (MI_DEBUG>1)\n#define mi_assert_internal mi_assert\n#else\n#define mi_assert_internal(x)\n#endif\n\n#if (MI_DEBUG>2)\n#define mi_assert_expensive mi_assert\n#else\n#define mi_assert_expensive(x)\n#endif\n\n\n\n/* -----------------------------------------------------------\n Inlined definitions\n----------------------------------------------------------- */\n#define MI_UNUSED(x) (void)(x)\n#if (MI_DEBUG>1)\n#define MI_UNUSED_RELEASE(x)\n#else\n#define MI_UNUSED_RELEASE(x) MI_UNUSED(x)\n#endif\n\n#define MI_INIT4(x) x(),x(),x(),x()\n#define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x)\n#define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x)\n#define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x)\n#define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x)\n#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x)\n#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x)\n#define MI_INIT74(x) MI_INIT64(x),MI_INIT8(x),x(),x()\n\n#include \n// initialize a local variable to zero; use memset as compilers optimize constant sized memset's\n#define _mi_memzero_var(x) memset(&x,0,sizeof(x))\n\n// Is `x` a power of two? (0 is considered a power of two)\nstatic inline bool _mi_is_power_of_two(uintptr_t x) {\n return ((x & (x - 1)) == 0);\n}\n\n// Is a pointer aligned?\nstatic inline bool _mi_is_aligned(void* p, size_t alignment) {\n return (alignment==0 || ((uintptr_t)p % alignment) == 0);\n}\n\n// Align upwards\nstatic inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {\n mi_assert_internal(alignment != 0);\n uintptr_t mask = alignment - 1;\n if ((alignment & mask) == 0) { // power of two?\n return ((sz + mask) & ~mask);\n }\n else {\n return (((sz + mask)/alignment)*alignment);\n }\n}\n\n// Align downwards\nstatic inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {\n mi_assert_internal(alignment != 0);\n uintptr_t mask = alignment - 1;\n if ((alignment & mask) == 0) { // power of two?\n return (sz & ~mask);\n }\n else {\n return ((sz / alignment) * alignment);\n }\n}\n\n// Align a pointer upwards\nstatic inline void* _mi_align_up_ptr(const void* p, size_t alignment) {\n return (void*)_mi_align_up((uintptr_t)p, alignment);\n}\n\n// Align a pointer downwards\nstatic inline void* _mi_align_down_ptr(void* p, size_t alignment) {\n return (void*)_mi_align_down((uintptr_t)p, alignment);\n}\n\n\n// Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`.\nstatic inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) {\n mi_assert_internal(divider != 0);\n return (divider == 0 ? size : ((size + divider - 1) / divider));\n}\n\n\n// clamp an integer\nstatic inline size_t _mi_clamp(size_t sz, size_t min, size_t max) {\n if (sz < min) return min;\n else if (sz > max) return max;\n else return sz;\n}\n\n// Is memory zero initialized?\nstatic inline bool mi_mem_is_zero(const void* p, size_t size) {\n for (size_t i = 0; i < size; i++) {\n if (((uint8_t*)p)[i] != 0) return false;\n }\n return true;\n}\n\n\n// Align a byte size to a size in _machine words_,\n// i.e. byte size == `wsize*sizeof(void*)`.\nstatic inline size_t _mi_wsize_from_size(size_t size) {\n mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t));\n return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);\n}\n\n// Overflow detecting multiply\n#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5))\n#include // UINT_MAX, ULONG_MAX\n#if defined(_CLOCK_T) // for Illumos\n#undef _CLOCK_T\n#endif\nstatic inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {\n #if (SIZE_MAX == ULONG_MAX)\n return __builtin_umull_overflow(count, size, (unsigned long *)total);\n #elif (SIZE_MAX == UINT_MAX)\n return __builtin_umul_overflow(count, size, (unsigned int *)total);\n #else\n return __builtin_umulll_overflow(count, size, (unsigned long long *)total);\n #endif\n}\n#else /* __builtin_umul_overflow is unavailable */\nstatic inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {\n #define MI_MUL_COULD_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX)\n *total = count * size;\n // note: gcc/clang optimize this to directly check the overflow flag\n return ((size >= MI_MUL_COULD_OVERFLOW || count >= MI_MUL_COULD_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count);\n}\n#endif\n\n// Safe multiply `count*size` into `total`; return `true` on overflow.\nstatic inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) {\n if (count==1) { // quick check for the case where count is one (common for C++ allocators)\n *total = size;\n return false;\n }\n else if mi_unlikely(mi_mul_overflow(count, size, total)) {\n #if MI_DEBUG > 0\n _mi_error_message(EOVERFLOW, \"allocation request is too large (%zu * %zu bytes)\\n\", count, size);\n #endif\n *total = SIZE_MAX;\n return true;\n }\n else return false;\n}\n\n\n/*----------------------------------------------------------------------------------------\n Heap functions\n------------------------------------------------------------------------------------------- */\n\nextern mi_decl_hidden const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap\n\nstatic inline bool mi_heap_is_backing(const mi_heap_t* heap) {\n return (heap->tld->heap_backing == heap);\n}\n\nstatic inline bool mi_heap_is_initialized(mi_heap_t* heap) {\n mi_assert_internal(heap != NULL);\n return (heap != NULL && heap != &_mi_heap_empty);\n}\n\nstatic inline uintptr_t _mi_ptr_cookie(const void* p) {\n extern mi_decl_hidden mi_heap_t _mi_heap_main;\n mi_assert_internal(_mi_heap_main.cookie != 0);\n return ((uintptr_t)p ^ _mi_heap_main.cookie);\n}\n\n/* -----------------------------------------------------------\n Pages\n----------------------------------------------------------- */\n\nstatic inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) {\n mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));\n const size_t idx = _mi_wsize_from_size(size);\n mi_assert_internal(idx < MI_PAGES_DIRECT);\n return heap->pages_free_direct[idx];\n}\n\n// Segment that contains the pointer\n// Large aligned blocks may be aligned at N*MI_SEGMENT_SIZE (inside a huge segment > MI_SEGMENT_SIZE),\n// and we need align \"down\" to the segment info which is `MI_SEGMENT_SIZE` bytes before it;\n// therefore we align one byte before `p`.\n// We check for NULL afterwards on 64-bit systems to improve codegen for `mi_free`.\nstatic inline mi_segment_t* _mi_ptr_segment(const void* p) {\n mi_segment_t* const segment = (mi_segment_t*)(((uintptr_t)p - 1) & ~MI_SEGMENT_MASK);\n #if MI_INTPTR_SIZE <= 4\n return (p==NULL ? NULL : segment);\n #else\n return ((intptr_t)segment <= 0 ? NULL : segment);\n #endif\n}\n\nstatic inline mi_page_t* mi_slice_to_page(mi_slice_t* s) {\n mi_assert_internal(s->slice_offset== 0 && s->slice_count > 0);\n return (mi_page_t*)(s);\n}\n\nstatic inline mi_slice_t* mi_page_to_slice(mi_page_t* p) {\n mi_assert_internal(p->slice_offset== 0 && p->slice_count > 0);\n return (mi_slice_t*)(p);\n}\n\n// Segment belonging to a page\nstatic inline mi_segment_t* _mi_page_segment(const mi_page_t* page) {\n mi_assert_internal(page!=NULL);\n mi_segment_t* segment = _mi_ptr_segment(page);\n mi_assert_internal(segment == NULL || ((mi_slice_t*)page >= segment->slices && (mi_slice_t*)page < segment->slices + segment->slice_entries));\n return segment;\n}\n\nstatic inline mi_slice_t* mi_slice_first(const mi_slice_t* slice) {\n mi_slice_t* start = (mi_slice_t*)((uint8_t*)slice - slice->slice_offset);\n mi_assert_internal(start >= _mi_ptr_segment(slice)->slices);\n mi_assert_internal(start->slice_offset == 0);\n mi_assert_internal(start + start->slice_count > slice);\n return start;\n}\n\n// Get the page containing the pointer (performance critical as it is called in mi_free)\nstatic inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) {\n mi_assert_internal(p > (void*)segment);\n ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment;\n mi_assert_internal(diff > 0 && diff <= (ptrdiff_t)MI_SEGMENT_SIZE);\n size_t idx = (size_t)diff >> MI_SEGMENT_SLICE_SHIFT;\n mi_assert_internal(idx <= segment->slice_entries);\n mi_slice_t* slice0 = (mi_slice_t*)&segment->slices[idx];\n mi_slice_t* slice = mi_slice_first(slice0); // adjust to the block that holds the page data\n mi_assert_internal(slice->slice_offset == 0);\n mi_assert_internal(slice >= segment->slices && slice < segment->slices + segment->slice_entries);\n return mi_slice_to_page(slice);\n}\n\n// Quick page start for initialized pages\nstatic inline uint8_t* mi_page_start(const mi_page_t* page) {\n mi_assert_internal(page->page_start != NULL);\n mi_assert_expensive(_mi_segment_page_start(_mi_page_segment(page),page,NULL) == page->page_start);\n return page->page_start;\n}\n\n// Get the page containing the pointer\nstatic inline mi_page_t* _mi_ptr_page(void* p) {\n mi_assert_internal(p!=NULL);\n return _mi_segment_page_of(_mi_ptr_segment(p), p);\n}\n\n// Get the block size of a page (special case for huge objects)\nstatic inline size_t mi_page_block_size(const mi_page_t* page) {\n mi_assert_internal(page->block_size > 0);\n return page->block_size;\n}\n\nstatic inline bool mi_page_is_huge(const mi_page_t* page) {\n mi_assert_internal((page->is_huge && _mi_page_segment(page)->kind == MI_SEGMENT_HUGE) ||\n (!page->is_huge && _mi_page_segment(page)->kind != MI_SEGMENT_HUGE));\n return page->is_huge;\n}\n\n// Get the usable block size of a page without fixed padding.\n// This may still include internal padding due to alignment and rounding up size classes.\nstatic inline size_t mi_page_usable_block_size(const mi_page_t* page) {\n return mi_page_block_size(page) - MI_PADDING_SIZE;\n}\n\n// size of a segment\nstatic inline size_t mi_segment_size(mi_segment_t* segment) {\n return segment->segment_slices * MI_SEGMENT_SLICE_SIZE;\n}\n\nstatic inline uint8_t* mi_segment_end(mi_segment_t* segment) {\n return (uint8_t*)segment + mi_segment_size(segment);\n}\n\n// Thread free access\nstatic inline mi_block_t* mi_page_thread_free(const mi_page_t* page) {\n return (mi_block_t*)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free) & ~3);\n}\n\nstatic inline mi_delayed_t mi_page_thread_free_flag(const mi_page_t* page) {\n return (mi_delayed_t)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free) & 3);\n}\n\n// Heap access\nstatic inline mi_heap_t* mi_page_heap(const mi_page_t* page) {\n return (mi_heap_t*)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xheap));\n}\n\nstatic inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) {\n mi_assert_internal(mi_page_thread_free_flag(page) != MI_DELAYED_FREEING);\n mi_atomic_store_release(&page->xheap,(uintptr_t)heap);\n if (heap != NULL) { page->heap_tag = heap->tag; }\n}\n\n// Thread free flag helpers\nstatic inline mi_block_t* mi_tf_block(mi_thread_free_t tf) {\n return (mi_block_t*)(tf & ~0x03);\n}\nstatic inline mi_delayed_t mi_tf_delayed(mi_thread_free_t tf) {\n return (mi_delayed_t)(tf & 0x03);\n}\nstatic inline mi_thread_free_t mi_tf_make(mi_block_t* block, mi_delayed_t delayed) {\n return (mi_thread_free_t)((uintptr_t)block | (uintptr_t)delayed);\n}\nstatic inline mi_thread_free_t mi_tf_set_delayed(mi_thread_free_t tf, mi_delayed_t delayed) {\n return mi_tf_make(mi_tf_block(tf),delayed);\n}\nstatic inline mi_thread_free_t mi_tf_set_block(mi_thread_free_t tf, mi_block_t* block) {\n return mi_tf_make(block, mi_tf_delayed(tf));\n}\n\n// are all blocks in a page freed?\n// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`.\nstatic inline bool mi_page_all_free(const mi_page_t* page) {\n mi_assert_internal(page != NULL);\n return (page->used == 0);\n}\n\n// are there any available blocks?\nstatic inline bool mi_page_has_any_available(const mi_page_t* page) {\n mi_assert_internal(page != NULL && page->reserved > 0);\n return (page->used < page->reserved || (mi_page_thread_free(page) != NULL));\n}\n\n// are there immediately available blocks, i.e. blocks available on the free list.\nstatic inline bool mi_page_immediate_available(const mi_page_t* page) {\n mi_assert_internal(page != NULL);\n return (page->free != NULL);\n}\n\n// is more than 7/8th of a page in use?\nstatic inline bool mi_page_is_mostly_used(const mi_page_t* page) {\n if (page==NULL) return true;\n uint16_t frac = page->reserved / 8U;\n return (page->reserved - page->used <= frac);\n}\n\nstatic inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) {\n return &((mi_heap_t*)heap)->pages[_mi_bin(size)];\n}\n\n\n\n//-----------------------------------------------------------\n// Page flags\n//-----------------------------------------------------------\nstatic inline bool mi_page_is_in_full(const mi_page_t* page) {\n return page->flags.x.in_full;\n}\n\nstatic inline void mi_page_set_in_full(mi_page_t* page, bool in_full) {\n if (page->flags.x.in_full != in_full) {\n // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220)\n mi_heap_t* const heap = mi_page_heap(page);\n const size_t size = page->capacity * mi_page_block_size(page);\n if (in_full) { heap->pages_full_size += size; }\n else { mi_assert_internal(size <= heap->pages_full_size); heap->pages_full_size -= size; }\n }\n page->flags.x.in_full = in_full;\n}\n\nstatic inline bool mi_page_has_aligned(const mi_page_t* page) {\n return page->flags.x.has_aligned;\n}\n\nstatic inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) {\n page->flags.x.has_aligned = has_aligned;\n}\n\n/* -------------------------------------------------------------------\n Guarded objects\n------------------------------------------------------------------- */\nstatic inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) {\n#if MI_GUARDED\n const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block;\n return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED);\n#else\n MI_UNUSED(block); MI_UNUSED(p);\n return false;\n#endif \n}\n\n#if MI_GUARDED\nstatic inline bool mi_heap_malloc_use_guarded(mi_heap_t* heap, size_t size) {\n // this code is written to result in fast assembly as it is on the hot path for allocation\n const size_t count = heap->guarded_sample_count - 1; // if the rate was 0, this will underflow and count for a long time..\n if mi_likely(count != 0) {\n // no sample\n heap->guarded_sample_count = count;\n return false;\n }\n else if (size >= heap->guarded_size_min && size <= heap->guarded_size_max) {\n // use guarded allocation\n heap->guarded_sample_count = heap->guarded_sample_rate; // reset\n return (heap->guarded_sample_rate != 0);\n }\n else {\n // failed size criteria, rewind count (but don't write to an empty heap)\n if (heap->guarded_sample_rate != 0) { heap->guarded_sample_count = 1; }\n return false;\n }\n}\n\nmi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept;\n\n#endif\n\n\n/* -------------------------------------------------------------------\nEncoding/Decoding the free list next pointers\n\nThis is to protect against buffer overflow exploits where the\nfree list is mutated. Many hardened allocators xor the next pointer `p`\nwith a secret key `k1`, as `p^k1`. This prevents overwriting with known\nvalues but might be still too weak: if the attacker can guess\nthe pointer `p` this can reveal `k1` (since `p^k1^p == k1`).\nMoreover, if multiple blocks can be read as well, the attacker can\nxor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot\nabout the pointers (and subsequently `k1`).\n\nInstead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<> (MI_INTPTR_BITS - shift))));\n}\nstatic inline uintptr_t mi_rotr(uintptr_t x, uintptr_t shift) {\n shift %= MI_INTPTR_BITS;\n return (shift==0 ? x : ((x >> shift) | (x << (MI_INTPTR_BITS - shift))));\n}\n\nstatic inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) {\n void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]);\n return (p==null ? NULL : p);\n}\n\nstatic inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) {\n uintptr_t x = (uintptr_t)(p==NULL ? null : p);\n return mi_rotl(x ^ keys[1], keys[0]) + keys[0];\n}\n\nstatic inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) {\n const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys));\n // make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951)\n #ifdef MI_BIG_ENDIAN\n return (x & 0x00FFFFFF);\n #else\n return (x & 0xFFFFFF00);\n #endif\n}\n\nstatic inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) {\n mi_track_mem_defined(block,sizeof(mi_block_t));\n mi_block_t* next;\n #ifdef MI_ENCODE_FREELIST\n next = (mi_block_t*)mi_ptr_decode(null, block->next, keys);\n #else\n MI_UNUSED(keys); MI_UNUSED(null);\n next = (mi_block_t*)block->next;\n #endif\n mi_track_mem_noaccess(block,sizeof(mi_block_t));\n return next;\n}\n\nstatic inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) {\n mi_track_mem_undefined(block,sizeof(mi_block_t));\n #ifdef MI_ENCODE_FREELIST\n block->next = mi_ptr_encode(null, next, keys);\n #else\n MI_UNUSED(keys); MI_UNUSED(null);\n block->next = (mi_encoded_t)next;\n #endif\n mi_track_mem_noaccess(block,sizeof(mi_block_t));\n}\n\nstatic inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) {\n #ifdef MI_ENCODE_FREELIST\n mi_block_t* next = mi_block_nextx(page,block,page->keys);\n // check for free list corruption: is `next` at least in the same page?\n // TODO: check if `next` is `page->block_size` aligned?\n if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) {\n _mi_error_message(EFAULT, \"corrupted free list entry of size %zub at %p: value 0x%zx\\n\", mi_page_block_size(page), block, (uintptr_t)next);\n next = NULL;\n }\n return next;\n #else\n MI_UNUSED(page);\n return mi_block_nextx(page,block,NULL);\n #endif\n}\n\nstatic inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) {\n #ifdef MI_ENCODE_FREELIST\n mi_block_set_nextx(page,block,next, page->keys);\n #else\n MI_UNUSED(page);\n mi_block_set_nextx(page,block,next,NULL);\n #endif\n}\n\n\n// -------------------------------------------------------------------\n// commit mask\n// -------------------------------------------------------------------\n\nstatic inline void mi_commit_mask_create_empty(mi_commit_mask_t* cm) {\n for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {\n cm->mask[i] = 0;\n }\n}\n\nstatic inline void mi_commit_mask_create_full(mi_commit_mask_t* cm) {\n for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {\n cm->mask[i] = ~((size_t)0);\n }\n}\n\nstatic inline bool mi_commit_mask_is_empty(const mi_commit_mask_t* cm) {\n for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {\n if (cm->mask[i] != 0) return false;\n }\n return true;\n}\n\nstatic inline bool mi_commit_mask_is_full(const mi_commit_mask_t* cm) {\n for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {\n if (cm->mask[i] != ~((size_t)0)) return false;\n }\n return true;\n}\n\n// defined in `segment.c`:\nsize_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total);\nsize_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx);\n\n#define mi_commit_mask_foreach(cm,idx,count) \\\n idx = 0; \\\n while ((count = _mi_commit_mask_next_run(cm,&idx)) > 0) {\n\n#define mi_commit_mask_foreach_end() \\\n idx += count; \\\n }\n\n\n\n/* -----------------------------------------------------------\n memory id's\n----------------------------------------------------------- */\n\nstatic inline mi_memid_t _mi_memid_create(mi_memkind_t memkind) {\n mi_memid_t memid;\n _mi_memzero_var(memid);\n memid.memkind = memkind;\n return memid;\n}\n\nstatic inline mi_memid_t _mi_memid_none(void) {\n return _mi_memid_create(MI_MEM_NONE);\n}\n\nstatic inline mi_memid_t _mi_memid_create_os(void* base, size_t size, bool committed, bool is_zero, bool is_large) {\n mi_memid_t memid = _mi_memid_create(MI_MEM_OS);\n memid.mem.os.base = base;\n memid.mem.os.size = size;\n memid.initially_committed = committed;\n memid.initially_zero = is_zero;\n memid.is_pinned = is_large;\n return memid;\n}\n\n\n// -------------------------------------------------------------------\n// Fast \"random\" shuffle\n// -------------------------------------------------------------------\n\nstatic inline uintptr_t _mi_random_shuffle(uintptr_t x) {\n if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros\n#if (MI_INTPTR_SIZE>=8)\n // by Sebastiano Vigna, see: \n x ^= x >> 30;\n x *= 0xbf58476d1ce4e5b9UL;\n x ^= x >> 27;\n x *= 0x94d049bb133111ebUL;\n x ^= x >> 31;\n#elif (MI_INTPTR_SIZE==4)\n // by Chris Wellons, see: \n x ^= x >> 16;\n x *= 0x7feb352dUL;\n x ^= x >> 15;\n x *= 0x846ca68bUL;\n x ^= x >> 16;\n#endif\n return x;\n}\n\n\n\n// -----------------------------------------------------------------------\n// Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero)\n// -----------------------------------------------------------------------\n\n#if defined(__GNUC__)\n\n#include // LONG_MAX\n#define MI_HAVE_FAST_BITSCAN\nstatic inline size_t mi_clz(size_t x) {\n if (x==0) return MI_SIZE_BITS;\n #if (SIZE_MAX == ULONG_MAX)\n return __builtin_clzl(x);\n #else\n return __builtin_clzll(x);\n #endif\n}\nstatic inline size_t mi_ctz(size_t x) {\n if (x==0) return MI_SIZE_BITS;\n #if (SIZE_MAX == ULONG_MAX)\n return __builtin_ctzl(x);\n #else\n return __builtin_ctzll(x);\n #endif\n}\n\n#elif defined(_MSC_VER)\n\n#include // LONG_MAX\n#include // BitScanReverse64\n#define MI_HAVE_FAST_BITSCAN\nstatic inline size_t mi_clz(size_t x) {\n if (x==0) return MI_SIZE_BITS;\n unsigned long idx;\n #if (SIZE_MAX == ULONG_MAX)\n _BitScanReverse(&idx, x);\n #else\n _BitScanReverse64(&idx, x);\n #endif\n return ((MI_SIZE_BITS - 1) - (size_t)idx);\n}\nstatic inline size_t mi_ctz(size_t x) {\n if (x==0) return MI_SIZE_BITS;\n unsigned long idx;\n #if (SIZE_MAX == ULONG_MAX)\n _BitScanForward(&idx, x);\n #else\n _BitScanForward64(&idx, x);\n #endif\n return (size_t)idx;\n}\n\n#else\n\nstatic inline size_t mi_ctz_generic32(uint32_t x) {\n // de Bruijn multiplication, see \n static const uint8_t debruijn[32] = {\n 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,\n 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9\n };\n if (x==0) return 32;\n return debruijn[(uint32_t)((x & -(int32_t)x) * (uint32_t)(0x077CB531U)) >> 27];\n}\n\nstatic inline size_t mi_clz_generic32(uint32_t x) {\n // de Bruijn multiplication, see \n static const uint8_t debruijn[32] = {\n 31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1,\n 23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0\n };\n if (x==0) return 32;\n x |= x >> 1;\n x |= x >> 2;\n x |= x >> 4;\n x |= x >> 8;\n x |= x >> 16;\n return debruijn[(uint32_t)(x * (uint32_t)(0x07C4ACDDU)) >> 27];\n}\n\nstatic inline size_t mi_ctz(size_t x) {\n if (x==0) return MI_SIZE_BITS;\n #if (MI_SIZE_BITS <= 32)\n return mi_ctz_generic32((uint32_t)x);\n #else\n const uint32_t lo = (uint32_t)x;\n if (lo != 0) {\n return mi_ctz_generic32(lo);\n }\n else {\n return (32 + mi_ctz_generic32((uint32_t)(x>>32)));\n }\n #endif\n}\n\nstatic inline size_t mi_clz(size_t x) {\n if (x==0) return MI_SIZE_BITS;\n #if (MI_SIZE_BITS <= 32)\n return mi_clz_generic32((uint32_t)x);\n #else\n const uint32_t hi = (uint32_t)(x>>32);\n if (hi != 0) {\n return mi_clz_generic32(hi);\n }\n else {\n return 32 + mi_clz_generic32((uint32_t)x);\n }\n #endif\n}\n\n#endif\n\n// \"bit scan reverse\": Return index of the highest bit (or MI_SIZE_BITS if `x` is zero)\nstatic inline size_t mi_bsr(size_t x) {\n return (x==0 ? MI_SIZE_BITS : MI_SIZE_BITS - 1 - mi_clz(x));\n}\n\nsize_t _mi_popcount_generic(size_t x);\n\nstatic inline size_t mi_popcount(size_t x) {\n if (x<=1) return x;\n if (x==SIZE_MAX) return MI_SIZE_BITS;\n #if defined(__GNUC__)\n #if (SIZE_MAX == ULONG_MAX)\n return __builtin_popcountl(x);\n #else\n return __builtin_popcountll(x);\n #endif\n #else\n return _mi_popcount_generic(x);\n #endif\n}\n\n// ---------------------------------------------------------------------------------\n// Provide our own `_mi_memcpy` for potential performance optimizations.\n//\n// For now, only on Windows with msvc/clang-cl we optimize to `rep movsb` if\n// we happen to run on x86/x64 cpu's that have \"fast short rep movsb\" (FSRM) support\n// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253.\n// ---------------------------------------------------------------------------------\n\n#if !MI_TRACK_ENABLED && defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64))\n#include \nextern mi_decl_hidden bool _mi_cpu_has_fsrm;\nextern mi_decl_hidden bool _mi_cpu_has_erms;\nstatic inline void _mi_memcpy(void* dst, const void* src, size_t n) {\n if (_mi_cpu_has_fsrm && n <= 127) { // || (_mi_cpu_has_erms && n > 128)) {\n __movsb((unsigned char*)dst, (const unsigned char*)src, n);\n }\n else {\n memcpy(dst, src, n);\n }\n}\nstatic inline void _mi_memzero(void* dst, size_t n) {\n if (_mi_cpu_has_fsrm && n <= 127) { // || (_mi_cpu_has_erms && n > 128)) {\n __stosb((unsigned char*)dst, 0, n);\n }\n else {\n memset(dst, 0, n);\n }\n}\n#else\nstatic inline void _mi_memcpy(void* dst, const void* src, size_t n) {\n memcpy(dst, src, n);\n}\nstatic inline void _mi_memzero(void* dst, size_t n) {\n memset(dst, 0, n);\n}\n#endif\n\n// -------------------------------------------------------------------------------\n// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned\n// This is used for example in `mi_realloc`.\n// -------------------------------------------------------------------------------\n\n#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)\n// On GCC/CLang we provide a hint that the pointers are word aligned.\nstatic inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {\n mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));\n void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);\n const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE);\n _mi_memcpy(adst, asrc, n);\n}\n\nstatic inline void _mi_memzero_aligned(void* dst, size_t n) {\n mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);\n void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);\n _mi_memzero(adst, n);\n}\n#else\n// Default fallback on `_mi_memcpy`\nstatic inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {\n mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));\n _mi_memcpy(dst, src, n);\n}\n\nstatic inline void _mi_memzero_aligned(void* dst, size_t n) {\n mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);\n _mi_memzero(dst, n);\n}\n#endif\n\n\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/include/mimalloc/prim.h", "sha256": "98c53e031ef97d47729889423d8861956006afd50d77f426a5f5982a152521e8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_PRIM_H\n#define MIMALLOC_PRIM_H\n#include \"internal.h\" // mi_decl_hidden\n\n// --------------------------------------------------------------------------\n// This file specifies the primitive portability API.\n// Each OS/host needs to implement these primitives, see `src/prim`\n// for implementations on Window, macOS, WASI, and Linux/Unix.\n//\n// note: on all primitive functions, we always have result parameters != NULL, and:\n// addr != NULL and page aligned\n// size > 0 and page aligned\n// the return value is an error code as an `int` where 0 is success\n// --------------------------------------------------------------------------\n\n// OS memory configuration\ntypedef struct mi_os_mem_config_s {\n size_t page_size; // default to 4KiB\n size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows)\n size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB)\n size_t physical_memory_in_kib; // physical memory size in KiB\n size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization)\n bool has_overcommit; // can we reserve more memory than can be actually committed?\n bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc)\n bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory)\n} mi_os_mem_config_t;\n\n// Initialize\nvoid _mi_prim_mem_init( mi_os_mem_config_t* config );\n\n// Free OS memory\nint _mi_prim_free(void* addr, size_t size );\n\n// Allocate OS memory. Return NULL on error.\n// The `try_alignment` is just a hint and the returned pointer does not have to be aligned.\n// If `commit` is false, the virtual memory range only needs to be reserved (with no access)\n// which will later be committed explicitly using `_mi_prim_commit`.\n// `is_zero` is set to true if the memory was zero initialized (as on most OS's)\n// The `hint_addr` address is either `NULL` or a preferred allocation address but can be ignored.\n// pre: !commit => !allow_large\n// try_alignment >= _mi_os_page_size() and a power of 2\nint _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr);\n\n// Commit memory. Returns error code or 0 on success.\n// For example, on Linux this would make the memory PROT_READ|PROT_WRITE.\n// `is_zero` is set to true if the memory was zero initialized (e.g. on Windows)\nint _mi_prim_commit(void* addr, size_t size, bool* is_zero);\n\n// Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true\n// if the memory would need to be re-committed. For example, on Windows this is always true,\n// but on Linux we could use MADV_DONTNEED to decommit which does not need a recommit.\n// pre: needs_recommit != NULL\nint _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit);\n\n// Reset memory. The range keeps being accessible but the content might be reset to zero at any moment.\n// Returns error code or 0 on success.\nint _mi_prim_reset(void* addr, size_t size);\n\n// Reuse memory. This is called for memory that is already committed but\n// may have been reset (`_mi_prim_reset`) or decommitted (`_mi_prim_decommit`) where `needs_recommit` was false.\n// Returns error code or 0 on success. On most platforms this is a no-op.\nint _mi_prim_reuse(void* addr, size_t size);\n\n// Protect memory. Returns error code or 0 on success.\nint _mi_prim_protect(void* addr, size_t size, bool protect);\n\n// Allocate huge (1GiB) pages possibly associated with a NUMA node.\n// `is_zero` is set to true if the memory was zero initialized (as on most OS's)\n// pre: size > 0 and a multiple of 1GiB.\n// numa_node is either negative (don't care), or a numa node number.\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr);\n\n// Return the current NUMA node\nsize_t _mi_prim_numa_node(void);\n\n// Return the number of logical NUMA nodes\nsize_t _mi_prim_numa_node_count(void);\n\n// Clock ticks\nmi_msecs_t _mi_prim_clock_now(void);\n\n// Return process information (only for statistics)\ntypedef struct mi_process_info_s {\n mi_msecs_t elapsed;\n mi_msecs_t utime;\n mi_msecs_t stime;\n size_t current_rss;\n size_t peak_rss;\n size_t current_commit;\n size_t peak_commit;\n size_t page_faults;\n} mi_process_info_t;\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo);\n\n// Default stderr output. (only for warnings etc. with verbose enabled)\n// msg != NULL && _mi_strlen(msg) > 0\nvoid _mi_prim_out_stderr( const char* msg );\n\n// Get an environment variable. (only for options)\n// name != NULL, result != NULL, result_size >= 64\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size);\n\n\n// Fill a buffer with strong randomness; return `false` on error or if\n// there is no strong randomization available.\nbool _mi_prim_random_buf(void* buf, size_t buf_len);\n\n// Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination.\nvoid _mi_prim_thread_init_auto_done(void);\n\n// Called on process exit and may take action to clean up resources associated with the thread auto done.\nvoid _mi_prim_thread_done_auto_done(void);\n\n// Called when the default heap for a thread changes\nvoid _mi_prim_thread_associate_default_heap(mi_heap_t* heap);\n\n\n//-------------------------------------------------------------------\n// Access to TLS (thread local storage) slots.\n// We need fast access to both a unique thread id (in `free.c:mi_free`) and\n// to a thread-local heap pointer (in `alloc.c:mi_malloc`).\n// To achieve this we use specialized code for various platforms.\n//-------------------------------------------------------------------\n\n// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot.\n// The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform.\n// If you test on another platform and it works please send a PR :-)\n// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.\n//\n// Note: we would like to prefer `__builtin_thread_pointer()` nowadays instead of using assembly,\n// but unfortunately we can not detect support reliably (see issue #883)\n// We also use it on Apple OS as we use a TLS slot for the default heap there.\n#if defined(__GNUC__) && ( \\\n (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \\\n || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \\\n || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \\\n || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \\\n || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \\\n )\n\n#define MI_HAS_TLS_SLOT 1\n\nstatic inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {\n void* res;\n const size_t ofs = (slot*sizeof(void*));\n #if defined(__i386__)\n __asm__(\"movl %%gs:%1, %0\" : \"=r\" (res) : \"m\" (*((void**)ofs)) : ); // x86 32-bit always uses GS\n #elif defined(__APPLE__) && defined(__x86_64__)\n __asm__(\"movq %%gs:%1, %0\" : \"=r\" (res) : \"m\" (*((void**)ofs)) : ); // x86_64 macOSX uses GS\n #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)\n __asm__(\"movl %%fs:%1, %0\" : \"=r\" (res) : \"m\" (*((void**)ofs)) : ); // x32 ABI\n #elif defined(__x86_64__)\n __asm__(\"movq %%fs:%1, %0\" : \"=r\" (res) : \"m\" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS\n #elif defined(__arm__)\n void** tcb; MI_UNUSED(ofs);\n __asm__ volatile (\"mrc p15, 0, %0, c13, c0, 3\\nbic %0, %0, #3\" : \"=r\" (tcb));\n res = tcb[slot];\n #elif defined(__aarch64__)\n void** tcb; MI_UNUSED(ofs);\n #if defined(__APPLE__) // M1, issue #343\n __asm__ volatile (\"mrs %0, tpidrro_el0\\nbic %0, %0, #7\" : \"=r\" (tcb));\n #else\n __asm__ volatile (\"mrs %0, tpidr_el0\" : \"=r\" (tcb));\n #endif\n res = tcb[slot];\n #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781\n MI_UNUSED(ofs);\n res = pthread_getspecific(slot);\n #endif\n return res;\n}\n\n// setting a tls slot is only used on macOS for now\nstatic inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {\n const size_t ofs = (slot*sizeof(void*));\n #if defined(__i386__)\n __asm__(\"movl %1,%%gs:%0\" : \"=m\" (*((void**)ofs)) : \"rn\" (value) : ); // 32-bit always uses GS\n #elif defined(__APPLE__) && defined(__x86_64__)\n __asm__(\"movq %1,%%gs:%0\" : \"=m\" (*((void**)ofs)) : \"rn\" (value) : ); // x86_64 macOS uses GS\n #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)\n __asm__(\"movl %1,%%fs:%0\" : \"=m\" (*((void**)ofs)) : \"rn\" (value) : ); // x32 ABI\n #elif defined(__x86_64__)\n __asm__(\"movq %1,%%fs:%0\" : \"=m\" (*((void**)ofs)) : \"rn\" (value) : ); // x86_64 Linux, BSD uses FS\n #elif defined(__arm__)\n void** tcb; MI_UNUSED(ofs);\n __asm__ volatile (\"mrc p15, 0, %0, c13, c0, 3\\nbic %0, %0, #3\" : \"=r\" (tcb));\n tcb[slot] = value;\n #elif defined(__aarch64__)\n void** tcb; MI_UNUSED(ofs);\n #if defined(__APPLE__) // M1, issue #343\n __asm__ volatile (\"mrs %0, tpidrro_el0\\nbic %0, %0, #7\" : \"=r\" (tcb));\n #else\n __asm__ volatile (\"mrs %0, tpidr_el0\" : \"=r\" (tcb));\n #endif\n tcb[slot] = value;\n #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781\n MI_UNUSED(ofs);\n pthread_setspecific(slot, value);\n #endif\n}\n\n#elif _WIN32 && MI_WIN_USE_FIXED_TLS && !defined(MI_WIN_USE_FLS)\n\n// On windows we can store the thread-local heap at a fixed TLS slot to avoid\n// thread-local initialization checks in the fast path.\n// We allocate a user TLS slot at process initialization (see `windows/prim.c`)\n// and store the offset `_mi_win_tls_offset`.\n#define MI_HAS_TLS_SLOT 1 // 2 = we can reliably initialize the slot (saving a test on each malloc)\n\nextern mi_decl_hidden size_t _mi_win_tls_offset;\n\n#if MI_WIN_USE_FIXED_TLS > 1\n#define MI_TLS_SLOT (MI_WIN_USE_FIXED_TLS)\n#elif MI_SIZE_SIZE == 4\n#define MI_TLS_SLOT (0x0E10 + _mi_win_tls_offset) // User TLS slots \n#else\n#define MI_TLS_SLOT (0x1480 + _mi_win_tls_offset) // User TLS slots \n#endif\n\nstatic inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {\n #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)\n return (void*)__readgsqword((unsigned long)slot); // direct load at offset from gs\n #elif _M_IX86 && !defined(_M_ARM64EC)\n return (void*)__readfsdword((unsigned long)slot); // direct load at offset from fs\n #else\n return ((void**)NtCurrentTeb())[slot / sizeof(void*)];\n #endif\n}\nstatic inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {\n ((void**)NtCurrentTeb())[slot / sizeof(void*)] = value;\n}\n\n#endif\n\n\n\n//-------------------------------------------------------------------\n// Get a fast unique thread id.\n//\n// Getting the thread id should be performant as it is called in the\n// fast path of `_mi_free` and we specialize for various platforms as\n// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.\n// We only require _mi_prim_thread_id() to return a unique id\n// for each thread (unequal to zero).\n//-------------------------------------------------------------------\n\n\n// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id\n// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)\n// Nevertheless, it seems needed on older graviton platforms (see issue #851).\n// For now, we only enable this for specific platforms.\n#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */\n #if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly ()*/ \\\n && !defined(__CYGWIN__) \\\n && !defined(MI_LIBC_MUSL) \\\n && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot () */\n #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \\\n || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \\\n || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))\n #define MI_USE_BUILTIN_THREAD_POINTER 1\n #endif\n #endif\n#endif\n\n\n\n// defined in `init.c`; do not use these directly\nextern mi_decl_hidden mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from\nextern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called?\n\nstatic inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept;\n\n// Get a unique id for the current thread.\n#if defined(MI_PRIM_THREAD_ID)\n\nstatic inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {\n const mi_threadid_t tid = MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)\n mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits\n return tid;\n}\n\n#elif defined(_WIN32)\n\nstatic inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {\n // Windows: works on Intel and ARM in both 32- and 64-bit\n return (uintptr_t)NtCurrentTeb();\n}\n\n#elif MI_USE_BUILTIN_THREAD_POINTER\n\nstatic inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {\n // Works on most Unix based platforms with recent compilers\n return (uintptr_t)__builtin_thread_pointer();\n}\n\n#elif MI_HAS_TLS_SLOT\n\nstatic inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {\n #if defined(__BIONIC__)\n // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id\n // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86\n return (uintptr_t)mi_prim_tls_slot(1);\n #else\n // in all our other targets, slot 0 is the thread id\n // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h\n // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36\n return (uintptr_t)mi_prim_tls_slot(0);\n #endif\n}\n\n#else\n\n// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).\nstatic inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {\n return (uintptr_t)&_mi_heap_default;\n}\n\n#endif\n\n\n\n/* ----------------------------------------------------------------------------------------\nGet the thread local default heap: `_mi_prim_get_default_heap()`\n\nThis is inlined here as it is on the fast path for allocation functions.\n\nOn most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a\n__thread local variable (`_mi_heap_default`). With the initial-exec TLS model this ensures\nthat the storage will always be available (allocated on the thread stacks).\n\nOn some platforms though we cannot use that when overriding `malloc` since the underlying\nTLS implementation (or the loader) will call itself `malloc` on a first access and recurse.\nWe try to circumvent this in an efficient way:\n- macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the\n loader itself calls `malloc` even before the modules are initialized.\n- OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS).\n- DragonFly: defaults are working but seem slow compared to freeBSD (see PR #323)\n------------------------------------------------------------------------------------------- */\n\nstatic inline mi_heap_t* mi_prim_get_default_heap(void);\n\n#if defined(MI_MALLOC_OVERRIDE)\n#if defined(__APPLE__) // macOS\n #define MI_TLS_SLOT 89 // seems unused?\n // other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89)\n // see \n#elif defined(__OpenBSD__)\n // use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16)\n // see \n #define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24)\n // #elif defined(__DragonFly__)\n // #warning \"mimalloc is not working correctly on DragonFly yet.\"\n // #define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) \n#elif defined(__ANDROID__)\n // See issue #381\n #define MI_TLS_PTHREAD\n#endif\n#endif\n\n\n#if MI_TLS_SLOT\n# if !defined(MI_HAS_TLS_SLOT)\n# error \"trying to use a TLS slot for the default heap, but the mi_prim_tls_slot primitives are not defined\"\n# endif\n\nstatic inline mi_heap_t* mi_prim_get_default_heap(void) {\n mi_heap_t* heap = (mi_heap_t*)mi_prim_tls_slot(MI_TLS_SLOT);\n #if MI_HAS_TLS_SLOT == 1 // check if the TLS slot is initialized\n if mi_unlikely(heap == NULL) {\n #ifdef __GNUC__\n __asm(\"\"); // prevent conditional load of the address of _mi_heap_empty\n #endif\n heap = (mi_heap_t*)&_mi_heap_empty;\n }\n #endif\n return heap;\n}\n\n#elif defined(MI_TLS_PTHREAD_SLOT_OFS)\n\nstatic inline mi_heap_t** mi_prim_tls_pthread_heap_slot(void) {\n pthread_t self = pthread_self();\n #if defined(__DragonFly__)\n if (self==NULL) return NULL;\n #endif\n return (mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS);\n}\n\nstatic inline mi_heap_t* mi_prim_get_default_heap(void) {\n mi_heap_t** pheap = mi_prim_tls_pthread_heap_slot();\n if mi_unlikely(pheap == NULL) return _mi_heap_main_get();\n mi_heap_t* heap = *pheap;\n if mi_unlikely(heap == NULL) return (mi_heap_t*)&_mi_heap_empty;\n return heap;\n}\n\n#elif defined(MI_TLS_PTHREAD)\n\nextern mi_decl_hidden pthread_key_t _mi_heap_default_key;\nstatic inline mi_heap_t* mi_prim_get_default_heap(void) {\n mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? _mi_heap_main_get() : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key));\n return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap);\n}\n\n#else // default using a thread local variable; used on most platforms.\n\nstatic inline mi_heap_t* mi_prim_get_default_heap(void) {\n #if defined(MI_TLS_RECURSE_GUARD)\n if (mi_unlikely(!_mi_process_is_initialized)) return _mi_heap_main_get();\n #endif\n return _mi_heap_default;\n}\n\n#endif // mi_prim_get_default_heap()\n\n\n#endif // MIMALLOC_PRIM_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/include/mimalloc/track.h", "sha256": "4a1587dd06f7600b887b9a19e4103c65e165d680f93a359e47860baf97c7c78f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_TRACK_H\n#define MIMALLOC_TRACK_H\n\n/* ------------------------------------------------------------------------------------------------------\nTrack memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers.\nThese can be defined for tracking allocation:\n\n #define mi_track_malloc_size(p,reqsize,size,zero)\n #define mi_track_free_size(p,_size)\n\nThe macros are set up such that the size passed to `mi_track_free_size`\nalways matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`).\nThe `reqsize` is what the user requested, and `size >= reqsize`.\nThe `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled,\nor otherwise it is the usable block size which may be larger than the original request.\nUse `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc).\nThe `zero` parameter is `true` if the allocated block is zero initialized.\n\nOptional:\n\n #define mi_track_align(p,alignedp,offset,size)\n #define mi_track_resize(p,oldsize,newsize)\n #define mi_track_init()\n\nThe `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block.\nThe corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`).\nThe `mi_track_resize` is currently unused but could be called on reallocations within a block.\n`mi_track_init` is called at program start.\n\nThe following macros are for tools like asan and valgrind to track whether memory is\ndefined, undefined, or not accessible at all:\n\n #define mi_track_mem_defined(p,size)\n #define mi_track_mem_undefined(p,size)\n #define mi_track_mem_noaccess(p,size)\n\n-------------------------------------------------------------------------------------------------------*/\n\n#if MI_TRACK_VALGRIND\n// valgrind tool\n\n#define MI_TRACK_ENABLED 1\n#define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on heap_destroy\n#define MI_TRACK_TOOL \"valgrind\"\n\n#include \n#include \n\n#define mi_track_malloc_size(p,reqsize,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero)\n#define mi_track_free_size(p,_size) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/)\n#define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/)\n#define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size)\n#define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size)\n#define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size)\n\n#elif MI_TRACK_ASAN\n// address sanitizer\n\n#define MI_TRACK_ENABLED 1\n#define MI_TRACK_HEAP_DESTROY 0\n#define MI_TRACK_TOOL \"asan\"\n\n#include \n\n#define mi_track_malloc_size(p,reqsize,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size)\n#define mi_track_free_size(p,size) ASAN_POISON_MEMORY_REGION(p,size)\n#define mi_track_mem_defined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size)\n#define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size)\n#define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size)\n\n#elif MI_TRACK_ETW\n// windows event tracing\n\n#define MI_TRACK_ENABLED 1\n#define MI_TRACK_HEAP_DESTROY 1\n#define MI_TRACK_TOOL \"ETW\"\n\n#include \"../src/prim/windows/etw.h\"\n\n#define mi_track_init() EventRegistermicrosoft_windows_mimalloc()\n#define mi_track_done() EventUnregistermicrosoft_windows_mimalloc()\n#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size)\n#define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)(p), size)\n\n#else\n// no tracking\n\n#define MI_TRACK_ENABLED 0\n#define MI_TRACK_HEAP_DESTROY 0\n#define MI_TRACK_TOOL \"none\"\n\n#define mi_track_malloc_size(p,reqsize,size,zero)\n#define mi_track_free_size(p,_size)\n\n#endif\n\n// -------------------\n// Utility definitions\n\n#ifndef mi_track_resize\n#define mi_track_resize(p,oldsize,newsize) do{ mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false); } while(0)\n#endif\n\n#ifndef mi_track_align\n#define mi_track_align(p,alignedp,offset,size) mi_track_mem_noaccess(p,offset)\n#endif\n\n#ifndef mi_track_init\n#define mi_track_init()\n#endif\n\n#ifndef mi_track_done\n#define mi_track_done()\n#endif\n\n#ifndef mi_track_mem_defined\n#define mi_track_mem_defined(p,size)\n#endif\n\n#ifndef mi_track_mem_undefined\n#define mi_track_mem_undefined(p,size)\n#endif\n\n#ifndef mi_track_mem_noaccess\n#define mi_track_mem_noaccess(p,size)\n#endif\n\n\n#if MI_PADDING\n#define mi_track_malloc(p,reqsize,zero) \\\n if ((p)!=NULL) { \\\n mi_assert_internal(mi_usable_size(p)==(reqsize)); \\\n mi_track_malloc_size(p,reqsize,reqsize,zero); \\\n }\n#else\n#define mi_track_malloc(p,reqsize,zero) \\\n if ((p)!=NULL) { \\\n mi_assert_internal(mi_usable_size(p)>=(reqsize)); \\\n mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \\\n }\n#endif\n\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/include/mimalloc/types.h", "sha256": "449ac01b7e54cc8e350c1fbeaa033e57cab22011cb7ab9be39f86de5a2623671", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_TYPES_H\n#define MIMALLOC_TYPES_H\n\n// --------------------------------------------------------------------------\n// This file contains the main type definitions for mimalloc:\n// mi_heap_t : all data for a thread-local heap, contains\n// lists of all managed heap pages.\n// mi_segment_t : a larger chunk of memory (32MiB on 64-bit) from where pages\n// are allocated. A segment is divided in slices (64KiB) from\n// which pages are allocated.\n// mi_page_t : a \"mimalloc\" page (usually 64KiB or 512KiB) from\n// where objects are allocated.\n// Note: we write \"OS page\" for OS memory pages while\n// using plain \"page\" for mimalloc pages (`mi_page_t`).\n// --------------------------------------------------------------------------\n\n\n#include \n#include // ptrdiff_t\n#include // uintptr_t, uint16_t, etc\n#include // bool\n#include \"atomic.h\" // _Atomic\n\n#ifdef _MSC_VER\n#pragma warning(disable:4214) // bitfield is not int\n#endif\n\n// Minimal alignment necessary. On most platforms 16 bytes are needed\n// due to SSE registers for example. This must be at least `sizeof(void*)`\n#ifndef MI_MAX_ALIGN_SIZE\n#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t)\n#endif\n\n// ------------------------------------------------------\n// Variants\n// ------------------------------------------------------\n\n// Define NDEBUG in the release version to disable assertions.\n// #define NDEBUG\n\n// Define MI_TRACK_ to enable tracking support\n// #define MI_TRACK_VALGRIND 1\n// #define MI_TRACK_ASAN 1\n// #define MI_TRACK_ETW 1\n\n// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).\n// #define MI_STAT 1\n\n// Define MI_SECURE to enable security mitigations\n// #define MI_SECURE 1 // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption\n// #define MI_SECURE 2 // randomize relative allocation addresses (within mimalloc pages)\n// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)\n// #define MI_SECURE 4 // checks for double free (may be more expensive) (`-DMI_SECURE=ON`)\n// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`)\n\n#if !defined(MI_SECURE)\n#define MI_SECURE 0\n#endif\n\n// Define MI_DEBUG for assertion and invariant checking\n// #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON)\n// #define MI_DEBUG 2 // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON)\n// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON)\n#if !defined(MI_DEBUG)\n#if defined(MI_BUILD_RELEASE) || defined(NDEBUG)\n#define MI_DEBUG 0\n#else\n#define MI_DEBUG 2\n#endif\n#endif\n\n// Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options)\n#if !defined(MI_GUARDED) && MI_DEBUG\n#define MI_GUARDED 1\n#endif\n\n// Reserve extra padding at the end of each block to be more resilient against heap block overflows.\n// The padding can detect buffer overflow on free.\n#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW))\n#define MI_PADDING 1\n#endif\n\n// Check padding bytes; allows byte-precise buffer overflow detection\n#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1)\n#define MI_PADDING_CHECK 1\n#endif\n\n\n// Encoded free lists allow detection of corrupted free lists\n// and can detect buffer overflows, modify after free, and double `free`s.\n#if (MI_SECURE>=3 || MI_DEBUG>=1)\n#define MI_ENCODE_FREELIST 1\n#endif\n\n\n// We used to abandon huge pages in order to eagerly deallocate it if freed from another thread.\n// Unfortunately, that makes it not possible to visit them during a heap walk or include them in a\n// `mi_heap_destroy`. We therefore instead reset/decommit the huge blocks nowadays if freed from\n// another thread so the memory becomes \"virtually\" available (and eventually gets properly freed by\n// the owning thread).\n// #define MI_HUGE_PAGE_ABANDON 1\n\n\n// ------------------------------------------------------\n// Platform specific values\n// ------------------------------------------------------\n\n// ------------------------------------------------------\n// Size of a pointer.\n// We assume that `sizeof(void*)==sizeof(intptr_t)`\n// and it holds for all platforms we know of.\n//\n// However, the C standard only requires that:\n// p == (void*)((intptr_t)p))\n// but we also need:\n// i == (intptr_t)((void*)i)\n// or otherwise one might define an intptr_t type that is larger than a pointer...\n// ------------------------------------------------------\n\n#if INTPTR_MAX > INT64_MAX\n# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example)\n#elif INTPTR_MAX == INT64_MAX\n# define MI_INTPTR_SHIFT (3)\n#elif INTPTR_MAX == INT32_MAX\n# define MI_INTPTR_SHIFT (2)\n#else\n#error platform pointers must be 32, 64, or 128 bits\n#endif\n\n#if SIZE_MAX == UINT64_MAX\n# define MI_SIZE_SHIFT (3)\ntypedef int64_t mi_ssize_t;\n#elif SIZE_MAX == UINT32_MAX\n# define MI_SIZE_SHIFT (2)\ntypedef int32_t mi_ssize_t;\n#else\n#error platform objects must be 32 or 64 bits\n#endif\n\n#if (SIZE_MAX/2) > LONG_MAX\n# define MI_ZU(x) x##ULL\n# define MI_ZI(x) x##LL\n#else\n# define MI_ZU(x) x##UL\n# define MI_ZI(x) x##L\n#endif\n\n#define MI_INTPTR_SIZE (1< 4\n#define MI_SEGMENT_SHIFT ( 9 + MI_SEGMENT_SLICE_SHIFT) // 32MiB\n#else\n#define MI_SEGMENT_SHIFT ( 7 + MI_SEGMENT_SLICE_SHIFT) // 4MiB on 32-bit\n#endif\n#endif\n\n#ifndef MI_SMALL_PAGE_SHIFT\n#define MI_SMALL_PAGE_SHIFT (MI_SEGMENT_SLICE_SHIFT) // 64KiB\n#endif\n#ifndef MI_MEDIUM_PAGE_SHIFT\n#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB\n#endif\n\n// Derived constants\n#define MI_SEGMENT_SIZE (MI_ZU(1)<= 655360)\n#error \"mimalloc internal: define more bins\"\n#endif\n\n// Maximum block size for which blocks are guaranteed to be block size aligned. (see `segment.c:_mi_segment_page_start`)\n#define MI_MAX_ALIGN_GUARANTEE (MI_MEDIUM_OBJ_SIZE_MAX)\n\n// Alignments over MI_BLOCK_ALIGNMENT_MAX are allocated in dedicated huge page segments\n#define MI_BLOCK_ALIGNMENT_MAX (MI_SEGMENT_SIZE >> 1)\n\n// Maximum slice count (255) for which we can find the page for interior pointers\n#define MI_MAX_SLICE_OFFSET_COUNT ((MI_BLOCK_ALIGNMENT_MAX / MI_SEGMENT_SLICE_SIZE) - 1)\n\n// we never allocate more than PTRDIFF_MAX (see also )\n// on 64-bit+ systems we also limit the maximum allocation size such that the slice count fits in 32-bits. (issue #877)\n#if (PTRDIFF_MAX > INT32_MAX) && (PTRDIFF_MAX >= (MI_SEGMENT_SLIZE_SIZE * UINT32_MAX))\n#define MI_MAX_ALLOC_SIZE (MI_SEGMENT_SLICE_SIZE * (UINT32_MAX-1))\n#else\n#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX\n#endif\n\n\n// ------------------------------------------------------\n// Mimalloc pages contain allocated blocks\n// ------------------------------------------------------\n\n// The free lists use encoded next fields\n// (Only actually encodes when MI_ENCODED_FREELIST is defined.)\ntypedef uintptr_t mi_encoded_t;\n\n// thread id's\ntypedef size_t mi_threadid_t;\n\n// free lists contain blocks\ntypedef struct mi_block_s {\n mi_encoded_t next;\n} mi_block_t;\n\n#if MI_GUARDED\n// we always align guarded pointers in a block at an offset\n// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones\n#define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0))\n#define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED)\n#endif\n\n\n// The delayed flags are used for efficient multi-threaded free-ing\ntypedef enum mi_delayed_e {\n MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list\n MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap\n MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list\n MI_NEVER_DELAYED_FREE = 3 // sticky: used for abandoned pages without a owning heap; this only resets on page reclaim\n} mi_delayed_t;\n\n\n// The `in_full` and `has_aligned` page flags are put in a union to efficiently\n// test if both are false (`full_aligned == 0`) in the `mi_free` routine.\n#if !MI_TSAN\ntypedef union mi_page_flags_s {\n uint8_t full_aligned;\n struct {\n uint8_t in_full : 1;\n uint8_t has_aligned : 1;\n } x;\n} mi_page_flags_t;\n#else\n// under thread sanitizer, use a byte for each flag to suppress warning, issue #130\ntypedef union mi_page_flags_s {\n uint32_t full_aligned;\n struct {\n uint8_t in_full;\n uint8_t has_aligned;\n } x;\n} mi_page_flags_t;\n#endif\n\n// Thread free list.\n// We use the bottom 2 bits of the pointer for mi_delayed_t flags\ntypedef uintptr_t mi_thread_free_t;\n\n// A page contains blocks of one specific size (`block_size`).\n// Each page has three list of free blocks:\n// `free` for blocks that can be allocated,\n// `local_free` for freed blocks that are not yet available to `mi_malloc`\n// `thread_free` for freed blocks by other threads\n// The `local_free` and `thread_free` lists are migrated to the `free` list\n// when it is exhausted. The separate `local_free` list is necessary to\n// implement a monotonic heartbeat. The `thread_free` list is needed for\n// avoiding atomic operations in the common case.\n//\n// `used - |thread_free|` == actual blocks that are in use (alive)\n// `used - |thread_free| + |free| + |local_free| == capacity`\n//\n// We don't count `freed` (as |free|) but use `used` to reduce\n// the number of memory accesses in the `mi_page_all_free` function(s).\n//\n// Notes:\n// - Access is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc`\n// - Using `uint16_t` does not seem to slow things down\n// - The size is 12 words on 64-bit which helps the page index calculations\n// (and 14 words on 32-bit, and encoded free lists add 2 words)\n// - `xthread_free` uses the bottom bits as a delayed-free flags to optimize\n// concurrent frees where only the first concurrent free adds to the owning\n// heap `thread_delayed_free` list (see `free.c:mi_free_block_mt`).\n// The invariant is that no-delayed-free is only set if there is\n// at least one block that will be added, or as already been added, to\n// the owning heap `thread_delayed_free` list. This guarantees that pages\n// will be freed correctly even if only other threads free blocks.\ntypedef struct mi_page_s {\n // \"owned\" by the segment\n uint32_t slice_count; // slices in this page (0 if not a page)\n uint32_t slice_offset; // distance from the actual page data slice (0 if a page)\n uint8_t is_committed:1; // `true` if the page virtual memory is committed\n uint8_t is_zero_init:1; // `true` if the page was initially zero initialized\n uint8_t is_huge:1; // `true` if the page is in a huge segment (`segment->kind == MI_SEGMENT_HUGE`)\n // padding\n // layout like this to optimize access in `mi_malloc` and `mi_free`\n uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear`\n uint16_t reserved; // number of blocks reserved in memory\n mi_page_flags_t flags; // `in_full` and `has_aligned` flags (8 bits)\n uint8_t free_is_zero:1; // `true` if the blocks in the free list are zero initialized\n uint8_t retire_expire:7; // expiration count for retired blocks\n\n mi_block_t* free; // list of available free blocks (`malloc` allocates from this list)\n mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)\n uint16_t used; // number of blocks in use (including blocks in `thread_free`)\n uint8_t block_size_shift; // if not zero, then `(1 << block_size_shift) == block_size` (only used for fast path in `free.c:_mi_page_ptr_unalign`)\n uint8_t heap_tag; // tag of the owning heap, used to separate heaps by object type\n // padding\n size_t block_size; // size available in each block (always `>0`)\n uint8_t* page_start; // start of the page area containing the blocks\n\n #if (MI_ENCODE_FREELIST || MI_PADDING)\n uintptr_t keys[2]; // two random keys to encode the free lists (see `_mi_block_next`) or padding canary\n #endif\n\n _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads\n _Atomic(uintptr_t) xheap;\n\n struct mi_page_s* next; // next page owned by this thread with the same `block_size`\n struct mi_page_s* prev; // previous page owned by this thread with the same `block_size`\n\n // 64-bit 11 words, 32-bit 13 words, (+2 for secure)\n void* padding[1];\n} mi_page_t;\n\n\n\n// ------------------------------------------------------\n// Mimalloc segments contain mimalloc pages\n// ------------------------------------------------------\n\ntypedef enum mi_page_kind_e {\n MI_PAGE_SMALL, // small blocks go into 64KiB pages inside a segment\n MI_PAGE_MEDIUM, // medium blocks go into 512KiB pages inside a segment\n MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment\n MI_PAGE_HUGE // a huge page is a single page in a segment of variable size\n // used for blocks `> MI_LARGE_OBJ_SIZE_MAX` or an alignment `> MI_BLOCK_ALIGNMENT_MAX`.\n} mi_page_kind_t;\n\ntypedef enum mi_segment_kind_e {\n MI_SEGMENT_NORMAL, // MI_SEGMENT_SIZE size with pages inside.\n MI_SEGMENT_HUGE, // segment with just one huge page inside.\n} mi_segment_kind_t;\n\n// ------------------------------------------------------\n// A segment holds a commit mask where a bit is set if\n// the corresponding MI_COMMIT_SIZE area is committed.\n// The MI_COMMIT_SIZE must be a multiple of the slice\n// size. If it is equal we have the most fine grained\n// decommit (but setting it higher can be more efficient).\n// The MI_MINIMAL_COMMIT_SIZE is the minimal amount that will\n// be committed in one go which can be set higher than\n// MI_COMMIT_SIZE for efficiency (while the decommit mask\n// is still tracked in fine-grained MI_COMMIT_SIZE chunks)\n// ------------------------------------------------------\n\n#define MI_MINIMAL_COMMIT_SIZE (1*MI_SEGMENT_SLICE_SIZE)\n#define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE) // 64KiB\n#define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE)\n#define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS\n#define MI_COMMIT_MASK_FIELD_COUNT (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS)\n\n#if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_FIELD_COUNT * MI_COMMIT_MASK_FIELD_BITS))\n#error \"the segment size must be exactly divisible by the (commit size * size_t bits)\"\n#endif\n\ntypedef struct mi_commit_mask_s {\n size_t mask[MI_COMMIT_MASK_FIELD_COUNT];\n} mi_commit_mask_t;\n\ntypedef mi_page_t mi_slice_t;\ntypedef int64_t mi_msecs_t;\n\n\n// ---------------------------------------------------------------\n// a memory id tracks the provenance of arena/OS allocated memory\n// ---------------------------------------------------------------\n\n// Memory can reside in arena's, direct OS allocated, or statically allocated. The memid keeps track of this.\ntypedef enum mi_memkind_e {\n MI_MEM_NONE, // not allocated\n MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example)\n MI_MEM_STATIC, // allocated in a static area and should not be freed (for arena meta data for example)\n MI_MEM_OS, // allocated from the OS\n MI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory)\n MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`)\n MI_MEM_ARENA // allocated from an arena (the usual case)\n} mi_memkind_t;\n\nstatic inline bool mi_memkind_is_os(mi_memkind_t memkind) {\n return (memkind >= MI_MEM_OS && memkind <= MI_MEM_OS_REMAP);\n}\n\ntypedef struct mi_memid_os_info {\n void* base; // actual base address of the block (used for offset aligned allocations)\n size_t size; // full allocation size\n} mi_memid_os_info_t;\n\ntypedef struct mi_memid_arena_info {\n size_t block_index; // index in the arena\n mi_arena_id_t id; // arena id (>= 1)\n bool is_exclusive; // this arena can only be used for specific arena allocations\n} mi_memid_arena_info_t;\n\ntypedef struct mi_memid_s {\n union {\n mi_memid_os_info_t os; // only used for MI_MEM_OS\n mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA\n } mem;\n bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages)\n bool initially_committed;// `true` if the memory was originally allocated as committed\n bool initially_zero; // `true` if the memory was originally zero initialized\n mi_memkind_t memkind;\n} mi_memid_t;\n\n\n// -----------------------------------------------------------------------------------------\n// Segments are large allocated memory blocks (32mb on 64 bit) from arenas or the OS.\n//\n// Inside segments we allocated fixed size mimalloc pages (`mi_page_t`) that contain blocks.\n// The start of a segment is this structure with a fixed number of slice entries (`slices`)\n// usually followed by a guard OS page and the actual allocation area with pages.\n// While a page is not allocated, we view it's data as a `mi_slice_t` (instead of a `mi_page_t`).\n// Of any free area, the first slice has the info and `slice_offset == 0`; for any subsequent\n// slices part of the area, the `slice_offset` is the byte offset back to the first slice\n// (so we can quickly find the page info on a free, `internal.h:_mi_segment_page_of`).\n// For slices, the `block_size` field is repurposed to signify if a slice is used (`1`) or not (`0`).\n// Small and medium pages use a fixed amount of slices to reduce slice fragmentation, while\n// large and huge pages span a variable amount of slices.\n\ntypedef struct mi_subproc_s mi_subproc_t;\n\ntypedef struct mi_segment_s {\n // constant fields\n mi_memid_t memid; // memory id for arena/OS allocation\n bool allow_decommit; // can we decommmit the memory\n bool allow_purge; // can we purge the memory (reset or decommit)\n size_t segment_size;\n mi_subproc_t* subproc; // segment belongs to sub process\n\n // segment fields\n mi_msecs_t purge_expire; // purge slices in the `purge_mask` after this time\n mi_commit_mask_t purge_mask; // slices that can be purged\n mi_commit_mask_t commit_mask; // slices that are currently committed\n\n // from here is zero initialized\n struct mi_segment_s* next; // the list of freed segments in the cache (must be first field, see `segment.c:mi_segment_init`)\n bool was_reclaimed; // true if it was reclaimed (used to limit on-free reclamation)\n bool dont_free; // can be temporarily true to ensure the segment is not freed\n bool free_is_zero; // if free spans are zero\n\n size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`)\n size_t abandoned_visits; // count how often this segment is visited during abandoned reclamation (to force reclaim if it takes too long)\n size_t used; // count of pages in use\n uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie`\n\n struct mi_segment_s* abandoned_os_next; // only used for abandoned segments outside arena's, and only if `mi_option_visit_abandoned` is enabled\n struct mi_segment_s* abandoned_os_prev;\n\n size_t segment_slices; // for huge segments this may be different from `MI_SLICES_PER_SEGMENT`\n size_t segment_info_slices; // initial count of slices that we are using for segment info and possible guard pages.\n\n // layout like this to optimize access in `mi_free`\n mi_segment_kind_t kind;\n size_t slice_entries; // entries in the `slices` array, at most `MI_SLICES_PER_SEGMENT`\n _Atomic(mi_threadid_t) thread_id; // unique id of the thread owning this segment\n\n mi_slice_t slices[MI_SLICES_PER_SEGMENT+1]; // one extra final entry for huge blocks with large alignment\n} mi_segment_t;\n\n\n// ------------------------------------------------------\n// Heaps\n// Provide first-class heaps to allocate from.\n// A heap just owns a set of pages for allocation and\n// can only be allocate/reallocate from the thread that created it.\n// Freeing blocks can be done from any thread though.\n// Per thread, the segments are shared among its heaps.\n// Per thread, there is always a default heap that is\n// used for allocation; it is initialized to statically\n// point to an empty heap to avoid initialization checks\n// in the fast path.\n// ------------------------------------------------------\n\n// Thread local data\ntypedef struct mi_tld_s mi_tld_t;\n\n// Pages of a certain block size are held in a queue.\ntypedef struct mi_page_queue_s {\n mi_page_t* first;\n mi_page_t* last;\n size_t block_size;\n} mi_page_queue_t;\n\n#define MI_BIN_FULL (MI_BIN_HUGE+1)\n\n// Random context\ntypedef struct mi_random_cxt_s {\n uint32_t input[16];\n uint32_t output[16];\n int output_available;\n bool weak;\n} mi_random_ctx_t;\n\n\n// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows\n#if (MI_PADDING)\ntypedef struct mi_padding_s {\n uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)\n uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)\n} mi_padding_t;\n#define MI_PADDING_SIZE (sizeof(mi_padding_t))\n#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE)\n#else\n#define MI_PADDING_SIZE 0\n#define MI_PADDING_WSIZE 0\n#endif\n\n#define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1)\n\n\n// A heap owns a set of pages.\nstruct mi_heap_s {\n mi_tld_t* tld;\n _Atomic(mi_block_t*) thread_delayed_free;\n mi_threadid_t thread_id; // thread this heap belongs too\n mi_arena_id_t arena_id; // arena id if the heap belongs to a specific arena (or 0)\n uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`)\n uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list\n mi_random_ctx_t random; // random number context used for secure allocation\n size_t page_count; // total number of pages in the `pages` queues.\n size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues)\n size_t page_retired_max; // largest retired index into the `pages` array.\n size_t pages_full_size; // optimization: total size of blocks in the pages of the full queue (issue #1220)\n long generic_count; // how often is `_mi_malloc_generic` called?\n long generic_collect_count; // how often is `_mi_malloc_generic` called without collecting?\n mi_heap_t* next; // list of heaps per thread\n bool no_reclaim; // `true` if this heap should not reclaim abandoned pages\n uint8_t tag; // custom tag, can be used for separating heaps based on the object types\n #if MI_GUARDED\n size_t guarded_size_min; // minimal size for guarded objects\n size_t guarded_size_max; // maximal size for guarded objects\n size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages)\n size_t guarded_sample_count; // current sample count (counting down to 0)\n #endif\n mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.\n mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or \"bin\")\n};\n\n\n// ------------------------------------------------------\n// Sub processes do not reclaim or visit segments\n// from other sub processes. These are essentially the\n// static variables of a process.\n// ------------------------------------------------------\n\nstruct mi_subproc_s {\n _Atomic(size_t) abandoned_count; // count of abandoned segments for this sub-process\n _Atomic(size_t) abandoned_os_list_count; // count of abandoned segments in the os-list\n mi_lock_t abandoned_os_lock; // lock for the abandoned os segment list (outside of arena's) (this lock protect list operations)\n mi_lock_t abandoned_os_visit_lock; // ensure only one thread per subproc visits the abandoned os list\n mi_segment_t* abandoned_os_list; // doubly-linked list of abandoned segments outside of arena's (in OS allocated memory)\n mi_segment_t* abandoned_os_list_tail; // the tail-end of the list\n mi_memid_t memid; // provenance of this memory block\n};\n\n\n// ------------------------------------------------------\n// Thread Local data\n// ------------------------------------------------------\n\n// A \"span\" is is an available range of slices. The span queues keep\n// track of slice spans of at most the given `slice_count` (but more than the previous size class).\ntypedef struct mi_span_queue_s {\n mi_slice_t* first;\n mi_slice_t* last;\n size_t slice_count;\n} mi_span_queue_t;\n\n#define MI_SEGMENT_BIN_MAX (35) // 35 == mi_segment_bin(MI_SLICES_PER_SEGMENT)\n\n// Segments thread local data\ntypedef struct mi_segments_tld_s {\n mi_span_queue_t spans[MI_SEGMENT_BIN_MAX+1]; // free slice spans inside segments\n size_t count; // current number of segments;\n size_t peak_count; // peak number of segments\n size_t current_size; // current size of all segments\n size_t peak_size; // peak size of all segments\n size_t reclaim_count;// number of reclaimed (abandoned) segments\n mi_subproc_t* subproc; // sub-process this thread belongs to.\n mi_stats_t* stats; // points to tld stats\n} mi_segments_tld_t;\n\n// Thread local data\nstruct mi_tld_s {\n unsigned long long heartbeat; // monotonic heartbeat count\n bool recurse; // true if deferred was called; used to prevent infinite recursion.\n mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted)\n mi_heap_t* heaps; // list of heaps in this thread (so we can abandon all when the thread terminates)\n mi_segments_tld_t segments; // segment tld\n mi_stats_t stats; // statistics\n};\n\n\n// ------------------------------------------------------\n// Debug\n// ------------------------------------------------------\n\n#if !defined(MI_DEBUG_UNINIT)\n#define MI_DEBUG_UNINIT (0xD0)\n#endif\n#if !defined(MI_DEBUG_FREED)\n#define MI_DEBUG_FREED (0xDF)\n#endif\n#if !defined(MI_DEBUG_PADDING)\n#define MI_DEBUG_PADDING (0xDE)\n#endif\n\n\n// ------------------------------------------------------\n// Statistics\n// ------------------------------------------------------\n#ifndef MI_STAT\n#if (MI_DEBUG>0)\n#define MI_STAT 2\n#else\n#define MI_STAT 0\n#endif\n#endif\n\n// add to stat keeping track of the peak\nvoid _mi_stat_increase(mi_stat_count_t* stat, size_t amount);\nvoid _mi_stat_decrease(mi_stat_count_t* stat, size_t amount);\nvoid _mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount);\n// counters can just be increased\nvoid _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);\n\n#if (MI_STAT)\n#define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount)\n#define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount)\n#define mi_stat_adjust_decrease(stat,amount) _mi_stat_adjust_decrease( &(stat), amount)\n#define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount)\n#else\n#define mi_stat_increase(stat,amount) ((void)0)\n#define mi_stat_decrease(stat,amount) ((void)0)\n#define mi_stat_adjust_decrease(stat,amount) ((void)0)\n#define mi_stat_counter_increase(stat,amount) ((void)0)\n#endif\n\n#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount)\n#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount)\n#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount)\n#define mi_heap_stat_adjust_decrease(heap,stat,amount) mi_stat_adjust_decrease( (heap)->tld->stats.stat, amount)\n\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/alloc-aligned.c", "sha256": "09c37411ef1719d263343d215527bfaed257b2a003250897aef1cabb239cb9cb", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2021, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\" // mi_prim_get_default_heap\n\n#include // memset\n\n// ------------------------------------------------------\n// Aligned Allocation\n// ------------------------------------------------------\n\nstatic bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) {\n // objects up to `MI_MAX_ALIGN_GUARANTEE` are allocated aligned to their size (see `segment.c:_mi_segment_page_start`).\n mi_assert_internal(_mi_is_power_of_two(alignment) && (alignment > 0));\n if (alignment > size) return false;\n if (alignment <= MI_MAX_ALIGN_SIZE) return true;\n const size_t bsize = mi_good_size(size);\n return (bsize <= MI_MAX_ALIGN_GUARANTEE && (bsize & (alignment-1)) == 0);\n}\n\n#if MI_GUARDED\nstatic mi_decl_noinline mi_decl_restrict void* mi_heap_malloc_guarded_aligned(mi_heap_t* heap, size_t size, size_t alignment, bool zero) mi_attr_noexcept {\n // use over allocation for guarded blocksl\n mi_assert_internal(alignment > 0 && alignment < MI_BLOCK_ALIGNMENT_MAX);\n if mi_unlikely(alignment >= MI_BLOCK_ALIGNMENT_MAX || size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE - alignment)) {\n _mi_error_message(EOVERFLOW, \"(guarded) aligned allocation request is too large (size %zu, alignment %zu)\\n\", size, alignment);\n return NULL;\n }\n const size_t oversize = size + alignment - 1;\n void* const base = _mi_heap_malloc_guarded(heap, oversize, zero);\n if (base==NULL) return NULL;\n void* const p = _mi_align_up_ptr(base, alignment);\n mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size);\n mi_assert_internal(mi_usable_size(p) >= size);\n mi_assert_internal(_mi_is_aligned(p, alignment));\n return p;\n}\n\nstatic void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero, size_t* usable) {\n const size_t rate = heap->guarded_sample_rate;\n // only write if `rate!=0` so we don't write to the constant `_mi_heap_empty`\n if (rate != 0) { heap->guarded_sample_rate = 0; }\n void* p = _mi_heap_malloc_zero_ex(heap, size, zero, 0, usable);\n if (rate != 0) { heap->guarded_sample_rate = rate; }\n return p;\n}\n#else\nstatic void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero, size_t* usable) {\n return _mi_heap_malloc_zero_ex(heap, size, zero, 0, usable);\n}\n#endif\n\n// Fallback aligned allocation that over-allocates -- split out for better codegen\nstatic mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept\n{\n mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE));\n mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));\n\n void* p;\n size_t oversize;\n if mi_unlikely(alignment > MI_BLOCK_ALIGNMENT_MAX) {\n // use OS allocation for very large alignment and allocate inside a huge page (dedicated segment with 1 page)\n // This can support alignments >= MI_SEGMENT_SIZE by ensuring the object can be aligned at a point in the\n // first (and single) page such that the segment info is `MI_SEGMENT_SIZE` bytes before it (so it can be found by aligning the pointer down)\n if mi_unlikely(offset != 0) {\n // todo: cannot support offset alignment for very large alignments yet\n _mi_error_message(EOVERFLOW, \"aligned allocation with a very large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\\n\", size, alignment, offset);\n return NULL;\n }\n oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size);\n // note: no guarded as alignment > 0\n p = _mi_heap_malloc_zero_ex(heap, oversize, false, alignment, usable); // the page block size should be large enough to align in the single huge page block\n // zero afterwards as only the area from the aligned_p may be committed!\n if (p == NULL) return NULL;\n }\n else {\n // otherwise over-allocate\n mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_BLOCK_ALIGNMENT_MAX);\n mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow\n oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that.\n p = mi_heap_malloc_zero_no_guarded(heap, oversize, zero, usable);\n if (p == NULL) return NULL;\n }\n mi_page_t* page = _mi_ptr_page(p);\n\n // .. and align within the allocation\n const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)`\n const uintptr_t poffset = ((uintptr_t)p + offset) & align_mask;\n const uintptr_t adjust = (poffset == 0 ? 0 : alignment - poffset);\n mi_assert_internal(adjust < alignment);\n void* aligned_p = (void*)((uintptr_t)p + adjust);\n if (aligned_p != p) {\n mi_page_set_has_aligned(page, true);\n #if MI_GUARDED\n // set tag to aligned so mi_usable_size works with guard pages\n if (adjust >= sizeof(mi_block_t)) {\n mi_block_t* const block = (mi_block_t*)p;\n block->next = MI_BLOCK_TAG_ALIGNED;\n }\n #endif\n _mi_padding_shrink(page, (mi_block_t*)p, adjust + size);\n }\n // todo: expand padding if overallocated ?\n\n mi_assert_internal(mi_page_usable_block_size(page) >= adjust + size);\n mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0);\n mi_assert_internal(mi_usable_size(aligned_p)>=size);\n mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust);\n #if MI_DEBUG > 1\n mi_page_t* const apage = _mi_ptr_page(aligned_p);\n void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p);\n mi_assert_internal(p == unalign_p);\n #endif\n\n // now zero the block if needed\n if (alignment > MI_BLOCK_ALIGNMENT_MAX) {\n // for the tracker, on huge aligned allocations only the memory from the start of the large block is defined\n mi_track_mem_undefined(aligned_p, size);\n if (zero) {\n _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p));\n }\n }\n\n if (p != aligned_p) {\n mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p));\n #if MI_GUARDED\n mi_track_mem_defined(p, sizeof(mi_block_t));\n #endif\n }\n return aligned_p;\n}\n\n// Generic primitive aligned allocation -- split out for better codegen\nstatic mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept\n{\n mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));\n // we don't allocate more than MI_MAX_ALLOC_SIZE (see )\n if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) {\n _mi_error_message(EOVERFLOW, \"aligned allocation request is too large (size %zu, alignment %zu)\\n\", size, alignment);\n return NULL;\n }\n\n // use regular allocation if it is guaranteed to fit the alignment constraints.\n // this is important to try as the fast path in `mi_heap_malloc_zero_aligned` only works when there exist\n // a page with the right block size, and if we always use the over-alloc fallback that would never happen.\n if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) {\n void* p = mi_heap_malloc_zero_no_guarded(heap, size, zero, usable);\n mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);\n const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0;\n if mi_likely(is_aligned_or_null) {\n return p;\n }\n else {\n // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct..\n mi_assert(false);\n mi_free(p);\n }\n }\n\n // fall back to over-allocation\n return mi_heap_malloc_zero_aligned_at_overalloc(heap,size,alignment,offset,zero,usable);\n}\n\n\n// Primitive aligned allocation\nstatic void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, \n const size_t alignment, const size_t offset, const bool zero,\n size_t* usable) mi_attr_noexcept\n{\n // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size.\n if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see )\n #if MI_DEBUG > 0\n _mi_error_message(EOVERFLOW, \"aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\\n\", size, alignment);\n #endif\n return NULL;\n }\n\n #if MI_GUARDED\n if (offset==0 && alignment < MI_BLOCK_ALIGNMENT_MAX && mi_heap_malloc_use_guarded(heap,size)) {\n return mi_heap_malloc_guarded_aligned(heap, size, alignment, zero);\n }\n #endif\n\n // try first if there happens to be a small block available with just the right alignment\n if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) {\n const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`\n const size_t padsize = size + MI_PADDING_SIZE;\n mi_page_t* page = _mi_heap_get_free_small_page(heap, padsize);\n if mi_likely(page->free != NULL) {\n const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;\n if mi_likely(is_aligned)\n {\n if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }\n void* p = (zero ? _mi_page_malloc_zeroed(heap,page,padsize) : _mi_page_malloc(heap,page,padsize)); // call specific page malloc for better codegen\n mi_assert_internal(p != NULL);\n mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);\n mi_track_malloc(p,size,zero);\n return p;\n }\n }\n }\n\n // fallback to generic aligned allocation\n return mi_heap_malloc_zero_aligned_at_generic(heap, size, alignment, offset, zero, usable);\n}\n\n\n// ------------------------------------------------------\n// Optimized mi_heap_malloc_aligned / mi_malloc_aligned\n// ------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false, NULL);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_heap_malloc_aligned_at(heap, size, alignment, 0);\n}\n\n// ensure a definition is emitted\n#if defined(__cplusplus)\nvoid* _mi_extern_heap_malloc_aligned = (void*)&mi_heap_malloc_aligned;\n#endif\n\n// ------------------------------------------------------\n// Aligned Allocation\n// ------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true, NULL);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_heap_zalloc_aligned_at(heap, size, alignment, 0);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count, size, &total)) return NULL;\n return mi_heap_zalloc_aligned_at(heap, total, alignment, offset);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_heap_calloc_aligned_at(heap,count,size,alignment,0);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_heap_malloc_aligned_at(mi_prim_get_default_heap(), size, alignment, offset);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {\n return mi_heap_malloc_aligned(mi_prim_get_default_heap(), size, alignment);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {\n return mi_heap_malloc_zero_aligned_at(mi_prim_get_default_heap(), size, alignment, 0, false, block_size);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_heap_zalloc_aligned_at(mi_prim_get_default_heap(), size, alignment, offset);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {\n return mi_heap_zalloc_aligned(mi_prim_get_default_heap(), size, alignment);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {\n return mi_heap_malloc_zero_aligned_at(mi_prim_get_default_heap(), size, alignment, 0, true, block_size);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_heap_calloc_aligned_at(mi_prim_get_default_heap(), count, size, alignment, offset);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_heap_calloc_aligned(mi_prim_get_default_heap(), count, size, alignment);\n}\n\n\n// ------------------------------------------------------\n// Aligned re-allocation\n// ------------------------------------------------------\n\nstatic void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept {\n mi_assert(alignment > 0);\n if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_heap_realloc_zero(heap,p,newsize,zero,NULL,NULL);\n if (p == NULL) return mi_heap_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero,NULL);\n size_t size = mi_usable_size(p);\n if (newsize <= size && newsize >= (size - (size / 2))\n && (((uintptr_t)p + offset) % alignment) == 0) {\n return p; // reallocation still fits, is aligned and not more than 25% waste\n }\n else {\n // note: we don't zero allocate upfront so we only zero initialize the expanded part\n void* newp = mi_heap_malloc_aligned_at(heap,newsize,alignment,offset);\n if (newp != NULL) {\n if (zero && newsize > size) {\n // also set last word in the previous allocation to zero to ensure any padding is zero-initialized\n size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);\n _mi_memzero((uint8_t*)newp + start, newsize - start);\n }\n _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));\n mi_free(p); // only free if successful\n }\n return newp;\n }\n}\n\nstatic void* mi_heap_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept {\n mi_assert(alignment > 0);\n if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero,NULL,NULL);\n return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,0,zero);\n}\n\nmi_decl_nodiscard void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false);\n}\n\nmi_decl_nodiscard void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {\n return mi_heap_realloc_zero_aligned(heap,p,newsize,alignment,false);\n}\n\nmi_decl_nodiscard void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_heap_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true);\n}\n\nmi_decl_nodiscard void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {\n return mi_heap_realloc_zero_aligned(heap, p, newsize, alignment, true);\n}\n\nmi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(newcount, size, &total)) return NULL;\n return mi_heap_rezalloc_aligned_at(heap, p, total, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(newcount, size, &total)) return NULL;\n return mi_heap_rezalloc_aligned(heap, p, total, alignment);\n}\n\nmi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_heap_realloc_aligned_at(mi_prim_get_default_heap(), p, newsize, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {\n return mi_heap_realloc_aligned(mi_prim_get_default_heap(), p, newsize, alignment);\n}\n\nmi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_heap_rezalloc_aligned_at(mi_prim_get_default_heap(), p, newsize, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {\n return mi_heap_rezalloc_aligned(mi_prim_get_default_heap(), p, newsize, alignment);\n}\n\nmi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_heap_recalloc_aligned_at(mi_prim_get_default_heap(), p, newcount, size, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_heap_recalloc_aligned(mi_prim_get_default_heap(), p, newcount, size, alignment);\n}\n\n\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/alloc-override.c", "sha256": "90421ca6a329e6cc2513012508074a24607e92e58f2b41e527a78bfba1f07bed", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2026, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n#if !defined(MI_IN_ALLOC_C)\n#error \"this file should be included from 'alloc.c' (so aliases can work)\"\n#endif\n\n\n#if defined(MI_MALLOC_OVERRIDE) && !defined(_DLL)\n\n#if defined(__APPLE__)\n#include \nmi_decl_externc void vfree(void* p);\nmi_decl_externc size_t malloc_size(const void* p);\nmi_decl_externc size_t malloc_good_size(size_t size);\n#endif\n\n// helper definition for C override of C++ new\ntypedef void* mi_nothrow_t;\n\n// ------------------------------------------------------\n// Override system malloc\n// ------------------------------------------------------\n\n#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !MI_TRACK_ENABLED\n // gcc, clang: use aliasing to alias the exported function to one of our `mi_` functions\n #if (defined(__GNUC__) && __GNUC__ >= 9)\n #pragma GCC diagnostic ignored \"-Wattributes\" // or we get warnings that nodiscard is ignored on a forward\n #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility(\"default\"), copy(fun)));\n #else\n #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility(\"default\")));\n #endif\n #define MI_FORWARD1(fun,x) MI_FORWARD(fun)\n #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun)\n #define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun)\n #define MI_FORWARD0(fun,x) MI_FORWARD(fun)\n #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun)\n#else\n // otherwise use forwarding by calling our `mi_` function\n #define MI_FORWARD1(fun,x) { return fun(x); }\n #define MI_FORWARD2(fun,x,y) { return fun(x,y); }\n #define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); }\n #define MI_FORWARD0(fun,x) { fun(x); }\n #define MI_FORWARD02(fun,x,y) { fun(x,y); }\n#endif\n\n\n#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE)\n // define MI_OSX_IS_INTERPOSED as we should not provide forwarding definitions for\n // functions that are interposed (or the interposing does not work)\n #define MI_OSX_IS_INTERPOSED\n\n mi_decl_externc size_t mi_malloc_size_checked(void *p) {\n if (!mi_is_in_heap_region(p)) return 0;\n return mi_usable_size(p);\n }\n\n // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1`\n // See: \n struct mi_interpose_s {\n const void* replacement;\n const void* target;\n };\n #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun }\n #define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun)\n\n #define MI_INTERPOSE_DECLS(name) __attribute__((used)) static struct mi_interpose_s name[] __attribute__((section(\"__DATA, __interpose\")))\n\n MI_INTERPOSE_DECLS(_mi_interposes) =\n {\n MI_INTERPOSE_MI(malloc),\n MI_INTERPOSE_MI(calloc),\n MI_INTERPOSE_MI(realloc),\n MI_INTERPOSE_MI(strdup),\n MI_INTERPOSE_MI(realpath),\n MI_INTERPOSE_MI(posix_memalign),\n MI_INTERPOSE_MI(reallocf),\n MI_INTERPOSE_MI(valloc),\n MI_INTERPOSE_FUN(malloc_size,mi_malloc_size_checked),\n MI_INTERPOSE_MI(malloc_good_size),\n #ifdef MI_OSX_ZONE\n // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely\n MI_INTERPOSE_MI(free),\n MI_INTERPOSE_FUN(vfree,mi_free),\n #else\n // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity )\n MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us\n MI_INTERPOSE_FUN(vfree,mi_cfree),\n #endif\n };\n #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)\n MI_INTERPOSE_DECLS(_mi_interposes_10_7) = { MI_INTERPOSE_MI(strndup) };\n #endif\n #if defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15)\n MI_INTERPOSE_DECLS(_mi_interposes_10_15) = { MI_INTERPOSE_MI(aligned_alloc) };\n #endif\n\n #ifdef __cplusplus\n extern \"C\" {\n #endif\n void _ZdlPv(void* p); // delete\n void _ZdaPv(void* p); // delete[]\n void _ZdlPvm(void* p, size_t n); // delete\n void _ZdaPvm(void* p, size_t n); // delete[]\n void* _Znwm(size_t n); // new\n void* _Znam(size_t n); // new[]\n void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow\n void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow\n #ifdef __cplusplus\n }\n #endif\n __attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[] __attribute__((section(\"__DATA, __interpose\"))) =\n {\n MI_INTERPOSE_FUN(_ZdlPv,mi_free),\n MI_INTERPOSE_FUN(_ZdaPv,mi_free),\n MI_INTERPOSE_FUN(_ZdlPvm,mi_free_size),\n MI_INTERPOSE_FUN(_ZdaPvm,mi_free_size),\n MI_INTERPOSE_FUN(_Znwm,mi_new),\n MI_INTERPOSE_FUN(_Znam,mi_new),\n MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow),\n MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow),\n };\n\n#elif defined(_MSC_VER)\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_HYBRIDPATCHABLE\n void* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) {\n return mi_expand(_Block, _Size);\n }\n _Check_return_ _ACRTIMP \n size_t __cdecl _msize_base(_Pre_notnull_ void* _Block) _CRT_NOEXCEPT {\n return mi_malloc_size(_Block);\n }\n _Check_return_ _ACRTIMP _CRT_HYBRIDPATCHABLE \n size_t __cdecl _msize(_Pre_notnull_ void* _Block) {\n return mi_malloc_size(_Block);\n }\n _ACRTIMP \n void __cdecl _free_base(_Pre_maybenull_ _Post_invalid_ void* _Block) {\n mi_free(_Block);\n }\n _ACRTIMP _CRT_HYBRIDPATCHABLE \n void __cdecl free(_Pre_maybenull_ _Post_invalid_ void* _Block) {\n mi_free(_Block);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_JIT_INTRINSIC _CRTRESTRICT _CRT_HYBRIDPATCHABLE\n void* __cdecl malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size) {\n return mi_malloc(_Size);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _malloc_base(_In_ size_t _Size) {\n return mi_malloc(_Size);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _realloc_base(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ size_t _Size) {\n return mi_realloc(_Block, _Size);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT _CRT_HYBRIDPATCHABLE\n void* __cdecl realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) {\n return mi_realloc(_Block, _Size);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT \n void* __cdecl _calloc_base(_In_ size_t _Count, _In_ size_t _Size) {\n return mi_calloc(_Count, _Size);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRT_JIT_INTRINSIC _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl calloc(_In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) {\n return mi_calloc(_Count, _Size);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _recalloc_base(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ size_t _Count, _In_ size_t _Size) {\n return mi_recalloc(_Block, _Count, _Size);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) {\n return mi_recalloc(_Block, _Count, _Size);\n }\n _ACRTIMP \n void __cdecl _aligned_free(_Pre_maybenull_ _Post_invalid_ void* _Block) {\n mi_free(_Block);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {\n return mi_malloc_aligned(_Size, _Alignment);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_offset_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {\n return mi_malloc_aligned_at(_Size, _Alignment, _Offset);\n }\n _Check_return_ _ACRTIMP \n size_t __cdecl _aligned_msize(_Pre_notnull_ void* _Block, _In_ size_t _Alignment, _In_ size_t _Offset) {\n MI_UNUSED(_Alignment); MI_UNUSED(_Offset); return mi_malloc_size(_Block);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_offset_realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {\n return mi_realloc_aligned_at(_Block, _Size, _Alignment, _Offset);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_offset_recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {\n return mi_recalloc_aligned_at(_Block, _Count, _Size, _Alignment, _Offset);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {\n return mi_realloc_aligned(_Block, _Size, _Alignment);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {\n return mi_recalloc_aligned(_Block, _Count, _Size, _Alignment);\n }\n#else\n // On all other systems forward allocation primitives to our API\n mi_decl_export void* malloc(size_t size) MI_FORWARD1(mi_malloc, size)\n mi_decl_export void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n)\n mi_decl_export void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize)\n mi_decl_export void free(void* p) MI_FORWARD0(mi_free, p)\n // In principle we do not need to forward `strdup`/`strndup` but on some systems these do not use `malloc` internally (but a more primitive call)\n // We only override if `strdup` is not a macro (as on some older libc's, see issue #885)\n #if !defined(strdup)\n mi_decl_export char* strdup(const char* str) MI_FORWARD1(mi_strdup, str)\n #endif\n #if !defined(strndup) && (!defined(__APPLE__) || (defined(MAC_OS_X_VERSION_10_7) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7))\n mi_decl_export char* strndup(const char* str, size_t n) MI_FORWARD2(mi_strndup, str, n)\n #endif\n#endif\n\n#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)\n#pragma GCC visibility push(default)\n#endif\n\n// ------------------------------------------------------\n// Override new/delete\n// This is not really necessary as they usually call\n// malloc/free anyway, but it improves performance.\n// ------------------------------------------------------\n#ifdef __cplusplus\n // ------------------------------------------------------\n // With a C++ compiler we override the new/delete operators.\n // see \n // ------------------------------------------------------\n #include \n\n #ifndef MI_OSX_IS_INTERPOSED\n void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p)\n void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p)\n\n void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n)\n void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n)\n\n void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); }\n void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); }\n\n #if (__cplusplus >= 201402L || _MSC_VER >= 1916)\n void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)\n void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)\n #endif\n #endif\n\n #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5))\n void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); };\n void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); };\n void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); }\n\n void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); }\n void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); }\n void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); }\n void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); }\n #endif\n\n#elif (defined(__GNUC__) || defined(__clang__))\n // ------------------------------------------------------\n // Override by defining the mangled C++ names of the operators (as\n // used by GCC and CLang).\n // See \n // ------------------------------------------------------\n\n void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete\n void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[]\n void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)\n void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)\n\n void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); }\n void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); }\n void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }\n void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }\n\n void _ZdlPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete(void*, std::nothrow_t const&)\n void _ZdaPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete[](void*, std::nothrow_t const&)\n void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&)\n void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&)\n\n #if (MI_INTPTR_SIZE==8) || (MI_INTPTR_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257\n void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit\n void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit\n void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }\n void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }\n void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)\n void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)\n void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }\n void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }\n #elif (MI_INTPTR_SIZE==4)\n void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit\n void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit\n void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }\n void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }\n void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)\n void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)\n void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }\n void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }\n #else\n #error \"define overloads for new/delete for this platform (just for performance, can be skipped)\"\n #endif\n#endif // __cplusplus\n\n// ------------------------------------------------------\n// Further Posix & Unix functions definitions\n// ------------------------------------------------------\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#ifndef MI_OSX_IS_INTERPOSED\n // Forward Posix/Unix calls as well\n void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize)\n size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p)\n #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__)\n size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p)\n #else\n size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p)\n #endif\n\n // No forwarding here due to aliasing/name mangling issues\n void* valloc(size_t size) { return mi_valloc(size); }\n void vfree(void* p) { mi_free(p); }\n size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); }\n int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); }\n\n // `aligned_alloc` is only available when __USE_ISOC11 is defined.\n // Note: it seems __USE_ISOC11 is not defined in musl (and perhaps other libc's) so we only check\n // for it if using glibc.\n // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot\n // override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9.\n // Fortunately, in the case where `aligned_alloc` is declared as `static inline` it\n // uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves.\n #if !defined(__GLIBC__) || __USE_ISOC11\n void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); }\n #endif\n#endif\n\n// no forwarding here due to aliasing/name mangling issues\nvoid cfree(void* p) { mi_free(p); }\nvoid* pvalloc(size_t size) { return mi_pvalloc(size); }\nvoid* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); }\n#if !defined(_WIN32)\nvoid* _aligned_malloc(size_t size, size_t alignment) { return mi_malloc_aligned(size,alignment); }\n#endif\nvoid* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); }\n// some systems define reallocarr so mark it as a weak symbol (#751)\nmi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); }\n\n#if defined(__wasi__)\n // forward __libc interface (see PR #667)\n void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc, size)\n void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc, count, size)\n void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc, p, size)\n void __libc_free(void* p) MI_FORWARD0(mi_free, p)\n void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); }\n\n#elif defined(__linux__)\n // forward __libc interface (needed for glibc-based and musl-based Linux distributions)\n void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size)\n void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size)\n void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size)\n void __libc_free(void* p) MI_FORWARD0(mi_free,p)\n void __libc_cfree(void* p) MI_FORWARD0(mi_free,p)\n\n void* __libc_valloc(size_t size) { return mi_valloc(size); }\n void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); }\n void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); }\n int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); }\n#endif\n\n#ifdef __cplusplus\n}\n#endif\n\n#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)\n#pragma GCC visibility pop\n#endif\n\n#endif // MI_MALLOC_OVERRIDE\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/alloc-posix.c", "sha256": "98d0eb877d7b9cdf7e9f857aaf3c739b69f746b91221e0e427e82bdd7cec90b3", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2021, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// ------------------------------------------------------------------------\n// mi prefixed publi definitions of various Posix, Unix, and C++ functions\n// for convenience and used when overriding these functions.\n// ------------------------------------------------------------------------\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n\n// ------------------------------------------------------\n// Posix & Unix functions definitions\n// ------------------------------------------------------\n\n#include \n#include // memset\n#include // getenv\n\n#ifdef _MSC_VER\n#pragma warning(disable:4996) // getenv _wgetenv\n#endif\n\n#ifndef EINVAL\n#define EINVAL 22\n#endif\n#ifndef ENOMEM\n#define ENOMEM 12\n#endif\n\n\nmi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept {\n // if (!mi_is_in_heap_region(p)) return 0;\n return mi_usable_size(p);\n}\n\nmi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {\n // if (!mi_is_in_heap_region(p)) return 0;\n return mi_usable_size(p);\n}\n\nmi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept {\n return mi_good_size(size);\n}\n\nvoid mi_cfree(void* p) mi_attr_noexcept {\n if (mi_is_in_heap_region(p)) {\n mi_free(p);\n }\n}\n\nint mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept {\n // Note: The spec dictates we should not modify `*p` on an error. (issue#27)\n // \n if (p == NULL) return EINVAL;\n if ((alignment % sizeof(void*)) != 0) return EINVAL; // natural alignment\n // it is also required that alignment is a power of 2 and > 0; this is checked in `mi_malloc_aligned`\n if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2\n void* q = mi_malloc_aligned(size, alignment);\n if (q==NULL && size != 0) return ENOMEM;\n mi_assert_internal(_mi_is_aligned(q,alignment));\n *p = q;\n return 0;\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {\n void* p = mi_malloc_aligned(size, alignment);\n mi_assert_internal(_mi_is_aligned(p,alignment));\n return p;\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept {\n return mi_memalign( _mi_os_page_size(), size );\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept {\n size_t psize = _mi_os_page_size();\n if (size >= SIZE_MAX - psize) return NULL; // overflow\n size_t asize = _mi_align_up(size, psize);\n return mi_malloc_aligned(asize, psize);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {\n // C11 requires the size to be an integral multiple of the alignment, see .\n // unfortunately, it turns out quite some programs pass a size that is not an integral multiple so skip this check..\n /* if mi_unlikely((size & (alignment - 1)) != 0) { // C11 requires alignment>0 && integral multiple, see \n #if MI_DEBUG > 0\n _mi_error_message(EOVERFLOW, \"(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\\n\", size, alignment);\n #endif\n return NULL;\n }\n */\n // C11 also requires alignment to be a power-of-two (and > 0) which is checked in mi_malloc_aligned\n void* p = mi_malloc_aligned(size, alignment);\n mi_assert_internal(_mi_is_aligned(p,alignment));\n return p;\n}\n\nmi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD \n size_t total;\n if mi_unlikely(mi_count_size_overflow(count, size, &total)) {\n errno = EOVERFLOW;\n return NULL;\n }\n void* newp = mi_realloc(p,total);\n if (newp==NULL) { errno = ENOMEM; }\n return newp;\n}\n\nmi_decl_nodiscard int mi_reallocarr( void* ptrp, size_t count, size_t size ) mi_attr_noexcept { // NetBSD \n mi_assert(size != 0);\n mi_assert(ptrp != NULL);\n if (ptrp == NULL || size == 0) {\n return (errno = EINVAL);\n }\n size_t total;\n if mi_unlikely(mi_count_size_overflow(count, size, &total)) {\n return (errno = EOVERFLOW);\n }\n void** op = (void**)ptrp;\n if (total == 0) {\n free(*op);\n *op = NULL;\n return 0;\n }\n else {\n void* newp = mi_realloc(*op,total);\n if (newp == NULL) { return (errno = ENOMEM); }\n *op = newp;\n return 0;\n }\n}\n\nvoid* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft\n void* res = mi_expand(p, newsize);\n if (res == NULL) { errno = ENOMEM; }\n return res;\n}\n\nmi_decl_nodiscard mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept {\n if (s==NULL) return NULL;\n size_t wlen;\n for(wlen = 0; s[wlen] != 0 && wlen < PTRDIFF_MAX; wlen++) { } // prevent overflow on wlen+1\n size_t size; \n if (mi_mul_overflow(wlen+1, sizeof(wchar_t), &size) || size > PTRDIFF_MAX) return NULL;\n wchar_t* p = (wchar_t*)mi_malloc(size);\n if (p != NULL) {\n _mi_memcpy(p,s,size);\n }\n return p;\n}\n\nmi_decl_nodiscard mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept {\n return (unsigned char*)mi_strdup((const char*)s);\n}\n\nint mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept {\n if (size != NULL) *size = 0;\n if (buf==NULL || name==NULL) return EINVAL;\n char* p = getenv(name);\n if (p==NULL) {\n *buf = NULL;\n }\n else {\n *buf = mi_strdup(p);\n if (*buf==NULL) return ENOMEM;\n if (size != NULL) { *size = _mi_strlen(p) + 1; } // cannot overflow as mi_strdup is limited to PTRDIFF_MAX\n }\n return 0;\n}\n\nint mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept {\n if (size != NULL) *size = 0;\n if (buf==NULL || name==NULL) return EINVAL; \n#if !defined(_WIN32) || (defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP))\n // not supported\n *buf = NULL;\n return EINVAL;\n#else\n wchar_t* p = (wchar_t*)_wgetenv(name);\n if (p==NULL) {\n *buf = NULL;\n }\n else {\n *buf = mi_wcsdup(p);\n if (*buf==NULL) return ENOMEM;\n if (size != NULL) { *size = wcslen(p) + 1; } // cannot overflow as wcsdup is limited to PTRDIFF_MAX\n }\n return 0;\n#endif\n}\n\nmi_decl_nodiscard void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft\n return mi_recalloc_aligned_at(p, newcount, size, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft\n return mi_recalloc_aligned(p, newcount, size, alignment);\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/alloc.c", "sha256": "78e9dd453356984e63dcd210baa1d4ff8b9b21c1fb5ab2165b593e9b1c4ff856", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#ifndef _DEFAULT_SOURCE\n#define _DEFAULT_SOURCE // for realpath() on Linux\n#endif\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\" // _mi_prim_thread_id()\n\n#include // memset, strlen (for mi_strdup)\n#include // malloc, abort\n\n#define MI_IN_ALLOC_C\n#include \"alloc-override.c\"\n#include \"free.c\"\n#undef MI_IN_ALLOC_C\n\n// ------------------------------------------------------\n// Allocation\n// ------------------------------------------------------\n\n// Fast allocation in a page: just pop from the free list.\n// Fall back to generic allocation only if the list is empty.\n// Note: in release mode the (inlined) routine is about 7 instructions with a single test.\nextern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero, size_t* usable) mi_attr_noexcept\n{\n mi_assert_internal(size >= MI_PADDING_SIZE);\n mi_assert_internal(page->block_size == 0 /* empty heap */ || mi_page_block_size(page) >= size);\n\n // check the free list\n mi_block_t* const block = page->free;\n if mi_unlikely(block == NULL) {\n return _mi_malloc_generic(heap, size, zero, 0, usable);\n }\n mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);\n if (usable != NULL) { *usable = mi_page_usable_block_size(page); };\n // pop from the free list\n page->free = mi_block_next(page, block);\n page->used++;\n mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);\n mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE));\n\n #if MI_DEBUG>3\n if (page->free_is_zero && size > sizeof(*block)) {\n mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block)));\n }\n #endif\n\n // allow use of the block internally\n // note: when tracking we need to avoid ever touching the MI_PADDING since\n // that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc/track.h`)\n mi_track_mem_undefined(block, mi_page_usable_block_size(page));\n\n // zero the block? note: we need to zero the full block size (issue #63)\n if mi_unlikely(zero) {\n mi_assert_internal(page->block_size != 0); // do not call with zero'ing for huge blocks (see _mi_malloc_generic)\n #if MI_PADDING\n mi_assert_internal(page->block_size >= MI_PADDING_SIZE);\n #endif\n if (page->free_is_zero) {\n block->next = 0;\n mi_track_mem_defined(block, page->block_size - MI_PADDING_SIZE);\n }\n else {\n _mi_memzero_aligned(block, page->block_size - MI_PADDING_SIZE);\n }\n }\n\n #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN\n if (!zero && !mi_page_is_huge(page)) {\n memset(block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page));\n }\n #elif (MI_SECURE!=0)\n if (!zero) { block->next = 0; } // don't leak internal data\n #endif\n\n #if (MI_STAT>0)\n const size_t bsize = mi_page_usable_block_size(page);\n if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) {\n mi_heap_stat_increase(heap, malloc_normal, bsize);\n mi_heap_stat_counter_increase(heap, malloc_normal_count, 1);\n #if (MI_STAT>1)\n const size_t bin = _mi_bin(bsize);\n mi_heap_stat_increase(heap, malloc_bins[bin], 1);\n mi_heap_stat_increase(heap, malloc_requested, size - MI_PADDING_SIZE);\n #endif\n }\n #endif\n\n #if MI_PADDING // && !MI_TRACK_ENABLED\n mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page));\n ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE));\n #if (MI_DEBUG>=2)\n mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - MI_PADDING_SIZE + delta));\n #endif\n mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess\n padding->canary = mi_ptr_encode_canary(page,block,page->keys);\n padding->delta = (uint32_t)(delta);\n #if MI_PADDING_CHECK\n if (!mi_page_is_huge(page)) {\n uint8_t* fill = (uint8_t*)padding - delta;\n const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes\n for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }\n }\n #endif\n #endif\n\n return block;\n}\n\n// extra entries for improved efficiency in `alloc-aligned.c`.\nextern void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept {\n return _mi_page_malloc_zero(heap,page,size,false,NULL);\n}\nextern void* _mi_page_malloc_zeroed(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept {\n return _mi_page_malloc_zero(heap,page,size,true,NULL);\n}\n\n#if MI_GUARDED\nmi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept;\n#endif\n\nstatic inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {\n mi_assert(heap != NULL);\n mi_assert(size <= MI_SMALL_SIZE_MAX);\n #if MI_DEBUG\n const uintptr_t tid = _mi_thread_id();\n mi_assert(heap->thread_id == 0 || heap->thread_id == tid); // heaps are thread local\n #endif\n #if (MI_PADDING || MI_GUARDED)\n if (size == 0) { size = sizeof(void*); }\n #endif\n #if MI_GUARDED\n if (mi_heap_malloc_use_guarded(heap,size)) {\n return _mi_heap_malloc_guarded(heap, size, zero);\n }\n #endif\n\n // get page in constant time, and allocate from it\n mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE);\n void* const p = _mi_page_malloc_zero(heap, page, size + MI_PADDING_SIZE, zero, usable);\n mi_track_malloc(p,size,zero);\n\n #if MI_DEBUG>3\n if (p != NULL && zero) {\n mi_assert_expensive(mi_mem_is_zero(p, size));\n }\n #endif\n return p;\n}\n\n// allocate a small block\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {\n return mi_heap_malloc_small_zero(heap, size, false, NULL);\n}\n\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {\n return mi_heap_malloc_small_zero(heap, size, true, NULL);\n}\n\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept {\n return mi_heap_malloc_small(mi_prim_get_default_heap(), size);\n}\n\n// The main allocation function\nextern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {\n // fast path for small objects\n if mi_likely(size <= MI_SMALL_SIZE_MAX) {\n mi_assert_internal(huge_alignment == 0);\n return mi_heap_malloc_small_zero(heap, size, zero, usable);\n }\n #if MI_GUARDED\n else if (huge_alignment==0 && mi_heap_malloc_use_guarded(heap,size)) {\n return _mi_heap_malloc_guarded(heap, size, zero);\n }\n #endif\n else {\n // regular allocation\n mi_assert(heap!=NULL);\n mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local\n void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment, usable); // note: size can overflow but it is detected in malloc_generic\n mi_track_malloc(p,size,zero);\n\n #if MI_DEBUG>3\n if (p != NULL && zero) {\n mi_assert_expensive(mi_mem_is_zero(p, size));\n }\n #endif\n return p;\n }\n}\n\nextern inline void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept {\n return _mi_heap_malloc_zero_ex(heap, size, zero, 0, NULL);\n}\n\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {\n return _mi_heap_malloc_zero(heap, size, false);\n}\n\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept {\n return mi_heap_malloc(mi_prim_get_default_heap(), size);\n}\n\n// zero initialized small block\nmi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {\n return mi_heap_zalloc_small(mi_prim_get_default_heap(), size);\n}\n\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {\n return _mi_heap_malloc_zero(heap, size, true);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept {\n return mi_heap_zalloc(mi_prim_get_default_heap(),size);\n}\n\n\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count,size,&total)) return NULL;\n return mi_heap_zalloc(heap,total);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept {\n return mi_heap_calloc(mi_prim_get_default_heap(),count,size);\n}\n\n// Return usable size\nmi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* usable) mi_attr_noexcept {\n return mi_heap_malloc_small_zero(mi_prim_get_default_heap(), size, false, usable);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_umalloc(mi_heap_t* heap, size_t size, size_t* usable) mi_attr_noexcept {\n return _mi_heap_malloc_zero_ex(heap, size, false, 0, usable);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* usable) mi_attr_noexcept {\n return mi_heap_umalloc(mi_prim_get_default_heap(), size, usable);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* usable) mi_attr_noexcept {\n return _mi_heap_malloc_zero_ex(mi_prim_get_default_heap(), size, true, 0, usable);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* usable) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count,size,&total)) return NULL;\n return mi_uzalloc(total, usable);\n}\n\n// Uninitialized `calloc`\nmi_decl_nodiscard extern mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count, size, &total)) return NULL;\n return mi_heap_malloc(heap, total);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {\n return mi_heap_mallocn(mi_prim_get_default_heap(),count,size);\n}\n\n// Expand (or shrink) in place (or fail)\nvoid* mi_expand(void* p, size_t newsize) mi_attr_noexcept {\n #if MI_PADDING\n // we do not shrink/expand with padding enabled\n MI_UNUSED(p); MI_UNUSED(newsize);\n return NULL;\n #else\n if (p == NULL) return NULL;\n const mi_page_t* const page = mi_validate_ptr_page(p,\"mi_expand\"); \n const size_t size = _mi_usable_size(p,page);\n if (newsize > size) return NULL;\n return p; // it fits\n #endif\n}\n\nvoid* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {\n // if p == NULL then behave as malloc.\n // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).\n // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.)\n const mi_page_t* page;\n size_t size;\n if (p==NULL) {\n page = NULL;\n size = 0;\n if (usable_pre!=NULL) { *usable_pre = 0; }\n }\n else { \n page = mi_validate_ptr_page(p,\"mi_realloc\"); \n size = _mi_usable_size(p,page);\n if (usable_pre!=NULL) { *usable_pre = mi_page_usable_block_size(page); } \n }\n if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)\n mi_assert_internal(p!=NULL);\n // todo: do not track as the usable size is still the same in the free; adjust potential padding?\n // mi_track_resize(p,size,newsize)\n // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }\n if (usable_post!=NULL) { *usable_post = mi_page_usable_block_size(page); }\n return p; // reallocation still fits and not more than 50% waste\n }\n void* newp = mi_heap_umalloc(heap,newsize,usable_post);\n if mi_likely(newp != NULL) {\n if (zero && newsize > size) {\n // also set last word in the previous allocation to zero to ensure any padding is zero-initialized\n const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);\n _mi_memzero((uint8_t*)newp + start, newsize - start);\n }\n else if (newsize == 0) {\n ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725)\n }\n if mi_likely(p != NULL) {\n const size_t copysize = (newsize > size ? size : newsize);\n mi_track_mem_defined(p,copysize); // _mi_useable_size may be too large for byte precise memory tracking..\n _mi_memcpy(newp, p, copysize);\n mi_free(p); // only free the original pointer if successful\n }\n }\n return newp;\n}\n\nmi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {\n return _mi_heap_realloc_zero(heap, p, newsize, false, NULL, NULL);\n}\n\nmi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count, size, &total)) return NULL;\n return mi_heap_realloc(heap, p, total);\n}\n\n\n// Reallocate but free `p` on errors\nmi_decl_nodiscard void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {\n void* newp = mi_heap_realloc(heap, p, newsize);\n if (newp==NULL && p!=NULL) mi_free(p);\n return newp;\n}\n\nmi_decl_nodiscard void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {\n return _mi_heap_realloc_zero(heap, p, newsize, true, NULL, NULL);\n}\n\nmi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count, size, &total)) return NULL;\n return mi_heap_rezalloc(heap, p, total);\n}\n\n\nmi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept {\n return mi_heap_realloc(mi_prim_get_default_heap(),p,newsize);\n}\n\nmi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept {\n return mi_heap_reallocn(mi_prim_get_default_heap(),p,count,size);\n}\n\nmi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {\n return _mi_heap_realloc_zero(mi_prim_get_default_heap(),p,newsize, false, usable_pre, usable_post);\n}\n\n// Reallocate but free `p` on errors\nmi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {\n return mi_heap_reallocf(mi_prim_get_default_heap(),p,newsize);\n}\n\nmi_decl_nodiscard void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept {\n return mi_heap_rezalloc(mi_prim_get_default_heap(), p, newsize);\n}\n\nmi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept {\n return mi_heap_recalloc(mi_prim_get_default_heap(), p, count, size);\n}\n\n\n\n// ------------------------------------------------------\n// strdup, strndup, and realpath\n// ------------------------------------------------------\n\n// `strdup` using mi_malloc\nmi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept {\n if (s == NULL) return NULL;\n size_t len = _mi_strlen(s);\n if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1\n char* t = (char*)mi_heap_malloc(heap,len+1);\n if (t == NULL) return NULL;\n _mi_memcpy(t, s, len);\n t[len] = 0;\n return t;\n}\n\nmi_decl_nodiscard mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept {\n return mi_heap_strdup(mi_prim_get_default_heap(), s);\n}\n\n// `strndup` using mi_malloc\nmi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept {\n if (s == NULL) return NULL;\n const size_t len = _mi_strnlen(s,n); // len <= n\n if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1\n char* t = (char*)mi_heap_malloc(heap, len+1);\n if (t == NULL) return NULL;\n _mi_memcpy(t, s, len);\n t[len] = 0;\n return t;\n}\n\nmi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept {\n return mi_heap_strndup(mi_prim_get_default_heap(),s,n);\n}\n\n#ifndef __wasi__\n// `realpath` using mi_malloc\n#ifdef _WIN32\n#ifndef PATH_MAX\n#define PATH_MAX MAX_PATH\n#endif\n\nmi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept {\n // todo: use GetFullPathNameW to allow longer file names\n char buf[PATH_MAX];\n DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL);\n if (res == 0) {\n errno = GetLastError(); return NULL;\n }\n else if (res > PATH_MAX) {\n errno = EINVAL; return NULL;\n }\n else if (resolved_name != NULL) {\n return resolved_name;\n }\n else {\n return mi_heap_strndup(heap, buf, PATH_MAX);\n }\n}\n\n#else\n\n#include // pathconf\n\nstatic size_t mi_path_max(void) {\n static _Atomic(size_t) path_max = 0;\n size_t pmax = mi_atomic_load_acquire(&path_max);\n if (pmax == 0) {\n long m = 0;\n #ifdef _PC_PATH_MAX\n m = pathconf(\"/\",_PC_PATH_MAX);\n #endif\n if (m <= 0) pmax = 4096; // guess\n else if (m < 256) pmax = 256; // at least 256\n else if (m > 64*1024) pmax = 64*1024; // at most 64 KiB\n else pmax = m;\n size_t expected = 0;\n mi_atomic_cas_strong_acq_rel(&path_max, &expected, pmax);\n }\n return pmax;\n}\n\nchar* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept {\n if (resolved_name != NULL) {\n return realpath(fname,resolved_name);\n }\n else {\n /*\n char* rname = realpath(fname, NULL);\n if (rname == NULL) return NULL;\n char* result = mi_heap_strdup(heap, rname);\n mi_cfree(rname); // note: may leak the original pointer if allocated internally with the system allocator \n // note: with ASAN realpath is intercepted and mi_cfree may leak the returned pointer :-(\n return result; \n */\n const size_t n = mi_path_max();\n char* const buf = (char*)mi_zalloc(n+1);\n if (buf == NULL) {\n errno = ENOMEM;\n return NULL;\n }\n char* rname = realpath(fname,buf);\n char* result = mi_heap_strndup(heap,rname,n); // ok if `rname==NULL`\n mi_free(buf);\n return result;\n }\n}\n#endif\n\nmi_decl_nodiscard mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept {\n return mi_heap_realpath(mi_prim_get_default_heap(),fname,resolved_name);\n}\n#endif\n\n/*-------------------------------------------------------\nC++ new and new_aligned\nThe standard requires calling into `get_new_handler` and\nthrowing the bad_alloc exception on failure. If we compile\nwith a C++ compiler we can implement this precisely. If we\nuse a C compiler we cannot throw a `bad_alloc` exception\nbut we call `exit` instead (i.e. not returning).\n-------------------------------------------------------*/\n\n#ifdef __cplusplus\n#include \nstatic bool mi_try_new_handler(bool nothrow) {\n #if defined(_MSC_VER) || (__cplusplus >= 201103L)\n std::new_handler h = std::get_new_handler();\n #else\n std::new_handler h = std::set_new_handler();\n std::set_new_handler(h);\n #endif\n if (h==NULL) {\n _mi_error_message(ENOMEM, \"out of memory in 'new'\");\n #if defined(_CPPUNWIND) || defined(__cpp_exceptions) // exceptions are not always enabled\n if (!nothrow) {\n throw std::bad_alloc();\n }\n #else\n MI_UNUSED(nothrow);\n #endif\n return false;\n }\n else {\n h();\n return true;\n }\n}\n#else\ntypedef void (*std_new_handler_t)(void);\n\n#if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER))) // exclude clang-cl, see issue #631\nstd_new_handler_t __attribute__((weak)) _ZSt15get_new_handlerv(void) {\n return NULL;\n}\nstatic std_new_handler_t mi_get_new_handler(void) {\n return _ZSt15get_new_handlerv();\n}\n#else\n// note: on windows we could dynamically link to `?get_new_handler@std@@YAP6AXXZXZ`.\nstatic std_new_handler_t mi_get_new_handler(void) {\n return NULL;\n}\n#endif\n\nstatic bool mi_try_new_handler(bool nothrow) {\n std_new_handler_t h = mi_get_new_handler();\n if (h==NULL) {\n _mi_error_message(ENOMEM, \"out of memory in 'new'\");\n if (!nothrow) {\n abort(); // cannot throw in plain C, use abort\n }\n return false;\n }\n else {\n h();\n return true;\n }\n}\n#endif\n\nmi_decl_export mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow ) {\n void* p = NULL;\n while(p == NULL && mi_try_new_handler(nothrow)) {\n p = mi_heap_malloc(heap,size);\n }\n return p;\n}\n\nstatic mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) {\n return mi_heap_try_new(mi_prim_get_default_heap(), size, nothrow);\n}\n\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) {\n void* p = mi_heap_malloc(heap,size);\n if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false);\n return p;\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) {\n return mi_heap_alloc_new(mi_prim_get_default_heap(), size);\n}\n\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) {\n size_t total;\n if mi_unlikely(mi_count_size_overflow(count, size, &total)) {\n mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc\n return NULL;\n }\n else {\n return mi_heap_alloc_new(heap,total);\n }\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) {\n return mi_heap_alloc_new_n(mi_prim_get_default_heap(), count, size);\n}\n\n\nmi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept {\n void* p = mi_malloc(size);\n if mi_unlikely(p == NULL) return mi_try_new(size, true);\n return p;\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) {\n void* p;\n do {\n p = mi_malloc_aligned(size, alignment);\n }\n while(p == NULL && mi_try_new_handler(false));\n return p;\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept {\n void* p;\n do {\n p = mi_malloc_aligned(size, alignment);\n }\n while(p == NULL && mi_try_new_handler(true));\n return p;\n}\n\nmi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) {\n void* q;\n do {\n q = mi_realloc(p, newsize);\n } while (q == NULL && mi_try_new_handler(false));\n return q;\n}\n\nmi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) {\n size_t total;\n if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) {\n mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc\n return NULL;\n }\n else {\n return mi_new_realloc(p, total);\n }\n}\n\n#if MI_GUARDED\n// We always allocate a guarded allocation at an offset (`mi_page_has_aligned` will be true).\n// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`)\n// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size`\n\nstatic void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) {\n // TODO: we can still make padding work by moving it out of the guard page area\n mi_page_t* const page = _mi_ptr_page(block);\n mi_page_set_has_aligned(page, true);\n block->next = MI_BLOCK_TAG_GUARDED;\n\n // set guard page at the end of the block\n mi_segment_t* const segment = _mi_page_segment(page);\n const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local`\n const size_t os_page_size = _mi_os_page_size();\n mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t));\n if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) {\n // should never happen\n mi_free(block);\n return NULL;\n }\n uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size;\n mi_assert_internal(_mi_is_aligned(guard_page, os_page_size));\n if mi_likely(segment->allow_decommit && _mi_is_aligned(guard_page, os_page_size)) {\n const bool ok = _mi_os_protect(guard_page, os_page_size);\n if mi_unlikely(!ok) {\n _mi_warning_message(\"failed to set a guard page behind an object (object %p of size %zu)\\n\", block, block_size);\n }\n }\n else {\n _mi_warning_message(\"unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\\n\", block, block_size);\n }\n\n // align pointer just in front of the guard page\n size_t offset = block_size - os_page_size - obj_size;\n mi_assert_internal(offset > sizeof(mi_block_t));\n if (offset > MI_BLOCK_ALIGNMENT_MAX) {\n // give up to place it right in front of the guard page if the offset is too large for unalignment\n offset = MI_BLOCK_ALIGNMENT_MAX;\n }\n uint8_t* const p = (uint8_t*)block + offset;\n mi_assert_internal(p == guard_page - obj_size);\n mi_track_align(block, p, offset, obj_size);\n mi_track_mem_defined(block, sizeof(mi_block_t));\n return p;\n}\n\nmi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept\n{\n // allocate multiple of page size ending in a guard page\n // ensure minimal alignment requirement?\n if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow \n _mi_error_message(EOVERFLOW, \"(guarded) allocation request is too large (%zu bytes)\\n\", size);\n return NULL;\n }\n const size_t os_page_size = _mi_os_page_size();\n const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));\n const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);\n const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size);\n mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(heap, req_size, false /* don't zero */, 0 /* huge_alignment */, NULL);\n if (block==NULL) return NULL;\n void* const p = mi_block_ptr_set_guarded(block, obj_size);\n if (p == NULL) return NULL; \n if (zero) {\n _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`)\n }\n\n // stats\n mi_track_malloc(p, obj_size, zero); \n if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); }\n _mi_stat_counter_increase(&heap->tld->stats.malloc_guarded_count, 1);\n #if MI_STAT>1\n // adjust stats to only count the allocated size of the block (and not the guard page)\n mi_heap_stat_adjust_decrease(heap, malloc_requested, req_size);\n mi_heap_stat_increase(heap, malloc_requested, size);\n #endif\n #if MI_DEBUG>3\n if (zero) {\n mi_assert_expensive(mi_mem_is_zero(p, size));\n }\n #endif\n return p;\n}\n#endif\n\n// ------------------------------------------------------\n// ensure explicit external inline definitions are emitted!\n// ------------------------------------------------------\n\n#ifdef __cplusplus\nvoid* _mi_externs[] = {\n (void*)&_mi_page_malloc,\n (void*)&_mi_page_malloc_zero,\n (void*)&_mi_heap_malloc_zero,\n (void*)&_mi_heap_malloc_zero_ex,\n (void*)&mi_malloc,\n (void*)&mi_malloc_small,\n (void*)&mi_zalloc_small,\n (void*)&mi_heap_malloc,\n (void*)&mi_heap_zalloc,\n (void*)&mi_heap_malloc_small,\n // (void*)&mi_heap_alloc_new,\n // (void*)&mi_heap_alloc_new_n\n};\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/arena-abandon.c", "sha256": "9a663611e9c73e42f73bbd42ea4c7869099b8e34c7a5968e27b373af8db1e951", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n#if !defined(MI_IN_ARENA_C)\n#error \"this file should be included from 'arena.c' (so mi_arena_t is visible)\"\n// add includes help an IDE\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"bitmap.h\"\n#endif\n\n// Minimal exports for arena-abandoned.\nsize_t mi_arena_id_index(mi_arena_id_t id);\nmi_arena_t* mi_arena_from_index(size_t idx);\nsize_t mi_arena_get_count(void);\nvoid* mi_arena_block_start(mi_arena_t* arena, mi_bitmap_index_t bindex);\nbool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bitmap_index_t* bitmap_index);\n\n/* -----------------------------------------------------------\n Abandoned blocks/segments:\n\n _mi_arena_segment_clear_abandoned\n _mi_arena_segment_mark_abandoned\n\n This is used to atomically abandon/reclaim segments\n (and crosses the arena API but it is convenient to have here).\n\n Abandoned segments still have live blocks; they get reclaimed\n when a thread frees a block in it, or when a thread needs a fresh\n segment.\n\n Abandoned segments are atomically marked in the `block_abandoned`\n bitmap of arenas. Any segments allocated outside arenas are put\n in the sub-process `abandoned_os_list`. This list is accessed\n using locks but this should be uncommon and generally uncontended.\n Reclaim and visiting either scan through the `block_abandoned`\n bitmaps of the arena's, or visit the `abandoned_os_list`\n\n A potentially nicer design is to use arena's for everything\n and perhaps have virtual arena's to map OS allocated memory\n but this would lack the \"density\" of our current arena's. TBC.\n----------------------------------------------------------- */\n\n\n// reclaim a specific OS abandoned segment; `true` on success.\n// sets the thread_id.\nstatic bool mi_arena_segment_os_clear_abandoned(mi_segment_t* segment, bool take_lock) {\n mi_assert(segment->memid.memkind != MI_MEM_ARENA);\n // not in an arena, remove from list of abandoned os segments\n mi_subproc_t* const subproc = segment->subproc;\n if (take_lock && !mi_lock_try_acquire(&subproc->abandoned_os_lock)) {\n return false; // failed to acquire the lock, we just give up\n }\n // remove atomically from the abandoned os list (if possible!)\n bool reclaimed = false;\n mi_segment_t* const next = segment->abandoned_os_next;\n mi_segment_t* const prev = segment->abandoned_os_prev;\n if (next != NULL || prev != NULL || subproc->abandoned_os_list == segment) {\n #if MI_DEBUG>3\n // find ourselves in the abandoned list (and check the count)\n bool found = false;\n size_t count = 0;\n for (mi_segment_t* current = subproc->abandoned_os_list; current != NULL; current = current->abandoned_os_next) {\n if (current == segment) { found = true; }\n count++;\n }\n mi_assert_internal(found);\n mi_assert_internal(count == mi_atomic_load_relaxed(&subproc->abandoned_os_list_count));\n #endif\n // remove (atomically) from the list and reclaim\n if (prev != NULL) { prev->abandoned_os_next = next; }\n else { subproc->abandoned_os_list = next; }\n if (next != NULL) { next->abandoned_os_prev = prev; }\n else { subproc->abandoned_os_list_tail = prev; }\n segment->abandoned_os_next = NULL;\n segment->abandoned_os_prev = NULL;\n mi_atomic_decrement_relaxed(&subproc->abandoned_count);\n mi_atomic_decrement_relaxed(&subproc->abandoned_os_list_count);\n if (take_lock) { // don't reset the thread_id when iterating\n mi_atomic_store_release(&segment->thread_id, _mi_thread_id());\n }\n reclaimed = true;\n }\n if (take_lock) { mi_lock_release(&segment->subproc->abandoned_os_lock); }\n return reclaimed;\n}\n\n// reclaim a specific abandoned segment; `true` on success.\n// sets the thread_id.\nbool _mi_arena_segment_clear_abandoned(mi_segment_t* segment) {\n if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) {\n return mi_arena_segment_os_clear_abandoned(segment, true /* take lock */);\n }\n // arena segment: use the blocks_abandoned bitmap.\n size_t arena_idx;\n size_t bitmap_idx;\n mi_arena_memid_indices(segment->memid, &arena_idx, &bitmap_idx);\n mi_arena_t* arena = mi_arena_from_index(arena_idx);\n mi_assert_internal(arena != NULL);\n // reclaim atomically\n bool was_marked = _mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx);\n if (was_marked) {\n mi_assert_internal(mi_atomic_load_acquire(&segment->thread_id) == 0);\n mi_atomic_decrement_relaxed(&segment->subproc->abandoned_count);\n mi_atomic_store_release(&segment->thread_id, _mi_thread_id());\n }\n // mi_assert_internal(was_marked);\n mi_assert_internal(!was_marked || _mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx));\n //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx));\n return was_marked;\n}\n\n\n// mark a specific OS segment as abandoned\nstatic void mi_arena_segment_os_mark_abandoned(mi_segment_t* segment) {\n mi_assert(segment->memid.memkind != MI_MEM_ARENA);\n // not in an arena; we use a list of abandoned segments\n mi_subproc_t* const subproc = segment->subproc;\n mi_lock(&subproc->abandoned_os_lock) {\n // push on the tail of the list (important for the visitor)\n mi_segment_t* prev = subproc->abandoned_os_list_tail;\n mi_assert_internal(prev == NULL || prev->abandoned_os_next == NULL);\n mi_assert_internal(segment->abandoned_os_prev == NULL);\n mi_assert_internal(segment->abandoned_os_next == NULL);\n if (prev != NULL) { prev->abandoned_os_next = segment; }\n else { subproc->abandoned_os_list = segment; }\n subproc->abandoned_os_list_tail = segment;\n segment->abandoned_os_prev = prev;\n segment->abandoned_os_next = NULL;\n mi_atomic_increment_relaxed(&subproc->abandoned_os_list_count);\n mi_atomic_increment_relaxed(&subproc->abandoned_count);\n // and release the lock\n }\n return;\n}\n\n// mark a specific segment as abandoned\n// clears the thread_id.\nvoid _mi_arena_segment_mark_abandoned(mi_segment_t* segment)\n{\n mi_assert_internal(segment->used == segment->abandoned);\n mi_atomic_store_release(&segment->thread_id, (uintptr_t)0); // mark as abandoned for multi-thread free's\n if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) {\n mi_arena_segment_os_mark_abandoned(segment);\n return;\n }\n // segment is in an arena, mark it in the arena `blocks_abandoned` bitmap\n size_t arena_idx;\n size_t bitmap_idx;\n mi_arena_memid_indices(segment->memid, &arena_idx, &bitmap_idx);\n mi_arena_t* arena = mi_arena_from_index(arena_idx);\n mi_assert_internal(arena != NULL);\n // set abandonment atomically\n mi_subproc_t* const subproc = segment->subproc; // don't access the segment after setting it abandoned\n const bool was_unmarked = _mi_bitmap_claim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx, NULL);\n if (was_unmarked) { mi_atomic_increment_relaxed(&subproc->abandoned_count); }\n mi_assert_internal(was_unmarked);\n mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx));\n}\n\n\n/* -----------------------------------------------------------\n Iterate through the abandoned blocks/segments using a cursor.\n This is used for reclaiming and abandoned block visiting.\n----------------------------------------------------------- */\n\n// start a cursor at a randomized arena\nvoid _mi_arena_field_cursor_init(mi_heap_t* heap, mi_subproc_t* subproc, bool visit_all, mi_arena_field_cursor_t* current) {\n mi_assert_internal(heap == NULL || heap->tld->segments.subproc == subproc);\n current->bitmap_idx = 0;\n current->subproc = subproc;\n current->visit_all = visit_all;\n current->hold_visit_lock = false;\n const size_t abandoned_count = mi_atomic_load_relaxed(&subproc->abandoned_count);\n const size_t abandoned_list_count = mi_atomic_load_relaxed(&subproc->abandoned_os_list_count);\n const size_t max_arena = mi_arena_get_count();\n if (heap != NULL && heap->arena_id != _mi_arena_id_none()) {\n // for a heap that is bound to one arena, only visit that arena\n current->start = mi_arena_id_index(heap->arena_id);\n current->end = current->start + 1;\n current->os_list_count = 0;\n }\n else {\n // otherwise visit all starting at a random location\n if (abandoned_count > abandoned_list_count && max_arena > 0) {\n current->start = (heap == NULL || max_arena == 0 ? 0 : (mi_arena_id_t)(_mi_heap_random_next(heap) % max_arena));\n current->end = current->start + max_arena;\n }\n else {\n current->start = 0;\n current->end = 0;\n }\n current->os_list_count = abandoned_list_count; // max entries to visit in the os abandoned list\n }\n mi_assert_internal(current->start <= max_arena);\n}\n\nvoid _mi_arena_field_cursor_done(mi_arena_field_cursor_t* current) {\n if (current->hold_visit_lock) {\n mi_lock_release(¤t->subproc->abandoned_os_visit_lock);\n current->hold_visit_lock = false;\n }\n}\n\nstatic mi_segment_t* mi_arena_segment_clear_abandoned_at(mi_arena_t* arena, mi_subproc_t* subproc, mi_bitmap_index_t bitmap_idx) {\n // try to reclaim an abandoned segment in the arena atomically\n if (!_mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx)) return NULL;\n mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx));\n mi_segment_t* segment = (mi_segment_t*)mi_arena_block_start(arena, bitmap_idx);\n mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0);\n // check that the segment belongs to our sub-process\n // note: this is the reason we need the `abandoned_visit` lock in the case abandoned visiting is enabled.\n // without the lock an abandoned visit may otherwise fail to visit all abandoned segments in the sub-process.\n // for regular reclaim it is fine to miss one sometimes so without abandoned visiting we don't need the `abandoned_visit` lock.\n if (segment->subproc != subproc) {\n // it is from another sub-process, re-mark it and continue searching\n const bool was_zero = _mi_bitmap_claim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx, NULL);\n mi_assert_internal(was_zero); MI_UNUSED(was_zero);\n return NULL;\n }\n else {\n // success, we unabandoned a segment in our sub-process\n mi_atomic_decrement_relaxed(&subproc->abandoned_count);\n return segment;\n }\n}\n\nstatic mi_segment_t* mi_arena_segment_clear_abandoned_next_field(mi_arena_field_cursor_t* previous) {\n const size_t max_arena = mi_arena_get_count();\n size_t field_idx = mi_bitmap_index_field(previous->bitmap_idx);\n size_t bit_idx = mi_bitmap_index_bit_in_field(previous->bitmap_idx);\n // visit arena's (from the previous cursor)\n for (; previous->start < previous->end; previous->start++, field_idx = 0, bit_idx = 0) {\n // index wraps around\n size_t arena_idx = (previous->start >= max_arena ? previous->start % max_arena : previous->start);\n mi_arena_t* arena = mi_arena_from_index(arena_idx);\n if (arena != NULL) {\n bool has_lock = false;\n // visit the abandoned fields (starting at previous_idx)\n for (; field_idx < arena->field_count; field_idx++, bit_idx = 0) {\n size_t field = mi_atomic_load_relaxed(&arena->blocks_abandoned[field_idx]);\n if mi_unlikely(field != 0) { // skip zero fields quickly\n // we only take the arena lock if there are actually abandoned segments present\n if (!has_lock && mi_option_is_enabled(mi_option_visit_abandoned)) {\n has_lock = (previous->visit_all ? (mi_lock_acquire(&arena->abandoned_visit_lock),true) : mi_lock_try_acquire(&arena->abandoned_visit_lock));\n if (!has_lock) {\n if (previous->visit_all) {\n _mi_error_message(EFAULT, \"internal error: failed to visit all abandoned segments due to failure to acquire the visitor lock\");\n }\n // skip to next arena\n break;\n }\n }\n mi_assert_internal(has_lock || !mi_option_is_enabled(mi_option_visit_abandoned));\n // visit each set bit in the field (todo: maybe use `ctz` here?)\n for (; bit_idx < MI_BITMAP_FIELD_BITS; bit_idx++) {\n // pre-check if the bit is set\n size_t mask = ((size_t)1 << bit_idx);\n if mi_unlikely((field & mask) == mask) {\n mi_bitmap_index_t bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx);\n mi_segment_t* const segment = mi_arena_segment_clear_abandoned_at(arena, previous->subproc, bitmap_idx);\n if (segment != NULL) {\n //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx));\n if (has_lock) { mi_lock_release(&arena->abandoned_visit_lock); }\n previous->bitmap_idx = mi_bitmap_index_create_ex(field_idx, bit_idx + 1); // start at next one for the next iteration\n return segment;\n }\n }\n }\n }\n }\n if (has_lock) { mi_lock_release(&arena->abandoned_visit_lock); }\n }\n }\n return NULL;\n}\n\nstatic mi_segment_t* mi_arena_segment_clear_abandoned_next_list(mi_arena_field_cursor_t* previous) {\n // go through the abandoned_os_list\n // we only allow one thread per sub-process to do to visit guarded by the `abandoned_os_visit_lock`.\n // The lock is released when the cursor is released.\n if (!previous->hold_visit_lock) {\n previous->hold_visit_lock = (previous->visit_all ? (mi_lock_acquire(&previous->subproc->abandoned_os_visit_lock),true)\n : mi_lock_try_acquire(&previous->subproc->abandoned_os_visit_lock));\n if (!previous->hold_visit_lock) {\n if (previous->visit_all) {\n _mi_error_message(EFAULT, \"internal error: failed to visit all abandoned segments due to failure to acquire the OS visitor lock\");\n }\n return NULL; // we cannot get the lock, give up\n }\n }\n // One list entry at a time\n while (previous->os_list_count > 0) {\n previous->os_list_count--;\n mi_lock_acquire(&previous->subproc->abandoned_os_lock); // this could contend with concurrent OS block abandonment and reclaim from `free`\n mi_segment_t* segment = previous->subproc->abandoned_os_list;\n // pop from head of the list, a subsequent mark will push at the end (and thus we iterate through os_list_count entries)\n if (segment == NULL || mi_arena_segment_os_clear_abandoned(segment, false /* we already have the lock */)) {\n mi_lock_release(&previous->subproc->abandoned_os_lock);\n return segment;\n }\n // already abandoned, try again\n mi_lock_release(&previous->subproc->abandoned_os_lock);\n }\n // done\n mi_assert_internal(previous->os_list_count == 0);\n return NULL;\n}\n\n\n// reclaim abandoned segments\n// this does not set the thread id (so it appears as still abandoned)\nmi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous) {\n if (previous->start < previous->end) {\n // walk the arena\n mi_segment_t* segment = mi_arena_segment_clear_abandoned_next_field(previous);\n if (segment != NULL) { return segment; }\n }\n // no entries in the arena's anymore, walk the abandoned OS list\n mi_assert_internal(previous->start == previous->end);\n return mi_arena_segment_clear_abandoned_next_list(previous);\n}\n\n\nbool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {\n // (unfortunately) the visit_abandoned option must be enabled from the start.\n // This is to avoid taking locks if abandoned list visiting is not required (as for most programs)\n if (!mi_option_is_enabled(mi_option_visit_abandoned)) {\n _mi_error_message(EFAULT, \"internal error: can only visit abandoned blocks when MIMALLOC_VISIT_ABANDONED=ON\");\n return false;\n }\n mi_arena_field_cursor_t current;\n _mi_arena_field_cursor_init(NULL, _mi_subproc_from_id(subproc_id), true /* visit all (blocking) */, ¤t);\n mi_segment_t* segment;\n bool ok = true;\n while (ok && (segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL) {\n ok = _mi_segment_visit_blocks(segment, heap_tag, visit_blocks, visitor, arg);\n _mi_arena_segment_mark_abandoned(segment);\n }\n _mi_arena_field_cursor_done(¤t);\n return ok;\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/arena.c", "sha256": "029ed6c1f61a943127b90a5e1be5e7217bfac6b905512709dc0a6d0795e98b28", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* ----------------------------------------------------------------------------\n\"Arenas\" are fixed area's of OS memory from which we can allocate\nlarge blocks (>= MI_ARENA_MIN_BLOCK_SIZE, 4MiB).\nIn contrast to the rest of mimalloc, the arenas are shared between\nthreads and need to be accessed using atomic operations.\n\nArenas are also used to for huge OS page (1GiB) reservations or for reserving\nOS memory upfront which can be improve performance or is sometimes needed\non embedded devices. We can also employ this with WASI or `sbrk` systems\nto reserve large arenas upfront and be able to reuse the memory more effectively.\n\nThe arena allocation needs to be thread safe and we use an atomic bitmap to allocate.\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"bitmap.h\"\n\n\n/* -----------------------------------------------------------\n Arena allocation\n----------------------------------------------------------- */\n\n// A memory arena descriptor\ntypedef struct mi_arena_s {\n mi_arena_id_t id; // arena id; 0 for non-specific\n mi_memid_t memid; // memid of the memory area\n _Atomic(uint8_t*) start; // the start of the memory area\n size_t block_count; // size of the area in arena blocks (of `MI_ARENA_BLOCK_SIZE`)\n size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`)\n size_t meta_size; // size of the arena structure itself (including its bitmaps)\n mi_memid_t meta_memid; // memid of the arena structure itself (OS or static allocation)\n int numa_node; // associated NUMA node\n bool exclusive; // only allow allocations if specifically for this arena\n bool is_large; // memory area consists of large- or huge OS pages (always committed)\n mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited\n _Atomic(size_t) search_idx; // optimization to start the search for free blocks\n _Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be purged from `blocks_purge`.\n\n mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero?\n mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted)\n mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted)\n mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here)\n mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`)\n // do not add further fields here as the dirty, committed, purged, and abandoned bitmaps follow the inuse bitmap fields.\n} mi_arena_t;\n\n\n#define MI_ARENA_BLOCK_SIZE (MI_SEGMENT_SIZE) // 64MiB (must be at least MI_SEGMENT_ALIGN)\n#define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_BLOCK_SIZE/2) // 32MiB\n#define MI_MAX_ARENAS (132) // Limited as the reservation exponentially increases (and takes up .bss)\n\n// The available arenas\nstatic mi_decl_cache_align _Atomic(mi_arena_t*) mi_arenas[MI_MAX_ARENAS];\nstatic mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0\nstatic mi_decl_cache_align _Atomic(int64_t) mi_arenas_purge_expire; // set if there exist purgeable arenas\n\n#define MI_IN_ARENA_C\n#include \"arena-abandon.c\"\n#undef MI_IN_ARENA_C\n\n/* -----------------------------------------------------------\n Arena id's\n id = arena_index + 1\n----------------------------------------------------------- */\n\nsize_t mi_arena_id_index(mi_arena_id_t id) {\n return (size_t)(id <= 0 ? MI_MAX_ARENAS : id - 1);\n}\n\nstatic mi_arena_id_t mi_arena_id_create(size_t arena_index) {\n mi_assert_internal(arena_index < MI_MAX_ARENAS);\n return (int)arena_index + 1;\n}\n\nmi_arena_id_t _mi_arena_id_none(void) {\n return 0;\n}\n\nstatic bool mi_arena_id_is_suitable(mi_arena_id_t arena_id, bool arena_is_exclusive, mi_arena_id_t req_arena_id) {\n return ((!arena_is_exclusive && req_arena_id == _mi_arena_id_none()) ||\n (arena_id == req_arena_id));\n}\n\nbool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id) {\n if (memid.memkind == MI_MEM_ARENA) {\n return mi_arena_id_is_suitable(memid.mem.arena.id, memid.mem.arena.is_exclusive, request_arena_id);\n }\n else {\n return mi_arena_id_is_suitable(_mi_arena_id_none(), false, request_arena_id);\n }\n}\n\nbool _mi_arena_memid_is_os_allocated(mi_memid_t memid) {\n return (memid.memkind == MI_MEM_OS);\n}\n\nsize_t mi_arena_get_count(void) {\n return mi_atomic_load_relaxed(&mi_arena_count);\n}\n\nmi_arena_t* mi_arena_from_index(size_t idx) {\n mi_assert_internal(idx < mi_arena_get_count());\n return mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[idx]);\n}\n\n\n/* -----------------------------------------------------------\n Arena allocations get a (currently) 16-bit memory id where the\n lower 8 bits are the arena id, and the upper bits the block index.\n----------------------------------------------------------- */\n\nstatic size_t mi_block_count_of_size(size_t size) {\n return _mi_divide_up(size, MI_ARENA_BLOCK_SIZE);\n}\n\nstatic size_t mi_arena_block_size(size_t bcount) {\n return (bcount * MI_ARENA_BLOCK_SIZE);\n}\n\nstatic size_t mi_arena_size(mi_arena_t* arena) {\n return mi_arena_block_size(arena->block_count);\n}\n\nstatic mi_memid_t mi_memid_create_arena(mi_arena_id_t id, bool is_exclusive, mi_bitmap_index_t bitmap_index) {\n mi_memid_t memid = _mi_memid_create(MI_MEM_ARENA);\n memid.mem.arena.id = id;\n memid.mem.arena.block_index = bitmap_index;\n memid.mem.arena.is_exclusive = is_exclusive;\n return memid;\n}\n\nbool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bitmap_index_t* bitmap_index) {\n mi_assert_internal(memid.memkind == MI_MEM_ARENA);\n *arena_index = mi_arena_id_index(memid.mem.arena.id);\n *bitmap_index = memid.mem.arena.block_index;\n return memid.mem.arena.is_exclusive;\n}\n\n\n\n/* -----------------------------------------------------------\n Special static area for mimalloc internal structures\n to avoid OS calls (for example, for the arena metadata (~= 256b))\n----------------------------------------------------------- */\n\n#define MI_ARENA_STATIC_MAX ((MI_INTPTR_SIZE/2)*MI_KiB) // 4 KiB on 64-bit\n\nstatic mi_decl_cache_align uint8_t mi_arena_static[MI_ARENA_STATIC_MAX]; // must be cache aligned, see issue #895\nstatic mi_decl_cache_align _Atomic(size_t) mi_arena_static_top;\n\nstatic void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* memid) {\n *memid = _mi_memid_none();\n if (size == 0 || size > MI_ARENA_STATIC_MAX) return NULL;\n const size_t toplow = mi_atomic_load_relaxed(&mi_arena_static_top);\n if ((toplow + size) > MI_ARENA_STATIC_MAX) return NULL;\n\n // try to claim space\n if (alignment < MI_MAX_ALIGN_SIZE) { alignment = MI_MAX_ALIGN_SIZE; }\n const size_t oversize = size + alignment - 1;\n if (toplow + oversize > MI_ARENA_STATIC_MAX) return NULL;\n const size_t oldtop = mi_atomic_add_acq_rel(&mi_arena_static_top, oversize);\n size_t top = oldtop + oversize;\n if (top > MI_ARENA_STATIC_MAX) {\n // try to roll back, ok if this fails\n mi_atomic_cas_strong_acq_rel(&mi_arena_static_top, &top, oldtop);\n return NULL;\n }\n\n // success\n *memid = _mi_memid_create(MI_MEM_STATIC);\n memid->initially_zero = true;\n const size_t start = _mi_align_up(oldtop, alignment);\n uint8_t* const p = &mi_arena_static[start];\n _mi_memzero_aligned(p, size);\n return p;\n}\n\nvoid* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid) {\n *memid = _mi_memid_none();\n\n // try static\n void* p = mi_arena_static_zalloc(size, MI_MAX_ALIGN_SIZE, memid);\n if (p != NULL) return p;\n\n // or fall back to the OS\n p = _mi_os_zalloc(size, memid);\n if (p == NULL) return NULL;\n\n return p;\n}\n\nvoid _mi_arena_meta_free(void* p, mi_memid_t memid, size_t size) {\n if (mi_memkind_is_os(memid.memkind)) {\n _mi_os_free(p, size, memid);\n }\n else {\n mi_assert(memid.memkind == MI_MEM_STATIC);\n }\n}\n\nvoid* mi_arena_block_start(mi_arena_t* arena, mi_bitmap_index_t bindex) {\n return (arena->start + mi_arena_block_size(mi_bitmap_index_bit(bindex)));\n}\n\n\n/* -----------------------------------------------------------\n Thread safe allocation in an arena\n----------------------------------------------------------- */\n\n// claim the `blocks_inuse` bits\nstatic bool mi_arena_try_claim(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t* bitmap_idx)\n{\n size_t idx = 0; // mi_atomic_load_relaxed(&arena->search_idx); // start from last search; ok to be relaxed as the exact start does not matter\n if (_mi_bitmap_try_find_from_claim_across(arena->blocks_inuse, arena->field_count, idx, blocks, bitmap_idx)) {\n mi_atomic_store_relaxed(&arena->search_idx, mi_bitmap_index_field(*bitmap_idx)); // start search from found location next time around\n return true;\n };\n return false;\n}\n\n\n/* -----------------------------------------------------------\n Arena Allocation\n----------------------------------------------------------- */\n\nstatic mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t arena_index, size_t needed_bcount,\n bool commit, mi_memid_t* memid)\n{\n MI_UNUSED(arena_index);\n mi_assert_internal(mi_arena_id_index(arena->id) == arena_index);\n\n mi_bitmap_index_t bitmap_index;\n if (!mi_arena_try_claim(arena, needed_bcount, &bitmap_index)) return NULL;\n\n // claimed it!\n void* p = mi_arena_block_start(arena, bitmap_index);\n *memid = mi_memid_create_arena(arena->id, arena->exclusive, bitmap_index);\n memid->is_pinned = arena->memid.is_pinned;\n\n // none of the claimed blocks should be scheduled for a decommit\n if (arena->blocks_purge != NULL) {\n // this is thread safe as a potential purge only decommits parts that are not yet claimed as used (in `blocks_inuse`).\n _mi_bitmap_unclaim_across(arena->blocks_purge, arena->field_count, needed_bcount, bitmap_index);\n }\n\n // set the dirty bits (todo: no need for an atomic op here?)\n if (arena->memid.initially_zero && arena->blocks_dirty != NULL) {\n memid->initially_zero = _mi_bitmap_claim_across(arena->blocks_dirty, arena->field_count, needed_bcount, bitmap_index, NULL, NULL);\n }\n\n // set commit state\n if (arena->blocks_committed == NULL) {\n // always committed\n memid->initially_committed = true;\n }\n else if (commit) {\n // commit requested, but the range may not be committed as a whole: ensure it is committed now\n memid->initially_committed = true;\n const size_t commit_size = mi_arena_block_size(needed_bcount); \n bool any_uncommitted;\n size_t already_committed = 0;\n _mi_bitmap_claim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted, &already_committed);\n if (any_uncommitted) {\n mi_assert_internal(already_committed < needed_bcount);\n const size_t stat_commit_size = commit_size - mi_arena_block_size(already_committed);\n bool commit_zero = false;\n if (!_mi_os_commit_ex(p, commit_size, &commit_zero, stat_commit_size)) {\n // set all as uncommitted on commit failure\n _mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index);\n memid->initially_committed = false;\n }\n else {\n if (commit_zero) { memid->initially_zero = true; }\n }\n }\n else {\n // all are already committed: signal that we are reusing memory in case it was purged before\n _mi_os_reuse( p, commit_size );\n }\n }\n else {\n // no need to commit, but check if already fully committed\n size_t already_committed = 0;\n memid->initially_committed = _mi_bitmap_is_claimed_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &already_committed);\n if (!memid->initially_committed && already_committed > 0) {\n // partially committed: as it will be committed at some time, adjust the stats and pretend the range is fully uncommitted.\n mi_assert_internal(already_committed < needed_bcount);\n _mi_stat_decrease(&_mi_stats_main.committed, mi_arena_block_size(already_committed));\n _mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index);\n }\n }\n\n return p;\n}\n\n// allocate in a specific arena\nstatic void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_node, int numa_node, size_t size, size_t alignment,\n bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid )\n{\n MI_UNUSED_RELEASE(alignment);\n mi_assert(alignment <= MI_SEGMENT_ALIGN);\n const size_t bcount = mi_block_count_of_size(size);\n const size_t arena_index = mi_arena_id_index(arena_id);\n mi_assert_internal(arena_index < mi_atomic_load_relaxed(&mi_arena_count));\n mi_assert_internal(size <= mi_arena_block_size(bcount));\n\n // Check arena suitability\n mi_arena_t* arena = mi_arena_from_index(arena_index);\n if (arena == NULL) return NULL;\n if (!allow_large && arena->is_large) return NULL;\n if (!mi_arena_id_is_suitable(arena->id, arena->exclusive, req_arena_id)) return NULL;\n if (req_arena_id == _mi_arena_id_none()) { // in not specific, check numa affinity\n const bool numa_suitable = (numa_node < 0 || arena->numa_node < 0 || arena->numa_node == numa_node);\n if (match_numa_node) { if (!numa_suitable) return NULL; }\n else { if (numa_suitable) return NULL; }\n }\n\n // try to allocate\n void* p = mi_arena_try_alloc_at(arena, arena_index, bcount, commit, memid);\n mi_assert_internal(p == NULL || _mi_is_aligned(p, alignment));\n return p;\n}\n\n\n// allocate from an arena with fallback to the OS\nstatic mi_decl_noinline void* mi_arena_try_alloc(int numa_node, size_t size, size_t alignment,\n bool commit, bool allow_large,\n mi_arena_id_t req_arena_id, mi_memid_t* memid )\n{\n MI_UNUSED(alignment);\n mi_assert_internal(alignment <= MI_SEGMENT_ALIGN);\n const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count);\n if mi_likely(max_arena == 0) return NULL;\n\n if (req_arena_id != _mi_arena_id_none()) {\n // try a specific arena if requested\n if (mi_arena_id_index(req_arena_id) < max_arena) {\n void* p = mi_arena_try_alloc_at_id(req_arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid);\n if (p != NULL) return p;\n }\n }\n else {\n // try numa affine allocation\n for (size_t i = 0; i < max_arena; i++) {\n void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid);\n if (p != NULL) return p;\n }\n\n // try from another numa node instead..\n if (numa_node >= 0) { // if numa_node was < 0 (no specific affinity requested), all arena's have been tried already\n for (size_t i = 0; i < max_arena; i++) {\n void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), false /* only proceed if not numa local */, numa_node, size, alignment, commit, allow_large, req_arena_id, memid);\n if (p != NULL) return p;\n }\n }\n }\n return NULL;\n}\n\n// try to reserve a fresh arena space\nstatic bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t *arena_id)\n{\n if (_mi_preloading()) return false; // use OS only while pre loading\n\n const size_t arena_count = mi_atomic_load_acquire(&mi_arena_count);\n if (arena_count > (MI_MAX_ARENAS - 4)) return false;\n\n size_t arena_reserve = mi_option_get_size(mi_option_arena_reserve);\n if (arena_reserve == 0) return false;\n\n if (!_mi_os_has_virtual_reserve()) {\n arena_reserve = arena_reserve/4; // be conservative if virtual reserve is not supported (for WASM for example)\n }\n arena_reserve = _mi_align_up(arena_reserve, MI_ARENA_BLOCK_SIZE);\n arena_reserve = _mi_align_up(arena_reserve, MI_SEGMENT_SIZE);\n if (arena_count >= 8 && arena_count <= 128) {\n // scale up the arena sizes exponentially every 8 entries (128 entries get to 589TiB)\n const size_t multiplier = (size_t)1 << _mi_clamp(arena_count/8, 0, 16 );\n size_t reserve = 0;\n if (!mi_mul_overflow(multiplier, arena_reserve, &reserve)) {\n arena_reserve = reserve;\n }\n }\n if (arena_reserve < req_size) return false; // should be able to at least handle the current allocation size\n\n // commit eagerly?\n bool arena_commit = false;\n if (mi_option_get(mi_option_arena_eager_commit) == 2) { arena_commit = _mi_os_has_overcommit() || mi_option_is_enabled(mi_option_allow_large_os_pages); }\n else if (mi_option_get(mi_option_arena_eager_commit) == 1) { arena_commit = true; }\n\n return (mi_reserve_os_memory_ex(arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id) == 0);\n}\n\n\nvoid* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large,\n mi_arena_id_t req_arena_id, mi_memid_t* memid)\n{\n mi_assert_internal(memid != NULL);\n mi_assert_internal(size > 0);\n *memid = _mi_memid_none();\n\n const int numa_node = _mi_os_numa_node(); // current numa node\n\n // try to allocate in an arena if the alignment is small enough and the object is not too small (as for heap meta data)\n if (!mi_option_is_enabled(mi_option_disallow_arena_alloc)) { // is arena allocation allowed?\n if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0)\n {\n void* p = mi_arena_try_alloc(numa_node, size, alignment, commit, allow_large, req_arena_id, memid);\n if (p != NULL) return p;\n\n // otherwise, try to first eagerly reserve a new arena\n if (req_arena_id == _mi_arena_id_none()) {\n mi_arena_id_t arena_id = 0;\n if (mi_arena_reserve(size, allow_large, &arena_id)) {\n // and try allocate in there\n mi_assert_internal(req_arena_id == _mi_arena_id_none());\n p = mi_arena_try_alloc_at_id(arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid);\n if (p != NULL) return p;\n }\n }\n }\n }\n\n // if we cannot use OS allocation, return NULL\n if (mi_option_is_enabled(mi_option_disallow_os_alloc) || req_arena_id != _mi_arena_id_none()) {\n errno = ENOMEM;\n return NULL;\n }\n\n // finally, fall back to the OS\n if (align_offset > 0) {\n return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid);\n }\n else {\n return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);\n }\n}\n\nvoid* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid)\n{\n return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, 0, commit, allow_large, req_arena_id, memid);\n}\n\n\nvoid* mi_arena_area(mi_arena_id_t arena_id, size_t* size) {\n if (size != NULL) *size = 0;\n size_t arena_index = mi_arena_id_index(arena_id);\n if (arena_index >= MI_MAX_ARENAS) return NULL;\n mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_index]);\n if (arena == NULL) return NULL;\n if (size != NULL) { *size = mi_arena_block_size(arena->block_count); }\n return arena->start;\n}\n\n\n/* -----------------------------------------------------------\n Arena purge\n----------------------------------------------------------- */\n\nstatic long mi_arena_purge_delay(void) {\n // <0 = no purging allowed, 0=immediate purging, >0=milli-second delay\n return (mi_option_get(mi_option_purge_delay) * mi_option_get(mi_option_arena_purge_mult));\n}\n\n// reset or decommit in an arena and update the committed/decommit bitmaps\n// assumes we own the area (i.e. blocks_in_use is claimed by us)\nstatic void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks) {\n mi_assert_internal(arena->blocks_committed != NULL);\n mi_assert_internal(arena->blocks_purge != NULL);\n mi_assert_internal(!arena->memid.is_pinned);\n const size_t size = mi_arena_block_size(blocks);\n void* const p = mi_arena_block_start(arena, bitmap_idx);\n bool needs_recommit;\n size_t already_committed = 0;\n if (_mi_bitmap_is_claimed_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx, &already_committed)) {\n // all blocks are committed, we can purge freely\n mi_assert_internal(already_committed == blocks);\n needs_recommit = _mi_os_purge(p, size);\n }\n else {\n // some blocks are not committed -- this can happen when a partially committed block is freed\n // in `_mi_arena_free` and it is conservatively marked as uncommitted but still scheduled for a purge\n // we need to ensure we do not try to reset (as that may be invalid for uncommitted memory).\n mi_assert_internal(already_committed < blocks);\n mi_assert_internal(mi_option_is_enabled(mi_option_purge_decommits));\n needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, mi_arena_block_size(already_committed));\n }\n\n // clear the purged blocks\n _mi_bitmap_unclaim_across(arena->blocks_purge, arena->field_count, blocks, bitmap_idx);\n // update committed bitmap\n if (needs_recommit) {\n _mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx);\n }\n}\n\n// Schedule a purge. This is usually delayed to avoid repeated decommit/commit calls.\n// Note: assumes we (still) own the area as we may purge immediately\nstatic void mi_arena_schedule_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks) {\n mi_assert_internal(arena->blocks_purge != NULL);\n const long delay = mi_arena_purge_delay();\n if (delay < 0) return; // is purging allowed at all?\n\n if (_mi_preloading() || delay == 0) {\n // decommit directly\n mi_arena_purge(arena, bitmap_idx, blocks);\n }\n else {\n // schedule purge\n const mi_msecs_t expire = _mi_clock_now() + delay;\n mi_msecs_t expire0 = 0;\n if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) {\n // expiration was not yet set\n // maybe set the global arenas expire as well (if it wasn't set already)\n mi_atomic_casi64_strong_acq_rel(&mi_arenas_purge_expire, &expire0, expire);\n }\n else {\n // already an expiration was set\n }\n _mi_bitmap_claim_across(arena->blocks_purge, arena->field_count, blocks, bitmap_idx, NULL, NULL);\n }\n}\n\n// purge a range of blocks\n// return true if the full range was purged.\n// assumes we own the area (i.e. blocks_in_use is claimed by us)\nstatic bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx, size_t bitlen, size_t purge) {\n const size_t endidx = startidx + bitlen;\n size_t bitidx = startidx;\n bool all_purged = false;\n while (bitidx < endidx) {\n // count consecutive ones in the purge mask\n size_t count = 0;\n while (bitidx + count < endidx && (purge & ((size_t)1 << (bitidx + count))) != 0) {\n count++;\n }\n if (count > 0) {\n // found range to be purged\n const mi_bitmap_index_t range_idx = mi_bitmap_index_create(idx, bitidx);\n mi_arena_purge(arena, range_idx, count);\n if (count == bitlen) {\n all_purged = true;\n }\n }\n bitidx += (count+1); // +1 to skip the zero bit (or end)\n }\n return all_purged;\n}\n\n// returns \n// -1 = nothing was purged \n// 0 = nothing was purged yet because have not yet reached the expire time\n// 1 = some pages in the arena were purged\nstatic int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)\n{\n // check pre-conditions\n if (arena->memid.is_pinned) return -1;\n\n // expired yet?\n mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire);\n if (!force) {\n if (expire == 0) return -1;\n if (expire > now) return 0;\n }\n \n // reset expire (if not already set concurrently)\n mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire, (mi_msecs_t)0);\n _mi_stat_counter_increase(&_mi_stats_main.arena_purges, 1);\n\n // potential purges scheduled, walk through the bitmap\n bool any_purged = false;\n bool full_purge = true;\n for (size_t i = 0; i < arena->field_count; i++) {\n size_t purge = mi_atomic_load_relaxed(&arena->blocks_purge[i]);\n if (purge != 0) {\n size_t bitidx = 0;\n while (bitidx < MI_BITMAP_FIELD_BITS) {\n // find consecutive range of ones in the purge mask\n size_t bitlen = 0;\n while (bitidx + bitlen < MI_BITMAP_FIELD_BITS && (purge & ((size_t)1 << (bitidx + bitlen))) != 0) {\n bitlen++;\n }\n // temporarily claim the purge range as \"in-use\" to be thread-safe with allocation\n // try to claim the longest range of corresponding in_use bits\n const mi_bitmap_index_t bitmap_index = mi_bitmap_index_create(i, bitidx);\n while( bitlen > 0 ) {\n if (_mi_bitmap_try_claim(arena->blocks_inuse, arena->field_count, bitlen, bitmap_index)) {\n break;\n }\n bitlen--;\n }\n // actual claimed bits at `in_use`\n if (bitlen > 0) {\n // read purge again now that we have the in_use bits\n purge = mi_atomic_load_acquire(&arena->blocks_purge[i]);\n if (!mi_arena_purge_range(arena, i, bitidx, bitlen, purge)) {\n full_purge = false;\n }\n any_purged = true;\n // release the claimed `in_use` bits again\n _mi_bitmap_unclaim(arena->blocks_inuse, arena->field_count, bitlen, bitmap_index);\n }\n bitidx += (bitlen+1); // +1 to skip the zero (or end)\n } // while bitidx\n } // purge != 0\n }\n // if not fully purged, make sure to purge again in the future\n if (!full_purge) {\n const long delay = mi_arena_purge_delay();\n mi_msecs_t expected = 0;\n mi_atomic_casi64_strong_acq_rel(&arena->purge_expire,&expected,_mi_clock_now() + delay);\n }\n return (any_purged ? 1 : -1);\n}\n\nstatic void mi_arenas_try_purge( bool force, bool visit_all )\n{\n if (_mi_preloading() || mi_arena_purge_delay() <= 0) return; // nothing will be scheduled\n\n // check if any arena needs purging?\n const mi_msecs_t now = _mi_clock_now();\n mi_msecs_t arenas_expire = mi_atomic_loadi64_acquire(&mi_arenas_purge_expire);\n if (!force && (arenas_expire == 0 || arenas_expire > now)) return;\n\n const size_t max_arena = mi_atomic_load_acquire(&mi_arena_count);\n if (max_arena == 0) return;\n\n // allow only one thread to purge at a time\n static mi_atomic_guard_t purge_guard;\n mi_atomic_guard(&purge_guard)\n {\n // increase global expire: at most one purge per delay cycle\n mi_atomic_storei64_release(&mi_arenas_purge_expire, now + mi_arena_purge_delay());\n size_t max_purge_count = (visit_all ? max_arena : 2);\n bool all_visited = true;\n bool any_purged = false;\n for (size_t i = 0; i < max_arena; i++) {\n mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]);\n if (arena != NULL) {\n int purged = mi_arena_try_purge(arena, now, force);\n if (purged >= 0) { // purged, or not yet the expire-time reached\n any_purged = true;\n if (purged >= 1) { // purged at least one page\n if (max_purge_count <= 1) {\n all_visited = false;\n break;\n }\n max_purge_count--;\n }\n }\n }\n }\n if (all_visited && !any_purged) {\n // all arena's were visited and nothing needed to be purged: reset global expire\n mi_atomic_storei64_release(&mi_arenas_purge_expire, (mi_msecs_t)0);\n }\n }\n}\n\n\n/* -----------------------------------------------------------\n Arena free\n----------------------------------------------------------- */\n\nvoid _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memid) {\n mi_assert_internal(size > 0);\n mi_assert_internal(committed_size <= size);\n if (p==NULL) return;\n if (size==0) return;\n const bool all_committed = (committed_size == size);\n const size_t decommitted_size = (committed_size <= size ? size - committed_size : 0);\n\n // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.)\n mi_track_mem_undefined(p,size);\n\n if (mi_memkind_is_os(memid.memkind)) {\n // was a direct OS allocation, pass through\n if (!all_committed && decommitted_size > 0) {\n // if partially committed, adjust the committed stats (as `_mi_os_free` will decrease commit by the full size)\n _mi_stat_increase(&_mi_stats_main.committed, decommitted_size);\n }\n _mi_os_free(p, size, memid);\n }\n else if (memid.memkind == MI_MEM_ARENA) {\n // allocated in an arena\n size_t arena_idx;\n size_t bitmap_idx;\n mi_arena_memid_indices(memid, &arena_idx, &bitmap_idx);\n mi_assert_internal(arena_idx < MI_MAX_ARENAS);\n mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t,&mi_arenas[arena_idx]);\n mi_assert_internal(arena != NULL);\n const size_t blocks = mi_block_count_of_size(size);\n\n // checks\n if (arena == NULL) {\n _mi_error_message(EINVAL, \"trying to free from an invalid arena: %p, size %zu, memid: 0x%zx\\n\", p, size, memid);\n return;\n }\n mi_assert_internal(arena->field_count > mi_bitmap_index_field(bitmap_idx));\n if (arena->field_count <= mi_bitmap_index_field(bitmap_idx)) {\n _mi_error_message(EINVAL, \"trying to free from an invalid arena block: %p, size %zu, memid: 0x%zx\\n\", p, size, memid);\n return;\n }\n\n // potentially decommit\n if (arena->memid.is_pinned || arena->blocks_committed == NULL) {\n mi_assert_internal(all_committed);\n }\n else {\n mi_assert_internal(arena->blocks_committed != NULL);\n mi_assert_internal(arena->blocks_purge != NULL);\n\n if (!all_committed) {\n // mark the entire range as no longer committed (so we will recommit the full range when re-using)\n _mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx);\n mi_track_mem_noaccess(p,size);\n //if (committed_size > 0) {\n // if partially committed, adjust the committed stats (is it will be recommitted when re-using)\n // in the delayed purge, we do no longer decrease the commit if the range is not marked entirely as committed.\n _mi_stat_decrease(&_mi_stats_main.committed, committed_size);\n //}\n // note: if not all committed, it may be that the purge will reset/decommit the entire range\n // that contains already decommitted parts. Since purge consistently uses reset or decommit that\n // works (as we should never reset decommitted parts).\n }\n // (delay) purge the entire range\n mi_arena_schedule_purge(arena, bitmap_idx, blocks);\n }\n\n // and make it available to others again\n bool all_inuse = _mi_bitmap_unclaim_across(arena->blocks_inuse, arena->field_count, blocks, bitmap_idx);\n if (!all_inuse) {\n _mi_error_message(EAGAIN, \"trying to free an already freed arena block: %p, size %zu\\n\", p, size);\n return;\n };\n }\n else {\n // arena was none, external, or static; nothing to do\n mi_assert_internal(memid.memkind < MI_MEM_OS);\n }\n\n // purge expired decommits\n mi_arenas_try_purge(false, false);\n}\n\n// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`\n// for dynamic libraries that are unloaded and need to release all their allocated memory.\nstatic void mi_arenas_unsafe_destroy(void) {\n const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count);\n size_t new_max_arena = 0;\n for (size_t i = 0; i < max_arena; i++) {\n mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]);\n if (arena != NULL) {\n mi_lock_done(&arena->abandoned_visit_lock);\n if (arena->start != NULL && mi_memkind_is_os(arena->memid.memkind)) {\n mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[i], NULL);\n _mi_os_free(arena->start, mi_arena_size(arena), arena->memid);\n }\n else {\n new_max_arena = i;\n }\n _mi_arena_meta_free(arena, arena->meta_memid, arena->meta_size);\n }\n }\n\n // try to lower the max arena.\n size_t expected = max_arena;\n mi_atomic_cas_strong_acq_rel(&mi_arena_count, &expected, new_max_arena);\n}\n\n// Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired\nvoid _mi_arenas_collect(bool force_purge) {\n mi_arenas_try_purge(force_purge, force_purge /* visit all? */);\n}\n\n// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`\n// for dynamic libraries that are unloaded and need to release all their allocated memory.\nvoid _mi_arena_unsafe_destroy_all(void) {\n mi_arenas_unsafe_destroy();\n _mi_arenas_collect(true /* force purge */); // purge non-owned arenas\n}\n\n// Is a pointer inside any of our arenas?\nbool _mi_arena_contains(const void* p) {\n const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count);\n for (size_t i = 0; i < max_arena; i++) {\n mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]);\n if (arena != NULL && arena->start <= (const uint8_t*)p && arena->start + mi_arena_block_size(arena->block_count) > (const uint8_t*)p) {\n return true;\n }\n }\n return false;\n}\n\n/* -----------------------------------------------------------\n Add an arena.\n----------------------------------------------------------- */\n\nstatic bool mi_arena_add(mi_arena_t* arena, mi_arena_id_t* arena_id, mi_stats_t* stats) {\n mi_assert_internal(arena != NULL);\n mi_assert_internal((uintptr_t)mi_atomic_load_ptr_relaxed(uint8_t,&arena->start) % MI_SEGMENT_ALIGN == 0);\n mi_assert_internal(arena->block_count > 0);\n if (arena_id != NULL) { *arena_id = -1; }\n\n size_t i = mi_atomic_load_relaxed(&mi_arena_count);\n while (i < MI_MAX_ARENAS) {\n if (mi_atomic_cas_strong_acq_rel(&mi_arena_count, &i, i+1)) {\n _mi_stat_counter_increase(&stats->arena_count, 1);\n arena->id = mi_arena_id_create(i);\n mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[i], arena);\n if (arena_id != NULL) { *arena_id = arena->id; }\n return true;\n }\n }\n\n return false;\n}\n\nstatic bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int numa_node, bool exclusive, mi_memid_t memid, mi_arena_id_t* arena_id) mi_attr_noexcept\n{\n if (arena_id != NULL) *arena_id = _mi_arena_id_none();\n if (size < MI_ARENA_BLOCK_SIZE) {\n _mi_warning_message(\"the arena size is too small (memory at %p with size %zu)\\n\", start, size);\n return false;\n }\n if (is_large) {\n mi_assert_internal(memid.initially_committed && memid.is_pinned);\n }\n if (!_mi_is_aligned(start, MI_SEGMENT_ALIGN)) {\n void* const aligned_start = _mi_align_up_ptr(start, MI_SEGMENT_ALIGN);\n const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start;\n if (diff >= size || (size - diff) < MI_ARENA_BLOCK_SIZE) {\n _mi_warning_message(\"after alignment, the size of the arena becomes too small (memory at %p with size %zu)\\n\", start, size);\n return false;\n }\n start = aligned_start;\n size = size - diff;\n }\n\n const size_t bcount = size / MI_ARENA_BLOCK_SIZE;\n const size_t fields = _mi_divide_up(bcount, MI_BITMAP_FIELD_BITS);\n const size_t bitmaps = (memid.is_pinned ? 3 : 5);\n const size_t asize = sizeof(mi_arena_t) + (bitmaps*fields*sizeof(mi_bitmap_field_t));\n mi_memid_t meta_memid;\n mi_arena_t* arena = (mi_arena_t*)_mi_arena_meta_zalloc(asize, &meta_memid);\n if (arena == NULL) return false;\n\n // already zero'd due to zalloc\n // _mi_memzero(arena, asize);\n arena->id = _mi_arena_id_none();\n arena->memid = memid;\n arena->exclusive = exclusive;\n arena->meta_size = asize;\n arena->meta_memid = meta_memid;\n arena->block_count = bcount;\n arena->field_count = fields;\n arena->start = (uint8_t*)start;\n arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1)\n arena->is_large = is_large;\n arena->purge_expire = 0;\n arena->search_idx = 0;\n mi_lock_init(&arena->abandoned_visit_lock);\n // consecutive bitmaps\n arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap\n arena->blocks_abandoned = &arena->blocks_inuse[2 * fields]; // just after dirty bitmap\n arena->blocks_committed = (arena->memid.is_pinned ? NULL : &arena->blocks_inuse[3*fields]); // just after abandoned bitmap\n arena->blocks_purge = (arena->memid.is_pinned ? NULL : &arena->blocks_inuse[4*fields]); // just after committed bitmap\n // initialize committed bitmap?\n if (arena->blocks_committed != NULL && arena->memid.initially_committed) {\n memset((void*)arena->blocks_committed, 0xFF, fields*sizeof(mi_bitmap_field_t)); // cast to void* to avoid atomic warning\n }\n\n // and claim leftover blocks if needed (so we never allocate there)\n ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount;\n mi_assert_internal(post >= 0);\n if (post > 0) {\n // don't use leftover bits at the end\n mi_bitmap_index_t postidx = mi_bitmap_index_create(fields - 1, MI_BITMAP_FIELD_BITS - post);\n _mi_bitmap_claim(arena->blocks_inuse, fields, post, postidx, NULL);\n }\n return mi_arena_add(arena, arena_id, &_mi_stats_main);\n\n}\n\nbool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {\n mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL);\n memid.initially_committed = is_committed;\n memid.initially_zero = is_zero;\n memid.is_pinned = is_large;\n return mi_manage_os_memory_ex2(start,size,is_large,numa_node,exclusive,memid, arena_id);\n}\n\n// Reserve a range of regular OS memory\nint mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {\n if (arena_id != NULL) *arena_id = _mi_arena_id_none();\n size = _mi_align_up(size, MI_ARENA_BLOCK_SIZE); // at least one block\n mi_memid_t memid;\n void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, allow_large, &memid);\n if (start == NULL) return ENOMEM;\n const bool is_large = memid.is_pinned; // todo: use separate is_large field?\n if (!mi_manage_os_memory_ex2(start, size, is_large, -1 /* numa node */, exclusive, memid, arena_id)) {\n _mi_os_free_ex(start, size, commit, memid);\n _mi_verbose_message(\"failed to reserve %zu KiB memory\\n\", _mi_divide_up(size, 1024));\n return ENOMEM;\n }\n _mi_verbose_message(\"reserved %zu KiB memory%s\\n\", _mi_divide_up(size, 1024), is_large ? \" (in large os pages)\" : \"\");\n return 0;\n}\n\n\n// Manage a range of regular OS memory\nbool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept {\n return mi_manage_os_memory_ex(start, size, is_committed, is_large, is_zero, numa_node, false /* exclusive? */, NULL);\n}\n\n// Reserve a range of regular OS memory\nint mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept {\n return mi_reserve_os_memory_ex(size, commit, allow_large, false, NULL);\n}\n\n\n/* -----------------------------------------------------------\n Debugging\n----------------------------------------------------------- */\n\nstatic size_t mi_debug_show_bitmap(const char* prefix, const char* header, size_t block_count, mi_bitmap_field_t* fields, size_t field_count ) {\n _mi_message(\"%s%s:\\n\", prefix, header);\n size_t bcount = 0;\n size_t inuse_count = 0;\n for (size_t i = 0; i < field_count; i++) {\n char buf[MI_BITMAP_FIELD_BITS + 1];\n uintptr_t field = mi_atomic_load_relaxed(&fields[i]);\n for (size_t bit = 0; bit < MI_BITMAP_FIELD_BITS; bit++, bcount++) {\n if (bcount < block_count) {\n bool inuse = ((((uintptr_t)1 << bit) & field) != 0);\n if (inuse) inuse_count++;\n buf[bit] = (inuse ? 'x' : '.');\n }\n else {\n buf[bit] = ' ';\n }\n }\n buf[MI_BITMAP_FIELD_BITS] = 0;\n _mi_message(\"%s %s\\n\", prefix, buf);\n }\n _mi_message(\"%s total ('x'): %zu\\n\", prefix, inuse_count);\n return inuse_count;\n}\n\nvoid mi_debug_show_arenas(void) mi_attr_noexcept {\n const bool show_inuse = true;\n size_t max_arenas = mi_atomic_load_relaxed(&mi_arena_count);\n size_t inuse_total = 0;\n //size_t abandoned_total = 0;\n //size_t purge_total = 0;\n for (size_t i = 0; i < max_arenas; i++) {\n mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]);\n if (arena == NULL) break;\n _mi_message(\"arena %zu: %zu blocks of size %zuMiB (in %zu fields) %s\\n\", i, arena->block_count, (size_t)(MI_ARENA_BLOCK_SIZE / MI_MiB), arena->field_count, (arena->memid.is_pinned ? \", pinned\" : \"\"));\n if (show_inuse) {\n inuse_total += mi_debug_show_bitmap(\" \", \"inuse blocks\", arena->block_count, arena->blocks_inuse, arena->field_count);\n }\n if (arena->blocks_committed != NULL) {\n mi_debug_show_bitmap(\" \", \"committed blocks\", arena->block_count, arena->blocks_committed, arena->field_count);\n }\n //if (show_abandoned) {\n // abandoned_total += mi_debug_show_bitmap(\" \", \"abandoned blocks\", arena->block_count, arena->blocks_abandoned, arena->field_count);\n //}\n //if (show_purge && arena->blocks_purge != NULL) {\n // purge_total += mi_debug_show_bitmap(\" \", \"purgeable blocks\", arena->block_count, arena->blocks_purge, arena->field_count);\n //}\n }\n if (show_inuse) _mi_message(\"total inuse blocks : %zu\\n\", inuse_total);\n //if (show_abandoned) _mi_message(\"total abandoned blocks: %zu\\n\", abandoned_total);\n //if (show_purge) _mi_message(\"total purgeable blocks: %zu\\n\", purge_total);\n}\n\n\nvoid mi_arenas_print(void) mi_attr_noexcept {\n mi_debug_show_arenas();\n}\n\n\n/* -----------------------------------------------------------\n Reserve a huge page arena.\n----------------------------------------------------------- */\n// reserve at a specific numa node\nint mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {\n if (arena_id != NULL) *arena_id = -1;\n if (pages==0) return 0;\n if (numa_node < -1) numa_node = -1;\n if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count();\n size_t hsize = 0;\n size_t pages_reserved = 0;\n mi_memid_t memid;\n void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid);\n if (p==NULL || pages_reserved==0) {\n _mi_warning_message(\"failed to reserve %zu GiB huge pages\\n\", pages);\n return ENOMEM;\n }\n _mi_verbose_message(\"numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\\n\", numa_node, pages_reserved, pages);\n\n if (!mi_manage_os_memory_ex2(p, hsize, true, numa_node, exclusive, memid, arena_id)) {\n _mi_os_free(p, hsize, memid);\n return ENOMEM;\n }\n return 0;\n}\n\nint mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept {\n return mi_reserve_huge_os_pages_at_ex(pages, numa_node, timeout_msecs, false, NULL);\n}\n\n// reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected)\nint mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept {\n if (pages == 0) return 0;\n\n // pages per numa node\n int numa_count = (numa_nodes > 0 && numa_nodes <= INT_MAX ? (int)numa_nodes : _mi_os_numa_node_count());\n if (numa_count == 0) numa_count = 1;\n const size_t pages_per = pages / numa_count;\n const size_t pages_mod = pages % numa_count;\n const size_t timeout_per = (timeout_msecs==0 ? 0 : (timeout_msecs / numa_count) + 50);\n\n // reserve evenly among numa nodes\n for (int numa_node = 0; numa_node < numa_count && pages > 0; numa_node++) {\n size_t node_pages = pages_per; // can be 0\n if ((size_t)numa_node < pages_mod) node_pages++;\n int err = mi_reserve_huge_os_pages_at(node_pages, numa_node, timeout_per);\n if (err) return err;\n if (pages < node_pages) {\n pages = 0;\n }\n else {\n pages -= node_pages;\n }\n }\n\n return 0;\n}\n\nint mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept {\n MI_UNUSED(max_secs);\n _mi_warning_message(\"mi_reserve_huge_os_pages is deprecated: use mi_reserve_huge_os_pages_interleave/at instead\\n\");\n if (pages_reserved != NULL) *pages_reserved = 0;\n int err = mi_reserve_huge_os_pages_interleave(pages, 0, (size_t)(max_secs * 1000.0));\n if (err==0 && pages_reserved!=NULL) *pages_reserved = pages;\n return err;\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/bitmap.c", "sha256": "264bab5a13790372f12cdef044b2b7d3485da779a33eaf3514ccbc7fb08245ba", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2023 Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* ----------------------------------------------------------------------------\nConcurrent bitmap that can set/reset sequences of bits atomically,\nrepresented as an array of fields where each field is a machine word (`size_t`)\n\nThere are two api's; the standard one cannot have sequences that cross\nbetween the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS).\n\nThe `_across` postfixed functions do allow sequences that can cross over\nbetween the fields. (This is used in arena allocation)\n---------------------------------------------------------------------------- */\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"bitmap.h\"\n\n/* -----------------------------------------------------------\n Bitmap definition\n----------------------------------------------------------- */\n\n// The bit mask for a given number of blocks at a specified bit index.\nstatic inline size_t mi_bitmap_mask_(size_t count, size_t bitidx) {\n mi_assert_internal(count + bitidx <= MI_BITMAP_FIELD_BITS);\n mi_assert_internal(count > 0);\n if (count >= MI_BITMAP_FIELD_BITS) return MI_BITMAP_FIELD_FULL;\n if (count == 0) return 0;\n return ((((size_t)1 << count) - 1) << bitidx);\n}\n\n\n/* -----------------------------------------------------------\n Claim a bit sequence atomically\n----------------------------------------------------------- */\n\n// Try to atomically claim a sequence of `count` bits in a single\n// field at `idx` in `bitmap`. Returns `true` on success.\ninline bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx)\n{\n mi_assert_internal(bitmap_idx != NULL);\n mi_assert_internal(count <= MI_BITMAP_FIELD_BITS);\n mi_assert_internal(count > 0);\n mi_bitmap_field_t* field = &bitmap[idx];\n size_t map = mi_atomic_load_relaxed(field);\n if (map==MI_BITMAP_FIELD_FULL) return false; // short cut\n\n // search for 0-bit sequence of length count\n const size_t mask = mi_bitmap_mask_(count, 0);\n const size_t bitidx_max = MI_BITMAP_FIELD_BITS - count;\n\n#ifdef MI_HAVE_FAST_BITSCAN\n size_t bitidx = mi_ctz(~map); // quickly find the first zero bit if possible\n#else\n size_t bitidx = 0; // otherwise start at 0\n#endif\n size_t m = (mask << bitidx); // invariant: m == mask shifted by bitidx\n\n // scan linearly for a free range of zero bits\n while (bitidx <= bitidx_max) {\n const size_t mapm = (map & m);\n if (mapm == 0) { // are the mask bits free at bitidx?\n mi_assert_internal((m >> bitidx) == mask); // no overflow?\n const size_t newmap = (map | m);\n mi_assert_internal((newmap^map) >> bitidx == mask);\n if (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)) { // TODO: use weak cas here?\n // no success, another thread claimed concurrently.. keep going (with updated `map`)\n continue;\n }\n else {\n // success, we claimed the bits!\n *bitmap_idx = mi_bitmap_index_create(idx, bitidx);\n return true;\n }\n }\n else {\n // on to the next bit range\n#ifdef MI_HAVE_FAST_BITSCAN\n mi_assert_internal(mapm != 0);\n const size_t shift = (count == 1 ? 1 : (MI_SIZE_BITS - mi_clz(mapm) - bitidx));\n mi_assert_internal(shift > 0 && shift <= count);\n#else\n const size_t shift = 1;\n#endif\n bitidx += shift;\n m <<= shift;\n }\n }\n // no bits found\n return false;\n}\n\n// Find `count` bits of 0 and set them to 1 atomically; returns `true` on success.\n// Starts at idx, and wraps around to search in all `bitmap_fields` fields.\n// `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields.\nbool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) {\n size_t idx = start_field_idx;\n for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) {\n if (idx >= bitmap_fields) { idx = 0; } // wrap\n if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) {\n return true;\n }\n }\n return false;\n}\n\n// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled\nbool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, \n const size_t start_field_idx, const size_t count, \n mi_bitmap_pred_fun_t pred_fun, void* pred_arg, \n mi_bitmap_index_t* bitmap_idx) {\n size_t idx = start_field_idx;\n for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) {\n if (idx >= bitmap_fields) idx = 0; // wrap\n if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) {\n if (pred_fun == NULL || pred_fun(*bitmap_idx, pred_arg)) { \n return true;\n }\n // predicate returned false, unclaim and look further\n _mi_bitmap_unclaim(bitmap, bitmap_fields, count, *bitmap_idx);\n }\n }\n return false;\n}\n\n// Set `count` bits at `bitmap_idx` to 0 atomically\n// Returns `true` if all `count` bits were 1 previously.\nbool _mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {\n const size_t idx = mi_bitmap_index_field(bitmap_idx);\n const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);\n const size_t mask = mi_bitmap_mask_(count, bitidx);\n mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields);\n // mi_assert_internal((bitmap[idx] & mask) == mask);\n const size_t prev = mi_atomic_and_acq_rel(&bitmap[idx], ~mask);\n return ((prev & mask) == mask);\n}\n\n\n// Set `count` bits at `bitmap_idx` to 1 atomically\n// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit.\nbool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero) {\n const size_t idx = mi_bitmap_index_field(bitmap_idx);\n const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);\n const size_t mask = mi_bitmap_mask_(count, bitidx);\n mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields);\n //mi_assert_internal(any_zero != NULL || (bitmap[idx] & mask) == 0);\n size_t prev = mi_atomic_or_acq_rel(&bitmap[idx], mask);\n if (any_zero != NULL) { *any_zero = ((prev & mask) != mask); }\n return ((prev & mask) == 0);\n}\n\n// Returns `true` if all `count` bits were 1. `any_ones` is `true` if there was at least one bit set to one.\nstatic bool mi_bitmap_is_claimedx(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_ones) {\n const size_t idx = mi_bitmap_index_field(bitmap_idx);\n const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);\n const size_t mask = mi_bitmap_mask_(count, bitidx);\n mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields);\n const size_t field = mi_atomic_load_relaxed(&bitmap[idx]);\n if (any_ones != NULL) { *any_ones = ((field & mask) != 0); }\n return ((field & mask) == mask);\n}\n\n// Try to set `count` bits at `bitmap_idx` from 0 to 1 atomically.\n// Returns `true` if successful when all previous `count` bits were 0.\nbool _mi_bitmap_try_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {\n const size_t idx = mi_bitmap_index_field(bitmap_idx);\n const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);\n const size_t mask = mi_bitmap_mask_(count, bitidx);\n mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields);\n size_t expected = mi_atomic_load_relaxed(&bitmap[idx]);\n do {\n if ((expected & mask) != 0) return false;\n }\n while (!mi_atomic_cas_strong_acq_rel(&bitmap[idx], &expected, expected | mask));\n mi_assert_internal((expected & mask) == 0);\n return true;\n}\n\n\nbool _mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {\n return mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, NULL);\n}\n\nbool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {\n bool any_ones;\n mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, &any_ones);\n return any_ones;\n}\n\n\n//--------------------------------------------------------------------------\n// the `_across` functions work on bitmaps where sequences can cross over\n// between the fields. This is used in arena allocation\n//--------------------------------------------------------------------------\n\n// Try to atomically claim a sequence of `count` bits starting from the field\n// at `idx` in `bitmap` and crossing into subsequent fields. Returns `true` on success.\n// Only needs to consider crossing into the next fields (see `mi_bitmap_try_find_from_claim_across`)\nstatic bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t idx, const size_t count, const size_t retries, mi_bitmap_index_t* bitmap_idx)\n{\n mi_assert_internal(bitmap_idx != NULL);\n\n // check initial trailing zeros\n mi_bitmap_field_t* field = &bitmap[idx];\n size_t map = mi_atomic_load_relaxed(field);\n const size_t initial = mi_clz(map); // count of initial zeros starting at idx\n mi_assert_internal(initial <= MI_BITMAP_FIELD_BITS);\n if (initial == 0) return false;\n if (initial >= count) return _mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx); // no need to cross fields (this case won't happen for us)\n if (_mi_divide_up(count - initial, MI_BITMAP_FIELD_BITS) >= (bitmap_fields - idx)) return false; // not enough entries\n\n // scan ahead\n size_t found = initial;\n size_t mask = 0; // mask bits for the final field\n while(found < count) {\n field++;\n map = mi_atomic_load_relaxed(field);\n const size_t mask_bits = (found + MI_BITMAP_FIELD_BITS <= count ? MI_BITMAP_FIELD_BITS : (count - found));\n mi_assert_internal(mask_bits > 0 && mask_bits <= MI_BITMAP_FIELD_BITS);\n mask = mi_bitmap_mask_(mask_bits, 0);\n if ((map & mask) != 0) return false; // some part is already claimed\n found += mask_bits;\n }\n mi_assert_internal(field < &bitmap[bitmap_fields]);\n\n // we found a range of contiguous zeros up to the final field; mask contains mask in the final field\n // now try to claim the range atomically\n mi_bitmap_field_t* const final_field = field;\n const size_t final_mask = mask;\n mi_bitmap_field_t* const initial_field = &bitmap[idx];\n const size_t initial_idx = MI_BITMAP_FIELD_BITS - initial;\n const size_t initial_mask = mi_bitmap_mask_(initial, initial_idx);\n\n // initial field\n size_t newmap;\n field = initial_field;\n map = mi_atomic_load_relaxed(field);\n do {\n newmap = (map | initial_mask);\n if ((map & initial_mask) != 0) { goto rollback; };\n } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap));\n\n // intermediate fields\n while (++field < final_field) {\n newmap = MI_BITMAP_FIELD_FULL;\n map = 0;\n if (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)) { goto rollback; }\n }\n\n // final field\n mi_assert_internal(field == final_field);\n map = mi_atomic_load_relaxed(field);\n do {\n newmap = (map | final_mask);\n if ((map & final_mask) != 0) { goto rollback; }\n } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap));\n\n // claimed!\n *bitmap_idx = mi_bitmap_index_create(idx, initial_idx);\n return true;\n\nrollback:\n // roll back intermediate fields\n // (we just failed to claim `field` so decrement first)\n while (--field > initial_field) {\n newmap = 0;\n map = MI_BITMAP_FIELD_FULL;\n mi_assert_internal(mi_atomic_load_relaxed(field) == map);\n mi_atomic_store_release(field, newmap);\n }\n if (field == initial_field) { // (if we failed on the initial field, `field + 1 == initial_field`)\n map = mi_atomic_load_relaxed(field);\n do {\n mi_assert_internal((map & initial_mask) == initial_mask);\n newmap = (map & ~initial_mask);\n } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap));\n }\n mi_stat_counter_increase(_mi_stats_main.arena_rollback_count,1);\n // retry? (we make a recursive call instead of goto to be able to use const declarations)\n if (retries <= 2) {\n return mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, retries+1, bitmap_idx);\n }\n else {\n return false;\n }\n}\n\n\n// Find `count` bits of zeros and set them to 1 atomically; returns `true` on success.\n// Starts at idx, and wraps around to search in all `bitmap_fields` fields.\nbool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) {\n mi_assert_internal(count > 0);\n if (count <= 2) {\n // we don't bother with crossover fields for small counts\n return _mi_bitmap_try_find_from_claim(bitmap, bitmap_fields, start_field_idx, count, bitmap_idx);\n }\n\n // visit the fields\n size_t idx = start_field_idx;\n for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) {\n if (idx >= bitmap_fields) { idx = 0; } // wrap\n // first try to claim inside a field\n if (count <= MI_BITMAP_FIELD_BITS) {\n if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) {\n return true;\n }\n }\n // if that fails, then try to claim across fields\n if (mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, 0, bitmap_idx)) {\n return true;\n }\n }\n return false;\n}\n\n// Helper for masks across fields; returns the mid count, post_mask may be 0\nstatic size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, size_t* pre_mask, size_t* mid_mask, size_t* post_mask) {\n MI_UNUSED(bitmap_fields);\n const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);\n if mi_likely(bitidx + count <= MI_BITMAP_FIELD_BITS) {\n *pre_mask = mi_bitmap_mask_(count, bitidx);\n *mid_mask = 0;\n *post_mask = 0;\n mi_assert_internal(mi_bitmap_index_field(bitmap_idx) < bitmap_fields);\n return 0;\n }\n else {\n const size_t pre_bits = MI_BITMAP_FIELD_BITS - bitidx;\n mi_assert_internal(pre_bits < count);\n *pre_mask = mi_bitmap_mask_(pre_bits, bitidx);\n count -= pre_bits;\n const size_t mid_count = (count / MI_BITMAP_FIELD_BITS);\n *mid_mask = MI_BITMAP_FIELD_FULL;\n count %= MI_BITMAP_FIELD_BITS;\n *post_mask = (count==0 ? 0 : mi_bitmap_mask_(count, 0));\n mi_assert_internal(mi_bitmap_index_field(bitmap_idx) + mid_count + (count==0 ? 0 : 1) < bitmap_fields);\n return mid_count;\n }\n}\n\n// Set `count` bits at `bitmap_idx` to 0 atomically\n// Returns `true` if all `count` bits were 1 previously.\nbool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {\n size_t idx = mi_bitmap_index_field(bitmap_idx);\n size_t pre_mask;\n size_t mid_mask;\n size_t post_mask;\n size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask);\n bool all_one = true;\n mi_bitmap_field_t* field = &bitmap[idx];\n size_t prev = mi_atomic_and_acq_rel(field++, ~pre_mask); // clear first part\n if ((prev & pre_mask) != pre_mask) all_one = false;\n while(mid_count-- > 0) {\n prev = mi_atomic_and_acq_rel(field++, ~mid_mask); // clear mid part\n if ((prev & mid_mask) != mid_mask) all_one = false;\n }\n if (post_mask!=0) {\n prev = mi_atomic_and_acq_rel(field, ~post_mask); // clear end part\n if ((prev & post_mask) != post_mask) all_one = false;\n }\n return all_one;\n}\n\n// Set `count` bits at `bitmap_idx` to 1 atomically\n// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit.\nbool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero, size_t* already_set) {\n size_t idx = mi_bitmap_index_field(bitmap_idx);\n size_t pre_mask;\n size_t mid_mask;\n size_t post_mask;\n size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask);\n bool all_zero = true;\n bool any_zero = false;\n size_t one_count = 0;\n _Atomic(size_t)*field = &bitmap[idx];\n size_t prev = mi_atomic_or_acq_rel(field++, pre_mask);\n if ((prev & pre_mask) != 0) { all_zero = false; one_count += mi_popcount(prev & pre_mask); }\n if ((prev & pre_mask) != pre_mask) any_zero = true;\n while (mid_count-- > 0) {\n prev = mi_atomic_or_acq_rel(field++, mid_mask);\n if ((prev & mid_mask) != 0) { all_zero = false; one_count += mi_popcount(prev & mid_mask); }\n if ((prev & mid_mask) != mid_mask) any_zero = true;\n }\n if (post_mask!=0) {\n prev = mi_atomic_or_acq_rel(field, post_mask);\n if ((prev & post_mask) != 0) { all_zero = false; one_count += mi_popcount(prev & post_mask); }\n if ((prev & post_mask) != post_mask) any_zero = true;\n }\n if (pany_zero != NULL) { *pany_zero = any_zero; }\n if (already_set != NULL) { *already_set = one_count; };\n mi_assert_internal(all_zero ? one_count == 0 : one_count <= count);\n return all_zero;\n}\n\n\n// Returns `true` if all `count` bits were 1.\n// `any_ones` is `true` if there was at least one bit set to one.\nstatic bool mi_bitmap_is_claimedx_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_ones, size_t* already_set) {\n size_t idx = mi_bitmap_index_field(bitmap_idx);\n size_t pre_mask;\n size_t mid_mask;\n size_t post_mask;\n size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask);\n bool all_ones = true;\n bool any_ones = false;\n size_t one_count = 0;\n mi_bitmap_field_t* field = &bitmap[idx];\n size_t prev = mi_atomic_load_relaxed(field++);\n if ((prev & pre_mask) != pre_mask) all_ones = false;\n if ((prev & pre_mask) != 0) { any_ones = true; one_count += mi_popcount(prev & pre_mask); }\n while (mid_count-- > 0) {\n prev = mi_atomic_load_relaxed(field++);\n if ((prev & mid_mask) != mid_mask) all_ones = false;\n if ((prev & mid_mask) != 0) { any_ones = true; one_count += mi_popcount(prev & mid_mask); }\n }\n if (post_mask!=0) {\n prev = mi_atomic_load_relaxed(field);\n if ((prev & post_mask) != post_mask) all_ones = false;\n if ((prev & post_mask) != 0) { any_ones = true; one_count += mi_popcount(prev & post_mask); }\n }\n if (pany_ones != NULL) { *pany_ones = any_ones; }\n if (already_set != NULL) { *already_set = one_count; }\n mi_assert_internal(all_ones ? one_count == count : one_count < count);\n return all_ones;\n}\n\nbool _mi_bitmap_is_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, size_t* already_set) {\n return mi_bitmap_is_claimedx_across(bitmap, bitmap_fields, count, bitmap_idx, NULL, already_set);\n}\n\nbool _mi_bitmap_is_any_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {\n bool any_ones;\n mi_bitmap_is_claimedx_across(bitmap, bitmap_fields, count, bitmap_idx, &any_ones, NULL);\n return any_ones;\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/bitmap.h", "sha256": "a1c4d3d6eadfd36e0cd06abddccf54387abd6f43d8b2638cb36bd0f162d560c4", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2023 Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* ----------------------------------------------------------------------------\nConcurrent bitmap that can set/reset sequences of bits atomically,\nrepresented as an array of fields where each field is a machine word (`size_t`)\n\nThere are two api's; the standard one cannot have sequences that cross\nbetween the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS).\n(this is used in region allocation)\n\nThe `_across` postfixed functions do allow sequences that can cross over\nbetween the fields. (This is used in arena allocation)\n---------------------------------------------------------------------------- */\n#pragma once\n#ifndef MI_BITMAP_H\n#define MI_BITMAP_H\n\n/* -----------------------------------------------------------\n Bitmap definition\n----------------------------------------------------------- */\n\n#define MI_BITMAP_FIELD_BITS (8*MI_SIZE_SIZE)\n#define MI_BITMAP_FIELD_FULL (~((size_t)0)) // all bits set\n\n// An atomic bitmap of `size_t` fields\ntypedef _Atomic(size_t) mi_bitmap_field_t;\ntypedef mi_bitmap_field_t* mi_bitmap_t;\n\n// A bitmap index is the index of the bit in a bitmap.\ntypedef size_t mi_bitmap_index_t;\n\n// Create a bit index.\nstatic inline mi_bitmap_index_t mi_bitmap_index_create_ex(size_t idx, size_t bitidx) {\n mi_assert_internal(bitidx <= MI_BITMAP_FIELD_BITS);\n return (idx*MI_BITMAP_FIELD_BITS) + bitidx;\n}\nstatic inline mi_bitmap_index_t mi_bitmap_index_create(size_t idx, size_t bitidx) {\n mi_assert_internal(bitidx < MI_BITMAP_FIELD_BITS);\n return mi_bitmap_index_create_ex(idx,bitidx);\n}\n\n// Create a bit index.\nstatic inline mi_bitmap_index_t mi_bitmap_index_create_from_bit(size_t full_bitidx) { \n return mi_bitmap_index_create(full_bitidx / MI_BITMAP_FIELD_BITS, full_bitidx % MI_BITMAP_FIELD_BITS);\n}\n\n// Get the field index from a bit index.\nstatic inline size_t mi_bitmap_index_field(mi_bitmap_index_t bitmap_idx) {\n return (bitmap_idx / MI_BITMAP_FIELD_BITS);\n}\n\n// Get the bit index in a bitmap field\nstatic inline size_t mi_bitmap_index_bit_in_field(mi_bitmap_index_t bitmap_idx) {\n return (bitmap_idx % MI_BITMAP_FIELD_BITS);\n}\n\n// Get the full bit index\nstatic inline size_t mi_bitmap_index_bit(mi_bitmap_index_t bitmap_idx) {\n return bitmap_idx;\n}\n\n/* -----------------------------------------------------------\n Claim a bit sequence atomically\n----------------------------------------------------------- */\n\n// Try to atomically claim a sequence of `count` bits in a single\n// field at `idx` in `bitmap`. Returns `true` on success.\nbool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx);\n\n// Starts at idx, and wraps around to search in all `bitmap_fields` fields.\n// For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields.\nbool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx);\n\n// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled\ntypedef bool (mi_cdecl *mi_bitmap_pred_fun_t)(mi_bitmap_index_t bitmap_idx, void* pred_arg);\nbool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_pred_fun_t pred_fun, void* pred_arg, mi_bitmap_index_t* bitmap_idx);\n\n// Set `count` bits at `bitmap_idx` to 0 atomically\n// Returns `true` if all `count` bits were 1 previously.\nbool _mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);\n\n// Try to set `count` bits at `bitmap_idx` from 0 to 1 atomically. \n// Returns `true` if successful when all previous `count` bits were 0.\nbool _mi_bitmap_try_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);\n\n// Set `count` bits at `bitmap_idx` to 1 atomically\n// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit.\nbool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero);\n\nbool _mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);\nbool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);\n\n\n//--------------------------------------------------------------------------\n// the `_across` functions work on bitmaps where sequences can cross over\n// between the fields. This is used in arena allocation\n//--------------------------------------------------------------------------\n\n// Find `count` bits of zeros and set them to 1 atomically; returns `true` on success.\n// Starts at idx, and wraps around to search in all `bitmap_fields` fields.\nbool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx);\n\n// Set `count` bits at `bitmap_idx` to 0 atomically\n// Returns `true` if all `count` bits were 1 previously.\nbool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);\n\n// Set `count` bits at `bitmap_idx` to 1 atomically\n// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit.\nbool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero, size_t* already_set);\n\nbool _mi_bitmap_is_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, size_t* already_set);\nbool _mi_bitmap_is_any_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);\n\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/free.c", "sha256": "da31e9f443f90455689f0a93db61bb2bb72fda240bdcf4c5ffd243dda52cfdf3", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#if !defined(MI_IN_ALLOC_C)\n#error \"this file should be included from 'alloc.c' (so aliases can work from alloc-override)\"\n// add includes help an IDE\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\" // _mi_prim_thread_id()\n#endif\n\n// forward declarations\nstatic void mi_check_padding(const mi_page_t* page, const mi_block_t* block);\nstatic bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block);\nstatic size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded);\nstatic void mi_stat_free(const mi_page_t* page, const mi_block_t* block);\n\n\n// ------------------------------------------------------\n// Free\n// ------------------------------------------------------\n\n// forward declaration of multi-threaded free (`_mt`) (or free in huge block if compiled with MI_HUGE_PAGE_ABANDON)\nstatic mi_decl_noinline void mi_free_block_mt(mi_page_t* page, mi_segment_t* segment, mi_block_t* block, void* p, bool was_guarded);\n\n// regular free of a (thread local) block pointer\n// fast path written carefully to prevent spilling on the stack\nstatic inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full)\n{\n MI_UNUSED(was_guarded);\n // checks \n if mi_unlikely(mi_check_is_double_free(page, block)) return;\n if (!was_guarded) { mi_check_padding(page, block); }\n if (track_stats) { mi_stat_free(page, block); }\n #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN\n if (!mi_page_is_huge(page)) { // huge page content may be already decommitted\n memset(block, MI_DEBUG_FREED, mi_page_block_size(page));\n }\n #endif\n if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned\n\n // actual free: push on the local free list\n mi_block_set_next(page, block, page->local_free);\n page->local_free = block;\n if mi_unlikely(--page->used == 0) {\n _mi_page_retire(page);\n }\n else if mi_unlikely(check_full && mi_page_is_in_full(page)) {\n _mi_page_unfull(page);\n }\n}\n\n// Adjust a block that was allocated aligned, to the actual start of the block in the page.\n// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the\n// `page_start` and `block_size` fields; however these are constant and the page won't be\n// deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently.\nmi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {\n mi_assert_internal(page!=NULL && p!=NULL);\n\n size_t diff = (uint8_t*)p - page->page_start;\n size_t adjust = 0;\n if mi_likely(page->block_size_shift != 0) {\n adjust = diff & (((size_t)1 << page->block_size_shift) - 1);\n }\n else {\n const size_t block_size = page->block_size;\n if mi_likely(block_size != 0) {\n adjust = diff % block_size;\n }\n else {\n _mi_error_message(EFAULT, \"reading from invalid page, possibly corrupted meta-data (address=%p, page=%p)\\n\", p, page); \n }\n }\n\n return (mi_block_t*)((uintptr_t)p - adjust);\n}\n\n// forward declaration for a MI_GUARDED build\n#if MI_GUARDED\nstatic void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration\nstatic inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {\n if (mi_block_ptr_is_guarded(block, p)) { \n mi_block_unguard(page, block, p); \n return true;\n }\n else {\n return false;\n }\n}\n#else\nstatic inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {\n MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);\n return false;\n}\n#endif\n\n// free a local pointer (page parameter comes first for better codegen)\nstatic void mi_decl_noinline mi_free_generic_local(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept {\n MI_UNUSED(segment);\n mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(page, p) : (mi_block_t*)p);\n const bool was_guarded = mi_block_check_unguard(page, block, p);\n mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */);\n}\n\n// free a pointer owned by another thread (page parameter comes first for better codegen)\nstatic void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept {\n mi_block_t* const block = _mi_page_ptr_unalign(page, p); // don't check `has_aligned` flag to avoid a race (issue #865)\n const bool was_guarded = mi_block_check_unguard(page, block, p);\n mi_free_block_mt(page, segment, block, p, was_guarded);\n}\n\n// generic free (for runtime integration)\nvoid mi_decl_noinline _mi_free_generic(mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept {\n if (is_local) mi_free_generic_local(page,segment,p);\n else mi_free_generic_mt(page,segment,p);\n}\n\n// Get the segment data belonging to a pointer\n// This is just a single `and` in release mode but does further checks in debug mode\n// (and secure mode) to see if this was a valid pointer.\nstatic inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* msg)\n{\n MI_UNUSED(msg);\n\n #if (MI_DEBUG>0)\n if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) {\n _mi_error_message(EINVAL, \"%s: invalid (unaligned) pointer: %p\\n\", msg, p);\n return NULL;\n }\n #endif\n\n mi_segment_t* const segment = _mi_ptr_segment(p);\n if mi_unlikely(segment==NULL) return segment;\n\n #if (MI_DEBUG>0)\n if mi_unlikely(!mi_is_in_heap_region(p)) {\n #if (MI_INTPTR_SIZE == 8 && defined(__linux__))\n if (((uintptr_t)p >> 40) != 0x7F) { // linux tends to align large blocks above 0x7F000000000 (issue #640)\n #else\n {\n #endif\n _mi_warning_message(\"%s: pointer might not point to a valid heap region: %p\\n\"\n \"(this may still be a valid very large allocation (over 64MiB))\\n\", msg, p);\n if mi_likely(_mi_ptr_cookie(segment) == segment->cookie) {\n _mi_warning_message(\"(yes, the previous pointer %p was valid after all)\\n\", p);\n }\n }\n }\n #endif\n #if (MI_DEBUG>0 || MI_SECURE>=3)\n if mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie) {\n _mi_error_message(EINVAL, \"%s: pointer does not point to a valid heap space: %p\\n\", msg, p);\n return NULL;\n }\n #endif\n\n return segment;\n}\n\n// Free a block\n// Fast path written carefully to prevent register spilling on the stack\nstatic inline void mi_free_ex(void* p, size_t* usable) mi_attr_noexcept\n{\n mi_segment_t* const segment = mi_checked_ptr_segment(p,\"mi_free\");\n if mi_unlikely(segment==NULL) return;\n\n const bool is_local = (_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id));\n mi_page_t* const page = _mi_segment_page_of(segment, p);\n if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }\n \n if mi_likely(is_local) { // thread-local free?\n if mi_likely(page->flags.full_aligned == 0) { // and it is not a full page (full pages need to move from the full bin), nor has aligned blocks (aligned blocks need to be unaligned)\n // thread-local, aligned, and not a full page\n mi_block_t* const block = (mi_block_t*)p;\n mi_free_block_local(page, block, false /* is guarded */, true /* track stats */, false /* no need to check if the page is full */);\n }\n else {\n // page is full or contains (inner) aligned blocks; use generic path\n mi_free_generic_local(page, segment, p);\n }\n }\n else {\n // not thread-local; use generic path\n mi_free_generic_mt(page, segment, p);\n }\n}\n\nvoid mi_free(void* p) mi_attr_noexcept {\n mi_free_ex(p,NULL);\n}\n\nvoid mi_ufree(void* p, size_t* usable) mi_attr_noexcept {\n mi_free_ex(p,usable);\n}\n\nvoid mi_free_small(void* p) mi_attr_noexcept {\n mi_free(p);\n}\n\n\n// return true if successful\nbool _mi_free_delayed_block(mi_block_t* block) {\n // get segment and page\n mi_assert_internal(block!=NULL);\n const mi_segment_t* const segment = _mi_ptr_segment(block);\n mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie);\n mi_assert_internal(_mi_thread_id() == segment->thread_id);\n mi_page_t* const page = _mi_segment_page_of(segment, block);\n\n // Clear the no-delayed flag so delayed freeing is used again for this page.\n // This must be done before collecting the free lists on this page -- otherwise\n // some blocks may end up in the page `thread_free` list with no blocks in the\n // heap `thread_delayed_free` list which may cause the page to be never freed!\n // (it would only be freed if we happen to scan it in `mi_page_queue_find_free_ex`)\n if (!_mi_page_try_use_delayed_free(page, MI_USE_DELAYED_FREE, false /* dont overwrite never delayed */)) {\n return false;\n }\n\n // collect all other non-local frees (move from `thread_free` to `free`) to ensure up-to-date `used` count\n _mi_page_free_collect(page, false);\n\n // and free the block (possibly freeing the page as well since `used` is updated)\n mi_free_block_local(page, block, false /* is guarded */, false /* stats have already been adjusted */, true /* check for a full page */);\n return true;\n}\n\n// ------------------------------------------------------\n// Multi-threaded Free (`_mt`)\n// ------------------------------------------------------\n\n// Push a block that is owned by another thread on its page-local thread free\n// list or it's heap delayed free list. Such blocks are later collected by\n// the owning thread in `_mi_free_delayed_block`.\nstatic void mi_decl_noinline mi_free_block_delayed_mt( mi_page_t* page, mi_block_t* block )\n{\n // Try to put the block on either the page-local thread free list,\n // or the heap delayed free list (if this is the first non-local free in that page)\n mi_thread_free_t tfreex;\n bool use_delayed;\n mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free);\n do {\n use_delayed = (mi_tf_delayed(tfree) == MI_USE_DELAYED_FREE);\n if mi_unlikely(use_delayed) {\n // unlikely: this only happens on the first concurrent free in a page that is in the full list\n tfreex = mi_tf_set_delayed(tfree,MI_DELAYED_FREEING);\n }\n else {\n // usual: directly add to page thread_free list\n mi_block_set_next(page, block, mi_tf_block(tfree));\n tfreex = mi_tf_set_block(tfree,block);\n }\n } while (!mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex));\n\n // If this was the first non-local free, we need to push it on the heap delayed free list instead\n if mi_unlikely(use_delayed) {\n // racy read on `heap`, but ok because MI_DELAYED_FREEING is set (see `mi_heap_delete` and `mi_heap_collect_abandon`)\n mi_heap_t* const heap = (mi_heap_t*)(mi_atomic_load_acquire(&page->xheap)); //mi_page_heap(page);\n mi_assert_internal(heap != NULL);\n if (heap != NULL) {\n // add to the delayed free list of this heap. (do this atomically as the lock only protects heap memory validity)\n mi_block_t* dfree = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free);\n do {\n mi_block_set_nextx(heap,block,dfree, heap->keys);\n } while (!mi_atomic_cas_ptr_weak_release(mi_block_t,&heap->thread_delayed_free, &dfree, block));\n }\n\n // and reset the MI_DELAYED_FREEING flag\n tfree = mi_atomic_load_relaxed(&page->xthread_free);\n do {\n tfreex = tfree;\n mi_assert_internal(mi_tf_delayed(tfree) == MI_DELAYED_FREEING);\n tfreex = mi_tf_set_delayed(tfree,MI_NO_DELAYED_FREE);\n } while (!mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex));\n }\n}\n\n// Multi-threaded free (`_mt`) (or free in huge block if compiled with MI_HUGE_PAGE_ABANDON)\nstatic void mi_decl_noinline mi_free_block_mt(mi_page_t* page, mi_segment_t* segment, mi_block_t* block, void* p, bool was_guarded)\n{\n MI_UNUSED(was_guarded);\n\n // first see if the segment was abandoned and if we can reclaim it into our thread\n if (_mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0 &&\n #if MI_HUGE_PAGE_ABANDON\n segment->page_kind != MI_PAGE_HUGE &&\n #endif\n mi_atomic_load_relaxed(&segment->thread_id) == 0 && // segment is abandoned?\n mi_prim_get_default_heap() != (mi_heap_t*)&_mi_heap_empty) // and we did not already exit this thread (without this check, a fresh heap will be initalized (issue #944))\n {\n // the segment is abandoned, try to reclaim it into our heap\n if (_mi_segment_attempt_reclaim(mi_heap_get_default(), segment)) {\n mi_assert_internal(_mi_thread_id() == mi_atomic_load_relaxed(&segment->thread_id));\n mi_assert_internal(mi_heap_get_default()->tld->segments.subproc == segment->subproc);\n mi_free(p); // recursively free as now it will be a local free in our heap\n return;\n }\n }\n\n // The padding check may access the non-thread-owned page for the key values.\n // that is safe as these are constant and the page won't be freed (as the block is not freed yet).\n if (!was_guarded) { mi_check_padding(page, block); }\n\n // adjust stats (after padding check and potentially recursive `mi_free` above)\n mi_stat_free(page, block); // stat_free may access the padding\n mi_track_free_size(block, mi_page_usable_size_of(page,block,was_guarded));\n\n // for small size, ensure we can fit the delayed thread pointers without triggering overflow detection\n _mi_padding_shrink(page, block, sizeof(mi_block_t));\n\n if (segment->kind == MI_SEGMENT_HUGE) {\n #if MI_HUGE_PAGE_ABANDON\n // huge page segments are always abandoned and can be freed immediately\n _mi_segment_huge_page_free(segment, page, block);\n return;\n #else\n // huge pages are special as they occupy the entire segment\n // as these are large we reset the memory occupied by the page so it is available to other threads\n // (as the owning thread needs to actually free the memory later).\n _mi_segment_huge_page_reset(segment, page, block);\n #endif\n }\n else {\n #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading\n memset(block, MI_DEBUG_FREED, mi_usable_size(block));\n #endif\n }\n\n // and finally free the actual block by pushing it on the owning heap\n // thread_delayed free list (or heap delayed free list)\n mi_free_block_delayed_mt(page,block);\n}\n\n\n// ------------------------------------------------------\n// Usable size\n// ------------------------------------------------------\n\n// Bytes available in a block\nstatic size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept {\n const mi_block_t* block = _mi_page_ptr_unalign(page, p);\n const bool is_guarded = mi_block_ptr_is_guarded(block,p);\n const size_t size = mi_page_usable_size_of(page, block, is_guarded);\n const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;\n mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);\n const size_t aligned_size = (size - adjust); \n return aligned_size;\n}\n\nstatic inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) {\n const mi_segment_t* const segment = mi_checked_ptr_segment(p, msg);\n if mi_unlikely(segment==NULL) return NULL;\n mi_page_t* const page = _mi_segment_page_of(segment, p);\n return page;\n}\n\nstatic inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_attr_noexcept {\n if mi_unlikely(page==NULL) return 0;\n if mi_likely(!mi_page_has_aligned(page)) {\n const mi_block_t* block = (const mi_block_t*)p;\n return mi_page_usable_size_of(page, block, false /* is guarded */);\n }\n else {\n // split out to separate routine for improved code generation\n return mi_page_usable_aligned_size_of(page, p);\n }\n}\n\nmi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept {\n const mi_page_t* const page = mi_validate_ptr_page(p,\"mi_usable_size\");\n return _mi_usable_size(p,page);\n}\n\n\n// ------------------------------------------------------\n// Free variants\n// ------------------------------------------------------\n\nvoid mi_free_size(void* p, size_t size) mi_attr_noexcept {\n MI_UNUSED_RELEASE(size);\n #if MI_DEBUG\n const mi_page_t* const page = mi_validate_ptr_page(p,\"mi_free_size\"); \n const size_t available = _mi_usable_size(p,page);\n mi_assert(p == NULL || size <= available || available == 0 /* invalid pointer */ );\n #endif\n mi_free(p);\n}\n\nvoid mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {\n MI_UNUSED_RELEASE(alignment);\n mi_assert(((uintptr_t)p % alignment) == 0);\n mi_free_size(p,size);\n}\n\nvoid mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {\n MI_UNUSED_RELEASE(alignment);\n mi_assert(((uintptr_t)p % alignment) == 0);\n mi_free(p);\n}\n\n\n// ------------------------------------------------------\n// Check for double free in secure and debug mode\n// This is somewhat expensive so only enabled for secure mode 4\n// ------------------------------------------------------\n\n#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0))\n// linear check if the free list contains a specific element\nstatic bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) {\n while (list != NULL) {\n if (elem==list) return true;\n list = mi_block_next(page, list);\n }\n return false;\n}\n\nstatic mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) {\n // The decoded value is in the same page (or NULL).\n // Walk the free lists to verify positively if it is already freed\n if (mi_list_contains(page, page->free, block) ||\n mi_list_contains(page, page->local_free, block) ||\n mi_list_contains(page, mi_page_thread_free(page), block))\n {\n _mi_error_message(EAGAIN, \"double free detected of block %p with size %zu\\n\", block, mi_page_block_size(page));\n return true;\n }\n return false;\n}\n\n#define mi_track_page(page,access) { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); }\n\nstatic inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {\n bool is_double_free = false;\n mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field\n if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer?\n (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL?\n {\n // Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free?\n // (continue in separate function to improve code generation)\n is_double_free = mi_check_is_double_freex(page, block);\n }\n return is_double_free;\n}\n#else\nstatic inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {\n MI_UNUSED(page);\n MI_UNUSED(block);\n return false;\n}\n#endif\n\n\n// ---------------------------------------------------------------------------\n// Check for heap block overflow by setting up padding at the end of the block\n// ---------------------------------------------------------------------------\n\n#if MI_PADDING // && !MI_TRACK_ENABLED\nstatic bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) {\n *bsize = mi_page_usable_block_size(page);\n const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize);\n mi_track_mem_defined(padding,sizeof(mi_padding_t));\n *delta = padding->delta;\n uint32_t canary = padding->canary;\n uintptr_t keys[2];\n keys[0] = page->keys[0];\n keys[1] = page->keys[1];\n bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize);\n mi_track_mem_noaccess(padding,sizeof(mi_padding_t));\n return ok;\n}\n\n// Return the exact usable size of a block.\nstatic size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {\n if (is_guarded) {\n const size_t bsize = mi_page_block_size(page);\n return (bsize - _mi_os_page_size());\n }\n else {\n size_t bsize;\n size_t delta;\n bool ok = mi_page_decode_padding(page, block, &delta, &bsize);\n mi_assert_internal(ok); mi_assert_internal(delta <= bsize);\n return (ok ? bsize - delta : 0);\n }\n}\n\n// When a non-thread-local block is freed, it becomes part of the thread delayed free\n// list that is freed later by the owning heap. If the exact usable size is too small to\n// contain the pointer for the delayed list, then shrink the padding (by decreasing delta)\n// so it will later not trigger an overflow error in `mi_free_block`.\nvoid _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {\n size_t bsize;\n size_t delta;\n bool ok = mi_page_decode_padding(page, block, &delta, &bsize);\n mi_assert_internal(ok);\n if (!ok || (bsize - delta) >= min_size) return; // usually already enough space\n mi_assert_internal(bsize >= min_size);\n if (bsize < min_size) return; // should never happen\n size_t new_delta = (bsize - min_size);\n mi_assert_internal(new_delta < bsize);\n mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize);\n mi_track_mem_defined(padding,sizeof(mi_padding_t));\n padding->delta = (uint32_t)new_delta;\n mi_track_mem_noaccess(padding,sizeof(mi_padding_t));\n}\n#else\nstatic size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {\n MI_UNUSED(is_guarded); MI_UNUSED(block);\n return mi_page_usable_block_size(page);\n}\n\nvoid _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {\n MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(min_size);\n}\n#endif\n\n#if MI_PADDING && MI_PADDING_CHECK\n\nstatic bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) {\n size_t bsize;\n size_t delta;\n bool ok = mi_page_decode_padding(page, block, &delta, &bsize);\n *size = *wrong = bsize;\n if (!ok) return false;\n mi_assert_internal(bsize >= delta);\n *size = bsize - delta;\n if (!mi_page_is_huge(page)) {\n uint8_t* fill = (uint8_t*)block + bsize - delta;\n const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes\n mi_track_mem_defined(fill, maxpad);\n for (size_t i = 0; i < maxpad; i++) {\n if (fill[i] != MI_DEBUG_PADDING) {\n *wrong = bsize - delta + i;\n ok = false;\n break;\n }\n }\n mi_track_mem_noaccess(fill, maxpad);\n }\n return ok;\n}\n\nstatic void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {\n size_t size;\n size_t wrong;\n if (!mi_verify_padding(page,block,&size,&wrong)) {\n _mi_error_message(EFAULT, \"buffer overflow in heap block %p of size %zu: write after %zu bytes\\n\", block, size, wrong );\n }\n}\n\n#else\n\nstatic void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {\n MI_UNUSED(page);\n MI_UNUSED(block);\n}\n\n#endif\n\n// only maintain stats for smaller objects if requested\n#if (MI_STAT>0)\nstatic void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {\n MI_UNUSED(block);\n mi_heap_t* const heap = mi_heap_get_default();\n const size_t bsize = mi_page_usable_block_size(page);\n // #if (MI_STAT>1)\n // const size_t usize = mi_page_usable_size_of(page, block);\n // mi_heap_stat_decrease(heap, malloc_requested, usize);\n // #endif\n if (bsize <= MI_MEDIUM_OBJ_SIZE_MAX) {\n mi_heap_stat_decrease(heap, malloc_normal, bsize);\n #if (MI_STAT > 1)\n mi_heap_stat_decrease(heap, malloc_bins[_mi_bin(bsize)], 1);\n #endif\n }\n //else if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {\n // mi_heap_stat_decrease(heap, malloc_large, bsize);\n //}\n else {\n mi_heap_stat_decrease(heap, malloc_huge, bsize);\n }\n}\n#else\nstatic void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {\n MI_UNUSED(page); MI_UNUSED(block);\n}\n#endif\n\n\n// Remove guard page when building with MI_GUARDED\n#if MI_GUARDED\nstatic void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {\n MI_UNUSED(p);\n mi_assert_internal(mi_block_ptr_is_guarded(block, p));\n mi_assert_internal(mi_page_has_aligned(page));\n mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t));\n mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED);\n\n const size_t bsize = mi_page_block_size(page);\n const size_t psize = _mi_os_page_size();\n mi_assert_internal(bsize > psize);\n mi_assert_internal(_mi_page_segment(page)->allow_decommit);\n void* gpage = (uint8_t*)block + bsize - psize;\n mi_assert_internal(_mi_is_aligned(gpage, psize));\n _mi_os_unprotect(gpage, psize);\n}\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/heap.c", "sha256": "302ef412b4788fff18e44f7b315ac2df78cfd5d22a999e4f9e5c14660d956b06", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/*----------------------------------------------------------------------------\nCopyright (c) 2018-2021, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\" // mi_prim_get_default_heap\n\n#include // memset, memcpy\n\n#if defined(_MSC_VER) && (_MSC_VER < 1920)\n#pragma warning(disable:4204) // non-constant aggregate initializer\n#endif\n\n/* -----------------------------------------------------------\n Helpers\n----------------------------------------------------------- */\n\n// return `true` if ok, `false` to break\ntypedef bool (heap_page_visitor_fun)(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2);\n\n// Visit all pages in a heap; returns `false` if break was called.\nstatic bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, bool include_full, void* arg1, void* arg2)\n{\n if (heap==NULL || heap->page_count==0) return 0;\n\n // visit all pages\n #if MI_DEBUG>1\n size_t total = heap->page_count;\n size_t count = 0;\n #endif\n\n const size_t max_bin = (include_full ? MI_BIN_FULL : MI_BIN_FULL - 1);\n for (size_t i = 0; i <= max_bin; i++) {\n mi_page_queue_t* pq = &heap->pages[i];\n mi_page_t* page = pq->first;\n while(page != NULL) {\n mi_page_t* next = page->next; // save next in case the page gets removed from the queue\n mi_assert_internal(mi_page_heap(page) == heap);\n #if MI_DEBUG>1\n count++;\n #endif\n if (!fn(heap, pq, page, arg1, arg2)) return false;\n page = next; // and continue\n }\n }\n mi_assert_internal(!include_full || count == total);\n return true;\n}\n\n\n#if MI_DEBUG>=2\nstatic bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {\n MI_UNUSED(arg1);\n MI_UNUSED(arg2);\n MI_UNUSED(pq);\n mi_assert_internal(mi_page_heap(page) == heap);\n mi_segment_t* segment = _mi_page_segment(page);\n mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == heap->thread_id);\n mi_assert_expensive(_mi_page_is_valid(page));\n return true;\n}\n#endif\n#if MI_DEBUG>=3\nstatic bool mi_heap_is_valid(mi_heap_t* heap) {\n mi_assert_internal(heap!=NULL);\n mi_heap_visit_pages(heap, &mi_heap_page_is_valid, true, NULL, NULL);\n return true;\n}\n#endif\n\n\n\n\n/* -----------------------------------------------------------\n \"Collect\" pages by migrating `local_free` and `thread_free`\n lists and freeing empty pages. This is done when a thread\n stops (and in that case abandons pages if there are still\n blocks alive)\n----------------------------------------------------------- */\n\ntypedef enum mi_collect_e {\n MI_NORMAL,\n MI_FORCE,\n MI_ABANDON\n} mi_collect_t;\n\n\nstatic bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) {\n MI_UNUSED(arg2);\n MI_UNUSED(heap);\n mi_assert_internal(mi_heap_page_is_valid(heap, pq, page, NULL, NULL));\n mi_collect_t collect = *((mi_collect_t*)arg_collect);\n _mi_page_free_collect(page, collect >= MI_FORCE);\n if (collect == MI_FORCE) {\n // note: call before a potential `_mi_page_free` as the segment may be freed if this was the last used page in that segment.\n mi_segment_t* segment = _mi_page_segment(page);\n _mi_segment_collect(segment, true /* force? */);\n }\n if (mi_page_all_free(page)) {\n // no more used blocks, free the page.\n // note: this will free retired pages as well.\n _mi_page_free(page, pq, collect >= MI_FORCE);\n }\n else if (collect == MI_ABANDON) {\n // still used blocks but the thread is done; abandon the page\n _mi_page_abandon(page, pq);\n }\n return true; // don't break\n}\n\nstatic bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {\n MI_UNUSED(arg1);\n MI_UNUSED(arg2);\n MI_UNUSED(heap);\n MI_UNUSED(pq);\n _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false);\n return true; // don't break\n}\n\nstatic void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)\n{\n if (heap==NULL || !mi_heap_is_initialized(heap)) return;\n\n const bool force = (collect >= MI_FORCE);\n _mi_deferred_free(heap, force);\n\n // python/cpython#112532: we may be called from a thread that is not the owner of the heap\n const bool is_main_thread = (_mi_is_main_thread() && heap->thread_id == _mi_thread_id());\n\n // note: never reclaim on collect but leave it to threads that need storage to reclaim\n const bool force_main =\n #ifdef NDEBUG\n collect == MI_FORCE\n #else\n collect >= MI_FORCE\n #endif\n && is_main_thread && mi_heap_is_backing(heap) && !heap->no_reclaim;\n\n if (force_main) {\n // the main thread is abandoned (end-of-program), try to reclaim all abandoned segments.\n // if all memory is freed by now, all segments should be freed.\n // note: this only collects in the current subprocess\n _mi_abandoned_reclaim_all(heap, &heap->tld->segments);\n }\n\n // if abandoning, mark all pages to no longer add to delayed_free\n if (collect == MI_ABANDON) {\n mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, true, NULL, NULL);\n }\n\n // free all current thread delayed blocks.\n // (if abandoning, after this there are no more thread-delayed references into the pages.)\n _mi_heap_delayed_free_all(heap);\n\n // collect retired pages\n _mi_heap_collect_retired(heap, force);\n\n // collect all pages owned by this thread\n mi_heap_visit_pages(heap, &mi_heap_page_collect, (collect!=MI_NORMAL), &collect, NULL); // normally don't visit full pages, issue #1220\n mi_assert_internal( collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t,&heap->thread_delayed_free) == NULL );\n\n // collect abandoned segments (in particular, purge expired parts of segments in the abandoned segment list)\n // note: forced purge can be quite expensive if many threads are created/destroyed so we do not force on abandonment\n _mi_abandoned_collect(heap, collect == MI_FORCE /* force? */, &heap->tld->segments);\n\n // if forced, collect thread data cache on program-exit (or shared library unload)\n if (force && is_main_thread && mi_heap_is_backing(heap)) {\n _mi_thread_data_collect(); // collect thread data cache\n }\n\n // collect arenas (this is program wide so don't force purges on abandonment of threads)\n _mi_arenas_collect(collect == MI_FORCE /* force purge? */);\n\n // merge statistics\n if (collect <= MI_FORCE) { _mi_stats_merge_thread(heap->tld); }\n}\n\nvoid _mi_heap_collect_abandon(mi_heap_t* heap) {\n mi_heap_collect_ex(heap, MI_ABANDON);\n}\n\nvoid mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept {\n mi_heap_collect_ex(heap, (force ? MI_FORCE : MI_NORMAL));\n}\n\nvoid mi_collect(bool force) mi_attr_noexcept {\n mi_heap_collect(mi_prim_get_default_heap(), force);\n}\n\n\n/* -----------------------------------------------------------\n Heap new\n----------------------------------------------------------- */\n\nmi_heap_t* mi_heap_get_default(void) {\n mi_thread_init();\n return mi_prim_get_default_heap();\n}\n\nstatic bool mi_heap_is_default(const mi_heap_t* heap) {\n return (heap == mi_prim_get_default_heap());\n}\n\n\nmi_heap_t* mi_heap_get_backing(void) {\n mi_heap_t* heap = mi_heap_get_default();\n mi_assert_internal(heap!=NULL);\n mi_heap_t* bheap = heap->tld->heap_backing;\n mi_assert_internal(bheap!=NULL);\n mi_assert_internal(bheap->thread_id == _mi_thread_id());\n return bheap;\n}\n\nvoid _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag) {\n _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t));\n heap->tld = tld;\n heap->thread_id = _mi_thread_id();\n heap->arena_id = arena_id;\n heap->no_reclaim = noreclaim;\n heap->tag = tag;\n if (heap == tld->heap_backing) {\n #if defined(_WIN32) && !defined(MI_SHARED_LIB)\n _mi_random_init_weak(&heap->random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)\n #else\n _mi_random_init(&heap->random);\n #endif\n }\n else {\n _mi_random_split(&tld->heap_backing->random, &heap->random);\n }\n heap->cookie = _mi_heap_random_next(heap) | 1;\n heap->keys[0] = _mi_heap_random_next(heap);\n heap->keys[1] = _mi_heap_random_next(heap);\n _mi_heap_guarded_init(heap);\n // push on the thread local heaps list\n heap->next = heap->tld->heaps;\n heap->tld->heaps = heap;\n}\n\nmi_decl_nodiscard mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id) {\n mi_heap_t* bheap = mi_heap_get_backing();\n mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode?\n if (heap == NULL) return NULL;\n mi_assert(heap_tag >= 0 && heap_tag < 256);\n _mi_heap_init(heap, bheap->tld, arena_id, allow_destroy /* no reclaim? */, (uint8_t)heap_tag /* heap tag */);\n return heap;\n}\n\nmi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) {\n return mi_heap_new_ex(0 /* default heap tag */, false /* don't allow `mi_heap_destroy` */, arena_id);\n}\n\nmi_decl_nodiscard mi_heap_t* mi_heap_new(void) {\n // don't reclaim abandoned memory or otherwise destroy is unsafe\n return mi_heap_new_ex(0 /* default heap tag */, true /* no reclaim */, _mi_arena_id_none());\n}\n\nbool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid) {\n return _mi_arena_memid_is_suitable(memid, heap->arena_id);\n}\n\nuintptr_t _mi_heap_random_next(mi_heap_t* heap) {\n return _mi_random_next(&heap->random);\n}\n\n// zero out the page queues\nstatic void mi_heap_reset_pages(mi_heap_t* heap) {\n mi_assert_internal(heap != NULL);\n mi_assert_internal(mi_heap_is_initialized(heap));\n // TODO: copy full empty heap instead?\n memset(&heap->pages_free_direct, 0, sizeof(heap->pages_free_direct));\n _mi_memcpy_aligned(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages));\n heap->thread_delayed_free = NULL;\n heap->page_count = 0;\n}\n\n// called from `mi_heap_destroy` and `mi_heap_delete` to free the internal heap resources.\nstatic void mi_heap_free(mi_heap_t* heap) {\n mi_assert(heap != NULL);\n mi_assert_internal(mi_heap_is_initialized(heap));\n if (heap==NULL || !mi_heap_is_initialized(heap)) return;\n if (mi_heap_is_backing(heap)) return; // dont free the backing heap\n\n // reset default\n if (mi_heap_is_default(heap)) {\n _mi_heap_set_default_direct(heap->tld->heap_backing);\n }\n\n // remove ourselves from the thread local heaps list\n // linear search but we expect the number of heaps to be relatively small\n mi_heap_t* prev = NULL;\n mi_heap_t* curr = heap->tld->heaps;\n while (curr != heap && curr != NULL) {\n prev = curr;\n curr = curr->next;\n }\n mi_assert_internal(curr == heap);\n if (curr == heap) {\n if (prev != NULL) { prev->next = heap->next; }\n else { heap->tld->heaps = heap->next; }\n }\n mi_assert_internal(heap->tld->heaps != NULL);\n\n // and free the used memory\n mi_free(heap);\n}\n\n// return a heap on the same thread as `heap` specialized for the specified tag (if it exists)\nmi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) {\n if (heap->tag == tag) {\n return heap;\n }\n for (mi_heap_t *curr = heap->tld->heaps; curr != NULL; curr = curr->next) {\n if (curr->tag == tag) {\n return curr;\n }\n }\n return NULL;\n}\n\n/* -----------------------------------------------------------\n Heap destroy\n----------------------------------------------------------- */\n\nstatic bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {\n MI_UNUSED(arg1);\n MI_UNUSED(arg2);\n MI_UNUSED(heap);\n MI_UNUSED(pq);\n\n // ensure no more thread_delayed_free will be added\n _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false);\n\n // stats\n const size_t bsize = mi_page_block_size(page);\n if (bsize > MI_MEDIUM_OBJ_SIZE_MAX) {\n //if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {\n // mi_heap_stat_decrease(heap, malloc_large, bsize);\n //}\n //else \n {\n mi_heap_stat_decrease(heap, malloc_huge, bsize);\n }\n }\n #if (MI_STAT>0)\n _mi_page_free_collect(page, false); // update used count\n const size_t inuse = page->used;\n if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {\n mi_heap_stat_decrease(heap, malloc_normal, bsize * inuse);\n #if (MI_STAT>1)\n mi_heap_stat_decrease(heap, malloc_bins[_mi_bin(bsize)], inuse);\n #endif\n }\n // mi_heap_stat_decrease(heap, malloc_requested, bsize * inuse); // todo: off for aligned blocks...\n #endif\n\n /// pretend it is all free now\n mi_assert_internal(mi_page_thread_free(page) == NULL);\n page->used = 0;\n\n // and free the page\n // mi_page_free(page,false);\n page->next = NULL;\n page->prev = NULL;\n _mi_segment_page_free(page,false /* no force? */, &heap->tld->segments);\n\n return true; // keep going\n}\n\nvoid _mi_heap_destroy_pages(mi_heap_t* heap) {\n mi_heap_visit_pages(heap, &_mi_heap_page_destroy, true, NULL, NULL);\n mi_heap_reset_pages(heap);\n}\n\n#if MI_TRACK_HEAP_DESTROY\nstatic bool mi_cdecl mi_heap_track_block_free(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) {\n MI_UNUSED(heap); MI_UNUSED(area); MI_UNUSED(arg); MI_UNUSED(block_size);\n mi_track_free_size(block,mi_usable_size(block));\n return true;\n}\n#endif\n\nvoid mi_heap_destroy(mi_heap_t* heap) {\n mi_assert(heap != NULL);\n mi_assert(mi_heap_is_initialized(heap));\n mi_assert(heap->no_reclaim);\n mi_assert_expensive(mi_heap_is_valid(heap));\n if (heap==NULL || !mi_heap_is_initialized(heap)) return;\n #if MI_GUARDED\n // _mi_warning_message(\"'mi_heap_destroy' called but MI_GUARDED is enabled -- using `mi_heap_delete` instead (heap at %p)\\n\", heap);\n mi_heap_delete(heap);\n return;\n #else\n if (!heap->no_reclaim) {\n _mi_warning_message(\"'mi_heap_destroy' called but ignored as the heap was not created with 'allow_destroy' (heap at %p)\\n\", heap);\n // don't free in case it may contain reclaimed pages\n mi_heap_delete(heap);\n }\n else {\n // track all blocks as freed\n #if MI_TRACK_HEAP_DESTROY\n mi_heap_visit_blocks(heap, true, mi_heap_track_block_free, NULL);\n #endif\n // free all pages\n _mi_heap_destroy_pages(heap);\n mi_heap_free(heap);\n }\n #endif\n}\n\n// forcefully destroy all heaps in the current thread\nvoid _mi_heap_unsafe_destroy_all(mi_heap_t* heap) {\n mi_assert_internal(heap != NULL);\n if (heap == NULL) return;\n mi_heap_t* curr = heap->tld->heaps;\n while (curr != NULL) {\n mi_heap_t* next = curr->next;\n if (curr->no_reclaim) {\n mi_heap_destroy(curr);\n }\n else {\n _mi_heap_destroy_pages(curr);\n }\n curr = next;\n }\n}\n\n/* -----------------------------------------------------------\n Safe Heap delete\n----------------------------------------------------------- */\n\n// Transfer the pages from one heap to the other\nstatic void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {\n mi_assert_internal(heap!=NULL);\n if (from==NULL || from->page_count == 0) return;\n\n // reduce the size of the delayed frees\n _mi_heap_delayed_free_partial(from);\n\n // transfer all pages by appending the queues; this will set a new heap field\n // so threads may do delayed frees in either heap for a while.\n // note: appending waits for each page to not be in the `MI_DELAYED_FREEING` state\n // so after this only the new heap will get delayed frees\n for (size_t i = 0; i <= MI_BIN_FULL; i++) {\n mi_page_queue_t* pq = &heap->pages[i];\n mi_page_queue_t* append = &from->pages[i];\n size_t pcount = _mi_page_queue_append(heap, pq, append);\n heap->page_count += pcount;\n from->page_count -= pcount;\n }\n mi_assert_internal(from->page_count == 0);\n\n // and do outstanding delayed frees in the `from` heap\n // note: be careful here as the `heap` field in all those pages no longer point to `from`,\n // turns out to be ok as `_mi_heap_delayed_free` only visits the list and calls a\n // the regular `_mi_free_delayed_block` which is safe.\n _mi_heap_delayed_free_all(from);\n #if !defined(_MSC_VER) || (_MSC_VER > 1900) // somehow the following line gives an error in VS2015, issue #353\n mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_block_t,&from->thread_delayed_free) == NULL);\n #endif\n\n // and reset the `from` heap\n mi_heap_reset_pages(from);\n}\n\n// are two heaps compatible with respect to heap-tag, exclusive arena etc.\nstatic bool mi_heaps_are_compatible(mi_heap_t* heap1, mi_heap_t* heap2) {\n return (heap1->tag == heap2->tag && // store same kind of objects\n heap1->arena_id == heap2->arena_id); // same arena preference\n}\n\n// Safe delete a heap without freeing any still allocated blocks in that heap.\nvoid mi_heap_delete(mi_heap_t* heap)\n{\n mi_assert(heap != NULL);\n mi_assert(mi_heap_is_initialized(heap));\n mi_assert_expensive(mi_heap_is_valid(heap));\n if (heap==NULL || !mi_heap_is_initialized(heap)) return;\n\n mi_heap_t* bheap = heap->tld->heap_backing;\n if (bheap != heap && mi_heaps_are_compatible(bheap,heap)) {\n // transfer still used pages to the backing heap\n mi_heap_absorb(bheap, heap);\n }\n else {\n // the backing heap abandons its pages\n _mi_heap_collect_abandon(heap);\n }\n mi_assert_internal(heap->page_count==0);\n mi_heap_free(heap);\n}\n\nmi_heap_t* mi_heap_set_default(mi_heap_t* heap) {\n mi_assert(heap != NULL);\n mi_assert(mi_heap_is_initialized(heap));\n if (heap==NULL || !mi_heap_is_initialized(heap)) return NULL;\n mi_assert_expensive(mi_heap_is_valid(heap));\n mi_heap_t* old = mi_prim_get_default_heap();\n _mi_heap_set_default_direct(heap);\n return old;\n}\n\n\n\n\n/* -----------------------------------------------------------\n Analysis\n----------------------------------------------------------- */\n\n// static since it is not thread safe to access heaps from other threads.\nstatic mi_heap_t* mi_heap_of_block(const void* p) {\n if (p == NULL) return NULL;\n mi_segment_t* segment = _mi_ptr_segment(p);\n bool valid = (_mi_ptr_cookie(segment) == segment->cookie);\n mi_assert_internal(valid);\n if mi_unlikely(!valid) return NULL;\n return mi_page_heap(_mi_segment_page_of(segment,p));\n}\n\nbool mi_heap_contains_block(mi_heap_t* heap, const void* p) {\n mi_assert(heap != NULL);\n if (heap==NULL || !mi_heap_is_initialized(heap)) return false;\n return (heap == mi_heap_of_block(p));\n}\n\n\nstatic bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* p, void* vfound) {\n MI_UNUSED(heap);\n MI_UNUSED(pq);\n bool* found = (bool*)vfound;\n void* start = mi_page_start(page);\n void* end = (uint8_t*)start + (page->capacity * mi_page_block_size(page));\n *found = (p >= start && p < end);\n return (!*found); // continue if not found\n}\n\nbool mi_heap_check_owned(mi_heap_t* heap, const void* p) {\n mi_assert(heap != NULL);\n if (heap==NULL || !mi_heap_is_initialized(heap)) return false;\n if (((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) return false; // only aligned pointers\n bool found = false;\n mi_heap_visit_pages(heap, &mi_heap_page_check_owned, true, (void*)p, &found);\n return found;\n}\n\nbool mi_check_owned(const void* p) {\n return mi_heap_check_owned(mi_prim_get_default_heap(), p);\n}\n\n/* -----------------------------------------------------------\n Visit all heap blocks and areas\n Todo: enable visiting abandoned pages, and\n enable visiting all blocks of all heaps across threads\n----------------------------------------------------------- */\n\nvoid _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) {\n const size_t bsize = mi_page_block_size(page);\n const size_t ubsize = mi_page_usable_block_size(page);\n area->reserved = page->reserved * bsize;\n area->committed = page->capacity * bsize;\n area->blocks = mi_page_start(page);\n area->used = page->used; // number of blocks in use (#553)\n area->block_size = ubsize;\n area->full_block_size = bsize;\n area->heap_tag = page->heap_tag;\n}\n\n\nstatic void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) {\n mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX);\n *shift = MI_SIZE_BITS - mi_clz(divisor - 1);\n *magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1);\n}\n\nstatic size_t mi_fast_divide(size_t n, uint64_t magic, size_t shift) {\n mi_assert_internal(n <= UINT32_MAX);\n const uint64_t hi = ((uint64_t)n * magic) >> 32;\n return (size_t)((hi + n) >> shift);\n}\n\nbool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg) {\n mi_assert(area != NULL);\n if (area==NULL) return true;\n mi_assert(page != NULL);\n if (page == NULL) return true;\n\n // collect early so the page used count is reported correctly\n // _mi_page_free_collect(page,true); // collect both thread_delayed and local_free\n mi_assert_internal(page->local_free == NULL);\n if (page->used == 0) return true;\n\n size_t psize;\n uint8_t* const pstart = _mi_segment_page_start(_mi_page_segment(page), page, &psize);\n mi_heap_t* const heap = mi_page_heap(page);\n const size_t bsize = mi_page_block_size(page);\n const size_t ubsize = mi_page_usable_block_size(page); // without padding\n\n // optimize page with one block\n if (page->capacity == 1) {\n mi_assert_internal(page->used == 1 && page->free == NULL);\n return visitor(mi_page_heap(page), area, pstart, ubsize, arg);\n }\n mi_assert(bsize <= UINT32_MAX);\n\n // optimize full pages\n if (page->used == page->capacity) {\n uint8_t* block = pstart;\n for (size_t i = 0; i < page->capacity; i++) {\n if (!visitor(heap, area, block, ubsize, arg)) return false;\n block += bsize;\n }\n return true;\n }\n\n // create a bitmap of free blocks.\n #define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*))\n uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS];\n const uintptr_t bmapsize = _mi_divide_up(page->capacity, MI_INTPTR_BITS);\n memset(free_map, 0, bmapsize * sizeof(intptr_t));\n if (page->capacity % MI_INTPTR_BITS != 0) {\n // mark left-over bits at the end as free\n size_t shift = (page->capacity % MI_INTPTR_BITS);\n uintptr_t mask = (UINTPTR_MAX << shift);\n free_map[bmapsize - 1] = mask;\n }\n\n // fast repeated division by the block size\n uint64_t magic;\n size_t shift;\n mi_get_fast_divisor(bsize, &magic, &shift);\n\n #if MI_DEBUG>1\n size_t free_count = 0;\n #endif\n for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) {\n #if MI_DEBUG>1\n free_count++;\n #endif\n mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize));\n size_t offset = (uint8_t*)block - pstart;\n mi_assert_internal(offset % bsize == 0);\n mi_assert_internal(offset <= UINT32_MAX);\n size_t blockidx = mi_fast_divide(offset, magic, shift);\n mi_assert_internal(blockidx == offset / bsize);\n mi_assert_internal(blockidx < MI_MAX_BLOCKS);\n size_t bitidx = (blockidx / MI_INTPTR_BITS);\n size_t bit = blockidx - (bitidx * MI_INTPTR_BITS);\n free_map[bitidx] |= ((uintptr_t)1 << bit);\n }\n mi_assert_internal(page->capacity == (free_count + page->used));\n\n // walk through all blocks skipping the free ones\n #if MI_DEBUG>1\n size_t used_count = 0;\n #endif\n uint8_t* block = pstart;\n for (size_t i = 0; i < bmapsize; i++) {\n if (free_map[i] == 0) {\n // every block is in use\n for (size_t j = 0; j < MI_INTPTR_BITS; j++) {\n #if MI_DEBUG>1\n used_count++;\n #endif\n if (!visitor(heap, area, block, ubsize, arg)) return false;\n block += bsize;\n }\n }\n else {\n // visit the used blocks in the mask\n uintptr_t m = ~free_map[i];\n while (m != 0) {\n #if MI_DEBUG>1\n used_count++;\n #endif\n size_t bitidx = mi_ctz(m);\n if (!visitor(heap, area, block + (bitidx * bsize), ubsize, arg)) return false;\n m &= m - 1; // clear least significant bit\n }\n block += bsize * MI_INTPTR_BITS;\n }\n }\n mi_assert_internal(page->used == used_count);\n return true;\n}\n\n\n\n// Separate struct to keep `mi_page_t` out of the public interface\ntypedef struct mi_heap_area_ex_s {\n mi_heap_area_t area;\n mi_page_t* page;\n} mi_heap_area_ex_t;\n\ntypedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ex_t* area, void* arg);\n\nstatic bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) {\n MI_UNUSED(heap);\n MI_UNUSED(pq);\n if (page==NULL) return true;\n mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun;\n mi_heap_area_ex_t xarea;\n xarea.page = page; \n _mi_page_free_collect(page,true); // collect early so the page->used is accurate \n _mi_heap_area_init(&xarea.area, page);\n return fun(heap, &xarea, arg);\n}\n\n// Visit all heap pages as areas\nstatic bool mi_heap_visit_areas(const mi_heap_t* heap, mi_heap_area_visit_fun* visitor, void* arg) {\n if (visitor == NULL) return false;\n return mi_heap_visit_pages((mi_heap_t*)heap, &mi_heap_visit_areas_page, true, (void*)(visitor), arg); // note: function pointer to void* :-{\n}\n\n// Just to pass arguments\ntypedef struct mi_visit_blocks_args_s {\n bool visit_blocks;\n mi_block_visit_fun* visitor;\n void* arg;\n} mi_visit_blocks_args_t;\n\nstatic bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t* xarea, void* arg) {\n mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg;\n if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg)) return false;\n if (args->visit_blocks) {\n return _mi_heap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg);\n }\n else {\n return true;\n }\n}\n\n// Visit all blocks in a heap\nbool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {\n mi_visit_blocks_args_t args = { visit_blocks, visitor, arg };\n return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args);\n}\n\n\n\nstatic const mi_page_t* mi_safe_ptr_page(void* p) {\n const mi_segment_t* const segment = _mi_ptr_segment(p);\n if mi_unlikely(segment==NULL) return NULL;\n #ifndef NDEBUG\n if mi_unlikely(!mi_is_in_heap_region(p)) return NULL;\n #endif\n if mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie) return NULL;\n return _mi_segment_page_of(segment, p);\n}\n\n// unsafe heap utilization function for DragonFly (see issue #1258)\n// If the page of pointer `p` belongs to `heap` (or `heap==NULL`) and has less than `perc_threshold` used blocks in its used area return `true`.\n// This function is unsafe in general as it assumes we are the only thread accessing the page of `p`.\nbool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept {\n if (p==NULL) return false;\n const mi_page_t* const page = mi_safe_ptr_page(p); // Get the page containing this pointer\n if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false;\n // If the page is the head of the queue, it is currently being used for \n // allocations; we skip it to avoid immediate thrashing.\n if (page->prev == NULL) return false;\n\n // match heap?\n const mi_heap_t* const page_heap = mi_page_heap(page);\n if (page_heap==NULL) return false;\n if (heap!=NULL && page_heap!=heap) return false;\n \n // check utilization\n if (page->capacity==0) return false;\n if (perc_threshold>=100) return true;\n return (perc_threshold >= ((100UL*page->used) / page->capacity));\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/init.c", "sha256": "d1e5df48a718a2f623fb5c0a357793e6ebb70c23cffe0b32f4761a0d72d047eb", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2022, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\"\n\n#include // memcpy, memset\n#include // atexit\n\n\n// Empty page used to initialize the small free pages array\nconst mi_page_t _mi_page_empty = {\n 0,\n false, false, false, false,\n 0, // capacity\n 0, // reserved capacity\n { 0 }, // flags\n false, // is_zero\n 0, // retire_expire\n NULL, // free\n NULL, // local_free\n 0, // used\n 0, // block size shift\n 0, // heap tag\n 0, // block_size\n NULL, // page_start\n #if (MI_PADDING || MI_ENCODE_FREELIST)\n { 0, 0 },\n #endif\n MI_ATOMIC_VAR_INIT(0), // xthread_free\n MI_ATOMIC_VAR_INIT(0), // xheap\n NULL, NULL\n , { 0 } // padding\n};\n\n#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty)\n\n#if (MI_SMALL_WSIZE_MAX==128)\n#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8)\n#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }\n#elif (MI_PADDING>0)\n#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }\n#else\n#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() }\n#endif\n#else\n#error \"define right initialization sizes corresponding to MI_SMALL_WSIZE_MAX\"\n#endif\n\n// Empty page queues for every bin\n#define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) }\n#define MI_PAGE_QUEUES_EMPTY \\\n { QNULL(1), \\\n QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \\\n QNULL( 10), QNULL( 12), QNULL( 14), QNULL( 16), QNULL( 20), QNULL( 24), QNULL( 28), QNULL( 32), /* 16 */ \\\n QNULL( 40), QNULL( 48), QNULL( 56), QNULL( 64), QNULL( 80), QNULL( 96), QNULL( 112), QNULL( 128), /* 24 */ \\\n QNULL( 160), QNULL( 192), QNULL( 224), QNULL( 256), QNULL( 320), QNULL( 384), QNULL( 448), QNULL( 512), /* 32 */ \\\n QNULL( 640), QNULL( 768), QNULL( 896), QNULL( 1024), QNULL( 1280), QNULL( 1536), QNULL( 1792), QNULL( 2048), /* 40 */ \\\n QNULL( 2560), QNULL( 3072), QNULL( 3584), QNULL( 4096), QNULL( 5120), QNULL( 6144), QNULL( 7168), QNULL( 8192), /* 48 */ \\\n QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \\\n QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \\\n QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \\\n QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \\\n QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 2) /* Full queue */ }\n\n#define MI_STAT_COUNT_NULL() {0,0,0}\n\n// Empty statistics\n#define MI_STATS_NULL \\\n MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \\\n { 0 }, { 0 }, \\\n MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \\\n MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \\\n { 0 }, { 0 }, { 0 }, { 0 }, \\\n { 0 }, { 0 }, { 0 }, { 0 }, \\\n \\\n { 0 }, { 0 }, { 0 }, { 0 }, { 0 }, { 0 }, \\\n MI_INIT4(MI_STAT_COUNT_NULL), \\\n { 0 }, { 0 }, { 0 }, { 0 }, \\\n \\\n { MI_INIT4(MI_STAT_COUNT_NULL) }, \\\n { { 0 }, { 0 }, { 0 }, { 0 } }, \\\n \\\n { MI_INIT74(MI_STAT_COUNT_NULL) }, \\\n { MI_INIT74(MI_STAT_COUNT_NULL) }\n\n\n// Empty slice span queues for every bin\n#define SQNULL(sz) { NULL, NULL, sz }\n#define MI_SEGMENT_SPAN_QUEUES_EMPTY \\\n { SQNULL(1), \\\n SQNULL( 1), SQNULL( 2), SQNULL( 3), SQNULL( 4), SQNULL( 5), SQNULL( 6), SQNULL( 7), SQNULL( 10), /* 8 */ \\\n SQNULL( 12), SQNULL( 14), SQNULL( 16), SQNULL( 20), SQNULL( 24), SQNULL( 28), SQNULL( 32), SQNULL( 40), /* 16 */ \\\n SQNULL( 48), SQNULL( 56), SQNULL( 64), SQNULL( 80), SQNULL( 96), SQNULL( 112), SQNULL( 128), SQNULL( 160), /* 24 */ \\\n SQNULL( 192), SQNULL( 224), SQNULL( 256), SQNULL( 320), SQNULL( 384), SQNULL( 448), SQNULL( 512), SQNULL( 640), /* 32 */ \\\n SQNULL( 768), SQNULL( 896), SQNULL( 1024) /* 35 */ }\n\n\n// --------------------------------------------------------\n// Statically allocate an empty heap as the initial\n// thread local value for the default heap,\n// and statically allocate the backing heap for the main\n// thread so it can function without doing any allocation\n// itself (as accessing a thread local for the first time\n// may lead to allocation itself on some platforms)\n// --------------------------------------------------------\n\nmi_decl_cache_align const mi_heap_t _mi_heap_empty = {\n NULL,\n MI_ATOMIC_VAR_INIT(NULL),\n 0, // tid\n 0, // cookie\n 0, // arena id\n { 0, 0 }, // keys\n { {0}, {0}, 0, true }, // random\n 0, // page count\n MI_BIN_FULL, 0, // page retired min/max\n 0, // pages_full_size\n 0, 0, // generic count\n NULL, // next\n false, // can reclaim\n 0, // tag\n #if MI_GUARDED\n 0, 0, 0, 1, // count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`)\n #endif\n MI_SMALL_PAGES_EMPTY,\n MI_PAGE_QUEUES_EMPTY\n};\n\nstatic mi_decl_cache_align mi_subproc_t mi_subproc_default;\n\n#define tld_empty_stats ((mi_stats_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,stats)))\n\nmi_decl_cache_align static const mi_tld_t tld_empty = {\n 0,\n false,\n NULL, NULL,\n { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, &mi_subproc_default, tld_empty_stats }, // segments\n { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL } // stats\n};\n\nmi_threadid_t _mi_thread_id(void) mi_attr_noexcept {\n mi_threadid_t tid = _mi_prim_thread_id();\n mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits\n return tid;\n}\n\n// the thread-local default heap for allocation\nmi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;\n\nextern mi_decl_hidden mi_heap_t _mi_heap_main;\n\nstatic mi_decl_cache_align mi_tld_t tld_main = {\n 0, false,\n &_mi_heap_main, & _mi_heap_main,\n { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, &mi_subproc_default, &tld_main.stats }, // segments\n { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL } // stats\n};\n\nmi_decl_cache_align mi_heap_t _mi_heap_main = {\n &tld_main,\n MI_ATOMIC_VAR_INIT(NULL),\n 0, // thread id\n 0, // initial cookie\n 0, // arena id\n { 0, 0 }, // the key of the main heap can be fixed (unlike page keys that need to be secure!)\n { {0x846ca68b}, {0}, 0, true }, // random\n 0, // page count\n MI_BIN_FULL, 0, // page retired min/max\n 0, // pages_full_size\n 0, 0, // generic count\n NULL, // next heap\n false, // can reclaim\n 0, // tag\n #if MI_GUARDED\n 0, 0, 0, 0,\n #endif\n MI_SMALL_PAGES_EMPTY,\n MI_PAGE_QUEUES_EMPTY\n};\n\nbool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.\n\nmi_stats_t _mi_stats_main = { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL };\n\n#if MI_GUARDED\nmi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed) {\n heap->guarded_sample_rate = sample_rate;\n heap->guarded_sample_count = sample_rate; // count down samples\n if (heap->guarded_sample_rate > 1) {\n if (seed == 0) {\n seed = _mi_heap_random_next(heap);\n }\n heap->guarded_sample_count = (seed % heap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate`\n }\n}\n\nmi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max) {\n heap->guarded_size_min = min;\n heap->guarded_size_max = (min > max ? min : max);\n}\n\nvoid _mi_heap_guarded_init(mi_heap_t* heap) {\n mi_heap_guarded_set_sample_rate(heap,\n (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX),\n (size_t)mi_option_get(mi_option_guarded_sample_seed));\n mi_heap_guarded_set_size_bound(heap,\n (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX),\n (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) );\n}\n#else\nmi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed) {\n MI_UNUSED(heap); MI_UNUSED(sample_rate); MI_UNUSED(seed);\n}\n\nmi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max) {\n MI_UNUSED(heap); MI_UNUSED(min); MI_UNUSED(max);\n}\nvoid _mi_heap_guarded_init(mi_heap_t* heap) {\n MI_UNUSED(heap);\n}\n#endif\n\n\nstatic void mi_heap_main_init(void) {\n if (_mi_heap_main.cookie == 0) {\n _mi_heap_main.thread_id = _mi_thread_id();\n _mi_heap_main.cookie = 1;\n #if defined(_WIN32) && !defined(MI_SHARED_LIB)\n _mi_random_init_weak(&_mi_heap_main.random); // prevent allocation failure during bcrypt dll initialization with static linking\n #else\n _mi_random_init(&_mi_heap_main.random);\n #endif\n _mi_heap_main.cookie = _mi_heap_random_next(&_mi_heap_main);\n _mi_heap_main.keys[0] = _mi_heap_random_next(&_mi_heap_main);\n _mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main);\n mi_lock_init(&mi_subproc_default.abandoned_os_lock);\n mi_lock_init(&mi_subproc_default.abandoned_os_visit_lock);\n _mi_heap_guarded_init(&_mi_heap_main);\n }\n}\n\nmi_heap_t* _mi_heap_main_get(void) {\n mi_heap_main_init();\n return &_mi_heap_main;\n}\n\n/* -----------------------------------------------------------\n Sub process\n----------------------------------------------------------- */\n\nmi_subproc_id_t mi_subproc_main(void) {\n return NULL;\n}\n\nmi_subproc_id_t mi_subproc_new(void) {\n mi_memid_t memid = _mi_memid_none();\n mi_subproc_t* subproc = (mi_subproc_t*)_mi_arena_meta_zalloc(sizeof(mi_subproc_t), &memid);\n if (subproc == NULL) return NULL;\n subproc->memid = memid;\n subproc->abandoned_os_list = NULL;\n mi_lock_init(&subproc->abandoned_os_lock);\n mi_lock_init(&subproc->abandoned_os_visit_lock);\n return subproc;\n}\n\nmi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) {\n return (subproc_id == NULL ? &mi_subproc_default : (mi_subproc_t*)subproc_id);\n}\n\nvoid mi_subproc_delete(mi_subproc_id_t subproc_id) {\n if (subproc_id == NULL) return;\n mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);\n if (subproc==NULL) return;\n // check if there are no abandoned segments still..\n bool safe_to_delete = false;\n mi_lock(&subproc->abandoned_os_lock) {\n if (subproc->abandoned_os_list == NULL) {\n safe_to_delete = true;\n }\n }\n if (!safe_to_delete) return;\n // safe to release\n // todo: should we refcount subprocesses?\n mi_lock_done(&subproc->abandoned_os_lock);\n mi_lock_done(&subproc->abandoned_os_visit_lock);\n _mi_arena_meta_free(subproc, subproc->memid, sizeof(mi_subproc_t));\n}\n\nvoid mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) {\n mi_heap_t* heap = mi_heap_get_default();\n if (heap == NULL) return;\n mi_assert(heap->tld->segments.subproc == &mi_subproc_default);\n if (heap->tld->segments.subproc != &mi_subproc_default) return;\n heap->tld->segments.subproc = _mi_subproc_from_id(subproc_id);\n}\n\n\n\n/* -----------------------------------------------------------\n Initialization and freeing of the thread local heaps\n----------------------------------------------------------- */\n\n// note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size).\ntypedef struct mi_thread_data_s {\n mi_heap_t heap; // must come first due to cast in `_mi_heap_done`\n mi_tld_t tld;\n mi_memid_t memid; // must come last due to zero'ing\n} mi_thread_data_t;\n\n\n// Thread meta-data is allocated directly from the OS. For\n// some programs that do not use thread pools and allocate and\n// destroy many OS threads, this may causes too much overhead\n// per thread so we maintain a small cache of recently freed metadata.\n\n#define TD_CACHE_SIZE (32)\nstatic _Atomic(mi_thread_data_t*) td_cache[TD_CACHE_SIZE];\n\nstatic mi_thread_data_t* mi_thread_data_zalloc(void) {\n // try to find thread metadata in the cache\n mi_thread_data_t* td = NULL;\n for (int i = 0; i < TD_CACHE_SIZE; i++) {\n td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]);\n if (td != NULL) {\n // found cached allocation, try use it\n td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL);\n if (td != NULL) {\n _mi_memzero(td, offsetof(mi_thread_data_t,memid));\n return td;\n }\n }\n }\n\n // if that fails, allocate as meta data\n mi_memid_t memid;\n td = (mi_thread_data_t*)_mi_os_zalloc(sizeof(mi_thread_data_t), &memid);\n if (td == NULL) {\n // if this fails, try once more. (issue #257)\n td = (mi_thread_data_t*)_mi_os_zalloc(sizeof(mi_thread_data_t), &memid);\n if (td == NULL) {\n // really out of memory\n _mi_error_message(ENOMEM, \"unable to allocate thread local heap metadata (%zu bytes)\\n\", sizeof(mi_thread_data_t));\n return NULL;\n }\n }\n td->memid = memid;\n return td;\n}\n\nstatic void mi_thread_data_free( mi_thread_data_t* tdfree ) {\n // try to add the thread metadata to the cache\n for (int i = 0; i < TD_CACHE_SIZE; i++) {\n mi_thread_data_t* td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]);\n if (td == NULL) {\n mi_thread_data_t* expected = NULL;\n if (mi_atomic_cas_ptr_weak_acq_rel(mi_thread_data_t, &td_cache[i], &expected, tdfree)) {\n return;\n }\n }\n }\n // if that fails, just free it directly\n _mi_os_free(tdfree, sizeof(mi_thread_data_t), tdfree->memid);\n}\n\nvoid _mi_thread_data_collect(void) {\n // free all thread metadata from the cache\n for (int i = 0; i < TD_CACHE_SIZE; i++) {\n mi_thread_data_t* td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]);\n if (td != NULL) {\n td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL);\n if (td != NULL) {\n _mi_os_free(td, sizeof(mi_thread_data_t), td->memid);\n }\n }\n }\n}\n\n// Initialize the thread local default heap, called from `mi_thread_init`\nstatic bool _mi_thread_heap_init(void) {\n if (mi_heap_is_initialized(mi_prim_get_default_heap())) return true;\n if (_mi_is_main_thread()) {\n // mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization\n // the main heap is statically allocated\n mi_heap_main_init();\n _mi_heap_set_default_direct(&_mi_heap_main);\n //mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_prim_get_default_heap());\n }\n else {\n // use `_mi_os_alloc` to allocate directly from the OS\n mi_thread_data_t* td = mi_thread_data_zalloc();\n if (td == NULL) return false;\n\n mi_tld_t* tld = &td->tld;\n mi_heap_t* heap = &td->heap;\n _mi_tld_init(tld, heap); // must be before `_mi_heap_init`\n _mi_heap_init(heap, tld, _mi_arena_id_none(), false /* can reclaim */, 0 /* default tag */);\n _mi_heap_set_default_direct(heap);\n }\n return false;\n}\n\n// initialize thread local data\nvoid _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) {\n _mi_memcpy_aligned(tld, &tld_empty, sizeof(mi_tld_t));\n tld->heap_backing = bheap;\n tld->heaps = NULL;\n tld->segments.subproc = &mi_subproc_default;\n tld->segments.stats = &tld->stats;\n}\n\n// Free the thread local default heap (called from `mi_thread_done`)\nstatic bool _mi_thread_heap_done(mi_heap_t* heap) {\n if (!mi_heap_is_initialized(heap)) return true;\n\n // reset default heap\n _mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty);\n\n // switch to backing heap\n heap = heap->tld->heap_backing;\n if (!mi_heap_is_initialized(heap)) return false;\n\n // delete all non-backing heaps in this thread\n mi_heap_t* curr = heap->tld->heaps;\n while (curr != NULL) {\n mi_heap_t* next = curr->next; // save `next` as `curr` will be freed\n if (curr != heap) {\n mi_assert_internal(!mi_heap_is_backing(curr));\n mi_heap_delete(curr);\n }\n curr = next;\n }\n mi_assert_internal(heap->tld->heaps == heap && heap->next == NULL);\n mi_assert_internal(mi_heap_is_backing(heap));\n\n // collect if not the main thread\n if (heap != &_mi_heap_main) {\n _mi_heap_collect_abandon(heap);\n }\n\n // merge stats\n _mi_stats_done(&heap->tld->stats);\n\n // free if not the main thread\n if (heap != &_mi_heap_main) {\n // the following assertion does not always hold for huge segments as those are always treated\n // as abandoned: one may allocate it in one thread, but deallocate in another in which case\n // the count can be too large or negative. todo: perhaps not count huge segments? see issue #363\n // mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id());\n mi_thread_data_free((mi_thread_data_t*)heap);\n }\n else {\n #if 0\n // never free the main thread even in debug mode; if a dll is linked statically with mimalloc,\n // there may still be delete/free calls after the mi_fls_done is called. Issue #207\n _mi_heap_destroy_pages(heap);\n mi_assert_internal(heap->tld->heap_backing == &_mi_heap_main);\n #endif\n }\n return false;\n}\n\n\n\n// --------------------------------------------------------\n// Try to run `mi_thread_done()` automatically so any memory\n// owned by the thread but not yet released can be abandoned\n// and re-owned by another thread.\n//\n// 1. windows dynamic library:\n// call from DllMain on DLL_THREAD_DETACH\n// 2. windows static library:\n// use `FlsAlloc` to call a destructor when the thread is done\n// 3. unix, pthreads:\n// use a pthread key to call a destructor when a pthread is done\n//\n// In the last two cases we also need to call `mi_process_init`\n// to set up the thread local keys.\n// --------------------------------------------------------\n\n// Set up handlers so `mi_thread_done` is called automatically\nstatic void mi_process_setup_auto_thread_done(void) {\n mi_atomic_do_once {\n _mi_prim_thread_init_auto_done();\n _mi_heap_set_default_direct(&_mi_heap_main);\n }\n}\n\n\nbool _mi_is_main_thread(void) {\n return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id());\n}\n\nstatic _Atomic(size_t) thread_count = MI_ATOMIC_VAR_INIT(1);\n\nsize_t _mi_current_thread_count(void) {\n return mi_atomic_load_relaxed(&thread_count);\n}\n\n// This is called from the `mi_malloc_generic`\nvoid mi_thread_init(void) mi_attr_noexcept\n{\n // ensure our process has started already\n mi_process_init();\n\n // initialize the thread local default heap\n // (this will call `_mi_heap_set_default_direct` and thus set the\n // fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called)\n if (_mi_thread_heap_init()) return; // returns true if already initialized\n\n _mi_stat_increase(&_mi_stats_main.threads, 1);\n mi_atomic_increment_relaxed(&thread_count);\n //_mi_verbose_message(\"thread init: 0x%zx\\n\", _mi_thread_id());\n}\n\nvoid mi_thread_done(void) mi_attr_noexcept {\n _mi_thread_done(NULL);\n}\n\nvoid _mi_thread_done(mi_heap_t* heap)\n{\n // calling with NULL implies using the default heap\n if (heap == NULL) {\n heap = mi_prim_get_default_heap();\n if (heap == NULL) return;\n }\n\n // prevent re-entrancy through heap_done/heap_set_default_direct (issue #699)\n if (!mi_heap_is_initialized(heap)) {\n return;\n }\n\n // adjust stats\n mi_atomic_decrement_relaxed(&thread_count);\n _mi_stat_decrease(&_mi_stats_main.threads, 1);\n\n // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps...\n if (heap->thread_id != _mi_thread_id()) return;\n\n // abandon the thread local heap\n if (_mi_thread_heap_done(heap)) return; // returns true if already ran\n}\n\nvoid _mi_heap_set_default_direct(mi_heap_t* heap) {\n mi_assert_internal(heap != NULL);\n #if defined(MI_TLS_SLOT)\n mi_prim_tls_slot_set(MI_TLS_SLOT,heap);\n #elif defined(MI_TLS_PTHREAD_SLOT_OFS)\n *mi_prim_tls_pthread_heap_slot() = heap;\n #elif defined(MI_TLS_PTHREAD)\n // we use _mi_heap_default_key\n #else\n _mi_heap_default = heap;\n #endif\n\n // ensure the default heap is passed to `_mi_thread_done`\n // setting to a non-NULL value also ensures `mi_thread_done` is called.\n _mi_prim_thread_associate_default_heap(heap);\n}\n\nvoid mi_thread_set_in_threadpool(void) mi_attr_noexcept {\n // nothing\n}\n\n// --------------------------------------------------------\n// Run functions on process init/done, and thread init/done\n// --------------------------------------------------------\nstatic bool os_preloading = true; // true until this module is initialized\n\n// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false.\nbool mi_decl_noinline _mi_preloading(void) {\n return os_preloading;\n}\n\n// Returns true if mimalloc was redirected\nmi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept {\n return _mi_is_redirected();\n}\n\n// Called once by the process loader from `src/prim/prim.c`\nvoid _mi_auto_process_init(void) {\n mi_heap_main_init();\n #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD)\n volatile mi_heap_t* dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true;\n if (dummy == NULL) return; // use dummy or otherwise the access may get optimized away (issue #697)\n #endif\n os_preloading = false;\n mi_assert_internal(_mi_is_main_thread());\n _mi_options_init();\n mi_process_setup_auto_thread_done();\n mi_process_init();\n if (_mi_is_redirected()) _mi_verbose_message(\"malloc is redirected.\\n\");\n\n // show message from the redirector (if present)\n const char* msg = NULL;\n _mi_allocator_init(&msg);\n if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) {\n _mi_fputs(NULL,NULL,NULL,msg);\n }\n\n // reseed random\n _mi_random_reinit_if_weak(&_mi_heap_main.random);\n}\n\n#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64))\n#include \nmi_decl_cache_align bool _mi_cpu_has_fsrm = false;\nmi_decl_cache_align bool _mi_cpu_has_erms = false;\n\nstatic void mi_detect_cpu_features(void) {\n // FSRM for fast short rep movsb/stosb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))\n // EMRS for fast enhanced rep movsb/stosb support\n int32_t cpu_info[4];\n __cpuid(cpu_info, 7);\n _mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see \n _mi_cpu_has_erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see \n}\n#else\nstatic void mi_detect_cpu_features(void) {\n // nothing\n}\n#endif\n\n// Initialize the process; called by thread_init or the process loader\nstatic void mi_process_init_once(void) mi_attr_noexcept {\n _mi_process_is_initialized = true; \n _mi_verbose_message(\"process init: 0x%zx\\n\", _mi_thread_id());\n mi_process_setup_auto_thread_done();\n\n mi_detect_cpu_features();\n _mi_os_init();\n mi_heap_main_init();\n mi_thread_init();\n\n #if defined(_WIN32)\n // On windows, when building as a static lib the FLS cleanup happens to early for the main thread.\n // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup\n // will not call _mi_thread_done on the (still executing) main thread. See issue #508.\n _mi_prim_thread_associate_default_heap(NULL);\n #endif\n\n mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL)\n mi_track_init();\n\n if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {\n size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024);\n int reserve_at = (int)mi_option_get_clamp(mi_option_reserve_huge_os_pages_at, -1, INT_MAX);\n if (reserve_at != -1) {\n mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500);\n } else {\n mi_reserve_huge_os_pages_interleave(pages, 0, pages*500);\n }\n }\n if (mi_option_is_enabled(mi_option_reserve_os_memory)) {\n long ksize = mi_option_get(mi_option_reserve_os_memory);\n if (ksize > 0) {\n mi_reserve_os_memory((size_t)ksize*MI_KiB, true /* commit? */, true /* allow large pages? */);\n }\n }\n}\n\n// Initialize the process; called by thread_init or the process loader\nvoid mi_process_init(void) mi_attr_noexcept {\n #if _MSC_VER < 1920\n\tmi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main\n\t#endif\n mi_atomic_do_once {\n mi_process_init_once();\n }\n}\n\n// Called when the process is done (cdecl as it is used with `at_exit` on some platforms)\nvoid mi_cdecl mi_process_done(void) mi_attr_noexcept {\n // only shutdown if we were initialized\n if (!_mi_process_is_initialized) return;\n // ensure we are called once\n static bool process_done = false;\n if (process_done) return;\n process_done = true;\n\n // get the default heap so we don't need to acces thread locals anymore\n mi_heap_t* heap = mi_prim_get_default_heap(); // use prim to not initialize any heap\n mi_assert_internal(heap != NULL);\n\n // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread\n _mi_prim_thread_done_auto_done();\n\n\n #ifndef MI_SKIP_COLLECT_ON_EXIT\n #if (MI_DEBUG || !defined(MI_SHARED_LIB))\n // free all memory if possible on process exit. This is not needed for a stand-alone process\n // but should be done if mimalloc is statically linked into another shared library which\n // is repeatedly loaded/unloaded, see issue #281.\n mi_heap_collect(heap, true /* force */ );\n #endif\n #endif\n\n // done with tracking tools\n mi_track_done()\n\n // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free\n // since after process_done there might still be other code running that calls `free` (like at_exit routines,\n // or C-runtime termination code.\n if (mi_option_is_enabled(mi_option_destroy_on_exit)) {\n mi_heap_collect(heap, true /* force */);\n _mi_heap_unsafe_destroy_all(heap); // forcefully release all memory held by all heaps (of this thread only!)\n _mi_arena_unsafe_destroy_all();\n _mi_segment_map_unsafe_destroy();\n }\n\n if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {\n mi_stats_print(NULL);\n }\n _mi_allocator_done();\n _mi_verbose_message(\"process done: 0x%zx\\n\", _mi_heap_main.thread_id);\n os_preloading = true; // don't call the C runtime anymore\n}\n\nvoid mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept {\n if (_mi_option_get_fast(mi_option_destroy_on_exit)>1) return;\n mi_process_done();\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/libc.c", "sha256": "b86471b36d9a9e216e29a8575f586c25428f9366975ed4068881e372d64abce3", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// --------------------------------------------------------\n// This module defines various std libc functions to reduce\n// the dependency on libc, and also prevent errors caused\n// by some libc implementations when called before `main`\n// executes (due to malloc redirection)\n// --------------------------------------------------------\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\" // mi_prim_getenv\n\nchar _mi_toupper(char c) {\n if (c >= 'a' && c <= 'z') return (c - 'a' + 'A');\n else return c;\n}\n\nint _mi_strnicmp(const char* s, const char* t, size_t n) {\n mi_assert_internal(s!=NULL && t!=NULL);\n if (n == 0) return 0;\n for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) {\n if (_mi_toupper(*s) != _mi_toupper(*t)) break;\n }\n return (n == 0 ? 0 : *s - *t);\n}\n\nbool _mi_streq(const char* s, const char* t) {\n if (s==NULL && t==NULL) return true;\n if (s==NULL || t==NULL) return false;\n for (; *s != 0 && *t != 0; s++, t++) {\n if (*s != *t) break;\n }\n return (*s == *t);\n}\n\nvoid _mi_strlcpy(char* dest, const char* src, size_t dest_size) {\n if (dest==NULL || src==NULL || dest_size == 0) return;\n // copy until end of src, or when dest is (almost) full\n while (*src != 0 && dest_size > 1) {\n *dest++ = *src++;\n dest_size--;\n }\n // always zero terminate\n *dest = 0;\n}\n\nvoid _mi_strlcat(char* dest, const char* src, size_t dest_size) {\n if (dest==NULL || src==NULL || dest_size == 0) return;\n // find end of string in the dest buffer\n while (*dest != 0 && dest_size > 1) {\n dest++;\n dest_size--;\n }\n // and catenate\n _mi_strlcpy(dest, src, dest_size);\n}\n\nsize_t _mi_strnlen(const char* s, size_t max_len) {\n if (s==NULL) return 0;\n size_t len = 0;\n while(s[len] != 0 && len < max_len) { len++; }\n return len;\n}\n\nsize_t _mi_strlen(const char* s) {\n return _mi_strnlen(s,PTRDIFF_MAX);\n}\n\n#ifdef MI_NO_GETENV\nbool _mi_getenv(const char* name, char* result, size_t result_size) {\n MI_UNUSED(name);\n MI_UNUSED(result);\n MI_UNUSED(result_size);\n return false;\n}\n#else\nbool _mi_getenv(const char* name, char* result, size_t result_size) {\n if (name==NULL || result == NULL || result_size < 64) return false;\n return _mi_prim_getenv(name,result,result_size);\n}\n#endif\n\n\n// --------------------------------------------------------\n// Define our own primitives for doing an action once\n// --------------------------------------------------------\n\n// Returns `true` only on the first invocation, signifying we can execute an action once.\n// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.\n// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.\nbool _mi_atomic_once_enter(mi_atomic_once_t* once) {\n const uintptr_t once_tid = mi_atomic_load_acquire(&once->tid);\n if mi_likely(once_tid == 1) {\n return false; // already executed\n }\n const mi_threadid_t current_tid = _mi_thread_id();\n if (once_tid == current_tid) {\n return false; // recursive invocation; we need this for process_init for example\n } \n\n mi_lock_acquire(&once->lock);\n uintptr_t expected = 0;\n if (mi_atomic_cas_strong_acq_rel(&once->tid, &expected, current_tid)) { // could use atomic_load/store as well\n return true; // should execute and release\n } \n else {\n mi_lock_release(&once->lock);\n return false; // already another thread entered and released\n }\n}\n\nvoid _mi_atomic_once_release(mi_atomic_once_t* once) {\n if (mi_atomic_load_acquire(&once->tid)>1) { // paranoia\n mi_atomic_store_release(&once->tid,1); // done executing\n mi_lock_release(&once->lock);\n } \n}\n\n\n// --------------------------------------------------------\n// Define our own limited `_mi_vsnprintf` and `_mi_snprintf`\n// This is mostly to avoid calling these when libc is not yet\n// initialized (and to reduce dependencies)\n//\n// format: d i, p x u, s\n// prec: z l ll L\n// width: 10\n// align-left: -\n// fill: 0\n// plus: +\n// --------------------------------------------------------\n\nstatic void mi_outc(char c, char** out, char* end) {\n char* p = *out;\n if (p >= end) return;\n *p = c;\n *out = p + 1;\n}\n\nstatic void mi_outs(const char* s, char** out, char* end) {\n if (s == NULL) return;\n char* p = *out;\n while (*s != 0 && p < end) {\n *p++ = *s++;\n }\n *out = p;\n}\n\nstatic void mi_out_fill(char fill, size_t len, char** out, char* end) {\n char* p = *out;\n for (size_t i = 0; i < len && p < end; i++) {\n *p++ = fill;\n }\n *out = p;\n}\n\nstatic void mi_out_alignright(char fill, char* start, size_t len, size_t extra, char* end) {\n if (len == 0 || extra == 0) return;\n if (start + len + extra >= end) return;\n // move `len` characters to the right (in reverse since it can overlap)\n for (size_t i = 1; i <= len; i++) {\n start[len + extra - i] = start[len - i];\n }\n // and fill the start\n for (size_t i = 0; i < extra; i++) {\n start[i] = fill;\n }\n}\n\n\nstatic void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* end)\n{\n if (x == 0 || base == 0 || base > 16) {\n if (prefix != 0) { mi_outc(prefix, out, end); }\n mi_outc('0',out,end);\n }\n else {\n // output digits in reverse\n char* start = *out;\n while (x > 0) {\n char digit = (char)(x % base);\n mi_outc((digit <= 9 ? '0' + digit : 'A' + digit - 10),out,end);\n x = x / base;\n }\n if (prefix != 0) {\n mi_outc(prefix, out, end);\n }\n size_t len = *out - start;\n // and reverse in-place\n for (size_t i = 0; i < (len / 2); i++) {\n char c = start[len - i - 1];\n start[len - i - 1] = start[i];\n start[i] = c;\n }\n }\n}\n\n\n#define MI_NEXTC() c = *in; if (c==0) break; in++;\n\nint _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {\n if (buf == NULL || bufsize == 0 || fmt == NULL) return 0;\n buf[bufsize - 1] = 0;\n char* const end = buf + (bufsize - 1);\n const char* in = fmt;\n char* out = buf;\n while (true) {\n if (out >= end) break;\n char c;\n MI_NEXTC();\n if (c != '%') {\n if ((c >= ' ' && c <= '~') || c=='\\n' || c=='\\r' || c=='\\t') { // output visible ascii or standard control only\n mi_outc(c, &out, end);\n }\n }\n else {\n MI_NEXTC();\n char fill = ' ';\n size_t width = 0;\n char numtype = 'd';\n char numplus = 0;\n bool alignright = true;\n if (c == '+' || c == ' ') { numplus = c; MI_NEXTC(); }\n if (c == '-') { alignright = false; MI_NEXTC(); }\n if (c == '0') { fill = '0'; MI_NEXTC(); }\n if (c >= '1' && c <= '9') {\n width = (c - '0'); MI_NEXTC();\n while (c >= '0' && c <= '9') {\n width = (10 * width) + (c - '0'); MI_NEXTC();\n }\n if (c == 0) break; // extra check due to while\n }\n if (c == 'z' || c == 't' || c == 'L') { numtype = c; MI_NEXTC(); }\n else if (c == 'l') {\n numtype = c; MI_NEXTC();\n if (c == 'l') { numtype = 'L'; MI_NEXTC(); }\n }\n\n char* start = out;\n if (c == 's') {\n // string\n const char* s = va_arg(args, const char*);\n mi_outs(s, &out, end);\n }\n else if (c == 'p' || c == 'x' || c == 'u') {\n // unsigned\n uintmax_t x = 0;\n if (c == 'x' || c == 'u') {\n if (numtype == 'z') x = va_arg(args, size_t);\n else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t\n else if (numtype == 'L') x = va_arg(args, unsigned long long);\n else if (numtype == 'l') x = va_arg(args, unsigned long);\n else x = va_arg(args, unsigned int);\n }\n else if (c == 'p') {\n void* const p = va_arg(args, void*);\n x = (uintptr_t)p;\n mi_outs(\"0x\", &out, end);\n start = out;\n width = (width >= 2 ? width - 2 : 0);\n }\n if (width == 0 && (c == 'x' || c == 'p')) {\n if (c == 'p') { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); }\n if (width == 0) { width = 2; }\n fill = '0';\n }\n mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end);\n }\n else if (c == 'i' || c == 'd') {\n // signed\n intmax_t x = 0;\n if (numtype == 'z') x = va_arg(args, intptr_t );\n else if (numtype == 't') x = va_arg(args, ptrdiff_t);\n else if (numtype == 'L') x = va_arg(args, long long);\n else if (numtype == 'l') x = va_arg(args, long);\n else x = va_arg(args, int);\n char pre = 0;\n if (x < 0) {\n pre = '-';\n if (x > INTMAX_MIN) { x = -x; }\n }\n else if (numplus != 0) {\n pre = numplus;\n }\n mi_out_num((uintmax_t)x, 10, pre, &out, end);\n }\n else if (c >= ' ' && c <= '~') {\n // unknown format\n mi_outc('%', &out, end);\n mi_outc(c, &out, end);\n }\n\n // fill & align\n mi_assert_internal(out <= end);\n mi_assert_internal(out >= start);\n const size_t len = out - start;\n if (len < width) {\n mi_out_fill(fill, width - len, &out, end);\n if (alignright && out <= end) {\n mi_out_alignright(fill, start, len, width - len, end);\n }\n }\n }\n }\n mi_assert_internal(out <= end);\n *out = 0;\n return (int)(out - buf);\n}\n\nint _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) {\n va_list args;\n va_start(args, fmt);\n const int written = _mi_vsnprintf(buf, buflen, fmt, args);\n va_end(args);\n return written;\n}\n\n\n#if MI_SIZE_SIZE == 4\n#define mi_mask_even_bits32 (0x55555555)\n#define mi_mask_even_pairs32 (0x33333333)\n#define mi_mask_even_nibbles32 (0x0F0F0F0F)\n\n// sum of all the bytes in `x` if it is guaranteed that the sum < 256!\nstatic size_t mi_byte_sum32(uint32_t x) {\n // perform `x * 0x01010101`: the highest byte contains the sum of all bytes.\n x += (x << 8);\n x += (x << 16);\n return (size_t)(x >> 24);\n}\n\nstatic size_t mi_popcount_generic32(uint32_t x) {\n // first count each 2-bit group `a`, where: a==0b00 -> 00, a==0b01 -> 01, a==0b10 -> 01, a==0b11 -> 10\n // in other words, `a - (a>>1)`; to do this in parallel, we need to mask to prevent spilling a bit pair\n // into the lower bit-pair:\n x = x - ((x >> 1) & mi_mask_even_bits32);\n // add the 2-bit pair results\n x = (x & mi_mask_even_pairs32) + ((x >> 2) & mi_mask_even_pairs32);\n // add the 4-bit nibble results\n x = (x + (x >> 4)) & mi_mask_even_nibbles32;\n // each byte now has a count of its bits, we can sum them now:\n return mi_byte_sum32(x);\n}\n\nmi_decl_noinline size_t _mi_popcount_generic(size_t x) {\n return mi_popcount_generic32(x);\n}\n\n#else\n#define mi_mask_even_bits64 (0x5555555555555555)\n#define mi_mask_even_pairs64 (0x3333333333333333)\n#define mi_mask_even_nibbles64 (0x0F0F0F0F0F0F0F0F)\n\n// sum of all the bytes in `x` if it is guaranteed that the sum < 256!\nstatic size_t mi_byte_sum64(uint64_t x) {\n x += (x << 8);\n x += (x << 16);\n x += (x << 32);\n return (size_t)(x >> 56);\n}\n\nstatic size_t mi_popcount_generic64(uint64_t x) {\n x = x - ((x >> 1) & mi_mask_even_bits64);\n x = (x & mi_mask_even_pairs64) + ((x >> 2) & mi_mask_even_pairs64);\n x = (x + (x >> 4)) & mi_mask_even_nibbles64;\n return mi_byte_sum64(x);\n}\n\nmi_decl_noinline size_t _mi_popcount_generic(size_t x) {\n return mi_popcount_generic64(x);\n}\n#endif\n\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/options.c", "sha256": "0efe6854ea11d7ba33d4387abb62f28aef36e494898fd1544383da4945b36d44", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2021, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\" // mi_prim_out_stderr\n\n#include // stdin/stdout\n#include // abort\n\n\n\nstatic long mi_max_error_count = 16; // stop outputting errors after this (use < 0 for no limit)\nstatic long mi_max_warning_count = 16; // stop outputting warnings after this (use < 0 for no limit)\n\nstatic void mi_add_stderr_output(void);\n\nint mi_version(void) mi_attr_noexcept {\n return MI_MALLOC_VERSION;\n}\n\n\n// --------------------------------------------------------\n// Options\n// These can be accessed by multiple threads and may be\n// concurrently initialized, but an initializing data race\n// is ok since they resolve to the same value.\n// --------------------------------------------------------\ntypedef enum mi_init_e {\n UNINIT, // not yet initialized\n DEFAULTED, // not found in the environment, use default value\n INITIALIZED // found in environment or set explicitly\n} mi_init_t;\n\ntypedef struct mi_option_desc_s {\n long value; // the value\n mi_init_t init; // is it initialized yet? (from the environment)\n mi_option_t option; // for debugging: the option index should match the option\n const char* name; // option name without `mimalloc_` prefix\n const char* legacy_name; // potential legacy option name\n} mi_option_desc_t;\n\n#define MI_OPTION(opt) mi_option_##opt, #opt, NULL\n#define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy\n\n// Some options can be set at build time for statically linked libraries\n// (use `-DMI_EXTRA_CPPDEFS=\"opt1=val1;opt2=val2\"`)\n//\n// This is useful if we cannot pass them as environment variables\n// (and setting them programmatically would be too late)\n\n#ifndef MI_DEFAULT_VERBOSE\n#define MI_DEFAULT_VERBOSE 0\n#endif\n\n#ifndef MI_DEFAULT_EAGER_COMMIT\n#define MI_DEFAULT_EAGER_COMMIT 1\n#endif\n\n#ifndef MI_DEFAULT_ARENA_EAGER_COMMIT\n#define MI_DEFAULT_ARENA_EAGER_COMMIT 2\n#endif\n\n// in KiB\n#ifndef MI_DEFAULT_ARENA_RESERVE\n #if (MI_INTPTR_SIZE>4)\n #define MI_DEFAULT_ARENA_RESERVE 1024L*1024L\n #else\n #define MI_DEFAULT_ARENA_RESERVE 128L*1024L\n #endif\n#endif\n\n#ifndef MI_DEFAULT_DISALLOW_ARENA_ALLOC\n#define MI_DEFAULT_DISALLOW_ARENA_ALLOC 0\n#endif\n\n#ifndef MI_DEFAULT_ALLOW_LARGE_OS_PAGES\n#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 0\n#endif\n\n#ifndef MI_DEFAULT_RESERVE_HUGE_OS_PAGES\n#define MI_DEFAULT_RESERVE_HUGE_OS_PAGES 0\n#endif\n\n#ifndef MI_DEFAULT_RESERVE_OS_MEMORY\n#define MI_DEFAULT_RESERVE_OS_MEMORY 0\n#endif\n\n#ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE\n#if MI_GUARDED && !MI_DEBUG\n#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000\n#else\n#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0\n#endif\n#endif\n\n\n#ifndef MI_DEFAULT_ALLOW_THP\n#if defined(__ANDROID__)\n#define MI_DEFAULT_ALLOW_THP 0\n#else\n#define MI_DEFAULT_ALLOW_THP 1\n#endif\n#endif\n\n// Static options\nstatic mi_option_desc_t options[_mi_option_last] =\n{\n // stable options\n #if MI_DEBUG || defined(MI_SHOW_ERRORS)\n { 1, UNINIT, MI_OPTION(show_errors) },\n #else\n { 0, UNINIT, MI_OPTION(show_errors) },\n #endif\n { 0, UNINIT, MI_OPTION(show_stats) },\n { MI_DEFAULT_VERBOSE, UNINIT, MI_OPTION(verbose) },\n\n // some of the following options are experimental and not all combinations are allowed.\n { MI_DEFAULT_EAGER_COMMIT,\n UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`)\n { MI_DEFAULT_ARENA_EAGER_COMMIT,\n UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux)\n { 1, UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit)\n { MI_DEFAULT_ALLOW_LARGE_OS_PAGES,\n UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's\n { MI_DEFAULT_RESERVE_HUGE_OS_PAGES,\n UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages\n {-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N\n { MI_DEFAULT_RESERVE_OS_MEMORY,\n UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`)\n { 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread\n { 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free\n { 0, UNINIT, MI_OPTION_LEGACY(abandoned_page_purge,abandoned_page_reset) }, // reset free page memory when a thread terminates\n { 0, UNINIT, MI_OPTION(deprecated_segment_reset) }, // reset segment memory on free (needs eager commit)\n#if defined(__NetBSD__)\n { 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed\n#else\n { 1, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed (but per page in the segment on demand)\n#endif\n { 10, UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds\n { 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.\n { 0, UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas)\n { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose\n { 32, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output\n { 32, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output\n { 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try.\n { 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees!\n { MI_DEFAULT_ARENA_RESERVE, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)\n { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's\n { 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) },\n { 0, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free\n { MI_DEFAULT_DISALLOW_ARENA_ALLOC, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's)\n { 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries.\n#if defined(MI_VISIT_ABANDONED)\n { 1, INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting heap blocks in abandoned segments; requires taking locks during reclaim.\n#else\n { 0, UNINIT, MI_OPTION(visit_abandoned) },\n#endif\n { 0, UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects\n { MI_GiB, UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects\n { 0, UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)\n { MI_DEFAULT_GUARDED_SAMPLE_RATE,\n UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded (=4000)\n { 0, UNINIT, MI_OPTION(guarded_sample_seed)},\n { 0, UNINIT, MI_OPTION(target_segments_per_thread) }, // abandon segments beyond this point, or 0 to disable.\n { 10000, UNINIT, MI_OPTION(generic_collect) }, // collect heaps every N (=10000) generic allocation calls\n { MI_DEFAULT_ALLOW_THP, \n UNINIT, MI_OPTION(allow_thp) } // allow transparent huge pages?\n};\n\nstatic void mi_option_init(mi_option_desc_t* desc);\n\nstatic bool mi_option_has_size_in_kib(mi_option_t option) {\n return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve);\n}\n\nvoid _mi_options_init(void) {\n // called on process load\n mi_add_stderr_output(); // now it safe to use stderr for output\n for(int i = 0; i < _mi_option_last; i++ ) {\n mi_option_t option = (mi_option_t)i;\n long l = mi_option_get(option); MI_UNUSED(l); // initialize\n }\n mi_max_error_count = mi_option_get(mi_option_max_errors);\n mi_max_warning_count = mi_option_get(mi_option_max_warnings);\n #if MI_GUARDED\n if (mi_option_get(mi_option_guarded_sample_rate) > 0) {\n if (mi_option_is_enabled(mi_option_allow_large_os_pages)) {\n mi_option_disable(mi_option_allow_large_os_pages);\n _mi_warning_message(\"option 'allow_large_os_pages' is disabled to allow for guarded objects\\n\");\n }\n }\n #endif\n if (mi_option_is_enabled(mi_option_verbose)) { mi_options_print(); }\n}\n\n#define mi_stringifyx(str) #str // and stringify\n#define mi_stringify(str) mi_stringifyx(str) // expand\n\nvoid mi_options_print(void) mi_attr_noexcept\n{\n // show version\n const int vermajor = MI_MALLOC_VERSION/10000;\n const int verminor = (MI_MALLOC_VERSION%10000)/100;\n const int verpatch = (MI_MALLOC_VERSION%100);\n _mi_message(\"v%i.%i.%i%s%s (built on %s, %s)\\n\", vermajor, verminor, verpatch,\n #if defined(MI_CMAKE_BUILD_TYPE)\n \", \" mi_stringify(MI_CMAKE_BUILD_TYPE)\n #else\n \"\"\n #endif\n ,\n #if defined(MI_GIT_DESCRIBE)\n \", git \" mi_stringify(MI_GIT_DESCRIBE)\n #else\n \"\"\n #endif\n , __DATE__, __TIME__);\n\n // show options\n for (int i = 0; i < _mi_option_last; i++) {\n mi_option_t option = (mi_option_t)i;\n long l = mi_option_get(option); MI_UNUSED(l); // possibly initialize\n mi_option_desc_t* desc = &options[option];\n _mi_message(\"option '%s': %ld %s\\n\", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? \"KiB\" : \"\"));\n }\n\n // show build configuration\n _mi_message(\"debug level : %d\\n\", MI_DEBUG );\n _mi_message(\"secure level: %d\\n\", MI_SECURE );\n _mi_message(\"mem tracking: %s\\n\", MI_TRACK_TOOL);\n #if MI_GUARDED\n _mi_message(\"guarded build: %s\\n\", mi_option_get(mi_option_guarded_sample_rate) != 0 ? \"enabled\" : \"disabled\");\n #endif\n #if MI_TSAN\n _mi_message(\"thread santizer enabled\\n\");\n #endif\n}\n\nlong _mi_option_get_fast(mi_option_t option) {\n mi_assert(option >= 0 && option < _mi_option_last);\n mi_option_desc_t* desc = &options[option];\n mi_assert(desc->option == option); // index should match the option\n //mi_assert(desc->init != UNINIT);\n return desc->value;\n}\n\n\nmi_decl_nodiscard long mi_option_get(mi_option_t option) {\n mi_assert(option >= 0 && option < _mi_option_last);\n if (option < 0 || option >= _mi_option_last) return 0;\n mi_option_desc_t* desc = &options[option];\n mi_assert(desc->option == option); // index should match the option\n if mi_unlikely(desc->init == UNINIT) {\n mi_option_init(desc);\n }\n return desc->value;\n}\n\nmi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long max) {\n long x = mi_option_get(option);\n return (x < min ? min : (x > max ? max : x));\n}\n\nmi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) {\n const long x = mi_option_get(option);\n size_t size = (x < 0 ? 0 : (size_t)x);\n if (mi_option_has_size_in_kib(option)) {\n if (mi_mul_overflow(size, MI_KiB, &size)) {\n size = MI_MAX_ALLOC_SIZE;\n }\n }\n return size;\n}\n\nvoid mi_option_set(mi_option_t option, long value) {\n mi_assert(option >= 0 && option < _mi_option_last);\n if (option < 0 || option >= _mi_option_last) return;\n mi_option_desc_t* desc = &options[option];\n mi_assert(desc->option == option); // index should match the option\n desc->value = value;\n desc->init = INITIALIZED;\n // ensure min/max range; be careful to not recurse.\n if (desc->option == mi_option_guarded_min && _mi_option_get_fast(mi_option_guarded_max) < value) {\n mi_option_set(mi_option_guarded_max, value);\n }\n else if (desc->option == mi_option_guarded_max && _mi_option_get_fast(mi_option_guarded_min) > value) {\n mi_option_set(mi_option_guarded_min, value);\n }\n}\n\nvoid mi_option_set_default(mi_option_t option, long value) {\n mi_assert(option >= 0 && option < _mi_option_last);\n if (option < 0 || option >= _mi_option_last) return;\n mi_option_desc_t* desc = &options[option];\n if (desc->init != INITIALIZED) {\n desc->value = value;\n }\n}\n\nmi_decl_nodiscard bool mi_option_is_enabled(mi_option_t option) {\n return (mi_option_get(option) != 0);\n}\n\nvoid mi_option_set_enabled(mi_option_t option, bool enable) {\n mi_option_set(option, (enable ? 1 : 0));\n}\n\nvoid mi_option_set_enabled_default(mi_option_t option, bool enable) {\n mi_option_set_default(option, (enable ? 1 : 0));\n}\n\nvoid mi_option_enable(mi_option_t option) {\n mi_option_set_enabled(option,true);\n}\n\nvoid mi_option_disable(mi_option_t option) {\n mi_option_set_enabled(option,false);\n}\n\nstatic void mi_cdecl mi_out_stderr(const char* msg, void* arg) {\n MI_UNUSED(arg);\n if (msg != NULL && msg[0] != 0) {\n _mi_prim_out_stderr(msg);\n }\n}\n\n// Since an output function can be registered earliest in the `main`\n// function we also buffer output that happens earlier. When\n// an output function is registered it is called immediately with\n// the output up to that point.\n#ifndef MI_MAX_DELAY_OUTPUT\n#define MI_MAX_DELAY_OUTPUT ((size_t)(16*1024))\n#endif\nstatic char out_buf[MI_MAX_DELAY_OUTPUT+1];\nstatic _Atomic(size_t) out_len;\nstatic mi_lock_t out_buf_lock = MI_LOCK_INITIALIZER;\n\nstatic void mi_cdecl mi_out_buf(const char* msg, void* arg) {\n MI_UNUSED(arg);\n if (msg==NULL) return;\n if (mi_atomic_load_acquire(&out_len)>=MI_MAX_DELAY_OUTPUT) return;\n size_t n = _mi_strlen(msg);\n if (n==0 || n >= MI_MAX_DELAY_OUTPUT) return;\n // copy msg into the buffer\n mi_lock(&out_buf_lock) {\n const size_t start = mi_atomic_add_acq_rel(&out_len, n);\n if (start < MI_MAX_DELAY_OUTPUT) {\n // check bound\n if (start+n >= MI_MAX_DELAY_OUTPUT) {\n n = MI_MAX_DELAY_OUTPUT-start-1;\n }\n _mi_memcpy(&out_buf[start], msg, n);\n }\n }\n}\n\nstatic void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) {\n if (out==NULL) return;\n // claim (if `no_more_buf == true`, no more output will be added after this point)\n mi_lock(&out_buf_lock) {\n size_t count = mi_atomic_add_acq_rel(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1));\n // and output the current contents\n if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT;\n out_buf[count] = 0;\n out(out_buf,arg);\n if (!no_more_buf) {\n out_buf[count] = '\\n'; // if continue with the buffer, insert a newline\n }\n }\n}\n\n\n// Once this module is loaded, switch to this routine\n// which outputs to stderr and the delayed output buffer.\nstatic void mi_cdecl mi_out_buf_stderr(const char* msg, void* arg) {\n mi_out_stderr(msg,arg);\n mi_out_buf(msg,arg);\n}\n\n\n\n// --------------------------------------------------------\n// Default output handler\n// --------------------------------------------------------\n\n// The program should only install a single output handler from a single thread\n// since otherwise the argument and output function may not match.\nstatic _Atomic(void*) mi_out_default; // = // is `mi_output_fun*` (but some platforms don't support atomic function pointers)\nstatic _Atomic(void*) mi_out_arg; // = NULL\n\nstatic mi_output_fun* mi_out_get_default(void** parg) {\n mi_output_fun* const out = (mi_output_fun*)mi_atomic_load_ptr_acquire(void,&mi_out_default);\n if (parg != NULL) { *parg = mi_atomic_load_ptr_acquire(void,&mi_out_arg); }\n return (out == NULL ? &mi_out_buf : out);\n}\n\nvoid mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept {\n mi_atomic_store_ptr_release(void,&mi_out_default, (void*)(out == NULL ? &mi_out_stderr : out)); // stop using the delayed output buffer\n mi_atomic_store_ptr_release(void,&mi_out_arg, arg);\n if (out!=NULL) { mi_out_buf_flush(out,true,arg); } // output all the delayed output now\n}\n\n// add stderr to the delayed output after the module is loaded\nstatic void mi_add_stderr_output(void) {\n mi_assert_internal(mi_out_default == NULL);\n mi_out_buf_flush(&mi_out_stderr, false, NULL); // flush current contents to stderr\n mi_atomic_store_ptr_release(void,&mi_out_default,(void*)&mi_out_buf_stderr); // and add stderr to the delayed output\n mi_atomic_store_ptr_release(void,&mi_out_arg,NULL);\n}\n\n// --------------------------------------------------------\n// Messages, all end up calling `_mi_fputs`.\n// --------------------------------------------------------\nstatic _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors\nstatic _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings\n\n// When overriding malloc, we may recurse into mi_vfprintf if an allocation\n// inside the C runtime causes another message.\n// In some cases (like on macOS) the loader already allocates which\n// calls into mimalloc; if we then access thread locals (like `recurse`)\n// this may crash as the access may call _tlv_bootstrap that tries to\n// (recursively) invoke malloc again to allocate space for the thread local\n// variables on demand. This is why we use a _mi_preloading test on such\n// platforms. However, C code generator may move the initial thread local address\n// load before the `if` and we therefore split it out in a separate function.\nstatic mi_decl_thread bool recurse = false;\n\nstatic mi_decl_noinline bool mi_recurse_enter_prim(void) {\n if (recurse) return false;\n recurse = true;\n return true;\n}\n\nstatic mi_decl_noinline void mi_recurse_exit_prim(void) {\n recurse = false;\n}\n\nstatic bool mi_recurse_enter(void) {\n #if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD)\n if (_mi_preloading()) return false;\n #endif\n return mi_recurse_enter_prim();\n}\n\nstatic void mi_recurse_exit(void) {\n #if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD)\n if (_mi_preloading()) return;\n #endif\n mi_recurse_exit_prim();\n}\n\nvoid _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) {\n if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // TODO: use mi_out_stderr for stderr?\n if (!mi_recurse_enter()) return;\n out = mi_out_get_default(&arg);\n if (prefix != NULL) out(prefix, arg);\n out(message, arg);\n mi_recurse_exit();\n }\n else {\n if (prefix != NULL) out(prefix, arg);\n out(message, arg);\n }\n}\n\n// Define our own limited `fprintf` that avoids memory allocation.\n// We do this using `_mi_vsnprintf` with a limited buffer.\nstatic void mi_vfprintf( mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args ) {\n char buf[512];\n if (fmt==NULL) return;\n if (!mi_recurse_enter()) return;\n _mi_vsnprintf(buf, sizeof(buf)-1, fmt, args);\n mi_recurse_exit();\n _mi_fputs(out,arg,prefix,buf);\n}\n\nvoid _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) {\n va_list args;\n va_start(args,fmt);\n mi_vfprintf(out,arg,NULL,fmt,args);\n va_end(args);\n}\n\nstatic void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) {\n if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) {\n char tprefix[64];\n _mi_snprintf(tprefix, sizeof(tprefix), \"%sthread 0x%tx: \", prefix, (uintptr_t)_mi_thread_id());\n mi_vfprintf(out, arg, tprefix, fmt, args);\n }\n else {\n mi_vfprintf(out, arg, prefix, fmt, args);\n }\n}\n\nvoid _mi_message(const char* fmt, ...) {\n va_list args;\n va_start(args, fmt);\n mi_vfprintf_thread(NULL, NULL, \"mimalloc: \", fmt, args);\n va_end(args);\n}\n\nvoid _mi_trace_message(const char* fmt, ...) {\n if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher\n va_list args;\n va_start(args, fmt);\n mi_vfprintf_thread(NULL, NULL, \"mimalloc: \", fmt, args);\n va_end(args);\n}\n\nvoid _mi_verbose_message(const char* fmt, ...) {\n if (!mi_option_is_enabled(mi_option_verbose)) return;\n va_list args;\n va_start(args,fmt);\n mi_vfprintf(NULL, NULL, \"mimalloc: \", fmt, args);\n va_end(args);\n}\n\nstatic void mi_show_error_message(const char* fmt, va_list args) {\n if (!mi_option_is_enabled(mi_option_verbose)) {\n if (!mi_option_is_enabled(mi_option_show_errors)) return;\n if (mi_max_error_count >= 0 && (long)mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return;\n }\n mi_vfprintf_thread(NULL, NULL, \"mimalloc: error: \", fmt, args);\n}\n\nvoid _mi_warning_message(const char* fmt, ...) {\n if (!mi_option_is_enabled(mi_option_verbose)) {\n if (!mi_option_is_enabled(mi_option_show_errors)) return;\n if (mi_max_warning_count >= 0 && (long)mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return;\n }\n va_list args;\n va_start(args,fmt);\n mi_vfprintf_thread(NULL, NULL, \"mimalloc: warning: \", fmt, args);\n va_end(args);\n}\n\n\n#if MI_DEBUG\nmi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) mi_attr_noexcept {\n _mi_fprintf(NULL, NULL, \"mimalloc: assertion failed: at \\\"%s\\\":%u, %s\\n assertion: \\\"%s\\\"\\n\", fname, line, (func==NULL?\"\":func), assertion);\n abort();\n}\n#endif\n\n// --------------------------------------------------------\n// Errors\n// --------------------------------------------------------\n\nstatic mi_error_fun* volatile mi_error_handler; // = NULL\nstatic _Atomic(void*) mi_error_arg; // = NULL\n\nstatic void mi_error_default(int err) {\n MI_UNUSED(err);\n#if (MI_DEBUG>0)\n if (err==EFAULT) {\n #ifdef _MSC_VER\n __debugbreak();\n #endif\n abort();\n }\n#endif\n#if (MI_SECURE>0)\n if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data)\n abort();\n }\n#endif\n#if defined(MI_XMALLOC)\n if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode\n abort();\n }\n#endif\n}\n\nvoid mi_register_error(mi_error_fun* fun, void* arg) {\n mi_error_handler = fun; // can be NULL\n mi_atomic_store_ptr_release(void,&mi_error_arg, arg);\n}\n\nvoid _mi_error_message(int err, const char* fmt, ...) {\n // show detailed error message\n va_list args;\n va_start(args, fmt);\n mi_show_error_message(fmt, args);\n va_end(args);\n // and call the error handler which may abort (or return normally)\n if (mi_error_handler != NULL) {\n mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg));\n }\n else {\n mi_error_default(err);\n }\n}\n\n// --------------------------------------------------------\n// Initialize options by checking the environment\n// --------------------------------------------------------\n\n// TODO: implement ourselves to reduce dependencies on the C runtime\n#include // strtol\n#include // strstr\n\n\nstatic void mi_option_init(mi_option_desc_t* desc) {\n // Read option value from the environment\n char s[64 + 1];\n char buf[64+1];\n _mi_strlcpy(buf, \"mimalloc_\", sizeof(buf));\n _mi_strlcat(buf, desc->name, sizeof(buf));\n bool found = _mi_getenv(buf, s, sizeof(s));\n if (!found && desc->legacy_name != NULL) {\n _mi_strlcpy(buf, \"mimalloc_\", sizeof(buf));\n _mi_strlcat(buf, desc->legacy_name, sizeof(buf));\n found = _mi_getenv(buf, s, sizeof(s));\n if (found) {\n _mi_warning_message(\"environment option \\\"mimalloc_%s\\\" is deprecated -- use \\\"mimalloc_%s\\\" instead.\\n\", desc->legacy_name, desc->name);\n }\n }\n\n if (found) {\n size_t len = _mi_strnlen(s, sizeof(buf) - 1);\n for (size_t i = 0; i < len; i++) {\n buf[i] = _mi_toupper(s[i]);\n }\n buf[len] = 0;\n if (buf[0] == 0 || _mi_streq(buf,\"1\") || _mi_streq(buf,\"TRUE\") || _mi_streq(buf,\"YES\") || _mi_streq(buf,\"ON\")) {\n desc->value = 1;\n desc->init = INITIALIZED;\n }\n else if (_mi_streq(buf,\"0\") || _mi_streq(buf,\"FALSE\") || _mi_streq(buf,\"NO\") || _mi_streq(buf,\"OFF\")) { \n desc->value = 0;\n desc->init = INITIALIZED;\n }\n else {\n char* end = buf;\n long value = strtol(buf, &end, 10);\n if (mi_option_has_size_in_kib(desc->option)) {\n // this option is interpreted in KiB to prevent overflow of `long` for large allocations\n // (long is 32-bit on 64-bit windows, which allows for 4TiB max.)\n size_t size = (value < 0 ? 0 : (size_t)value);\n bool overflow = false;\n if (*end == 'K') { end++; }\n else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; }\n else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; }\n else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; }\n else { size = (size + MI_KiB - 1) / MI_KiB; }\n if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB\n else if (*end == 'B') { end++; } // Kb, Mb, Gb, Tb\n if (overflow || size > (MI_MAX_ALLOC_SIZE / MI_KiB)) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); }\n value = (size > LONG_MAX ? LONG_MAX : (long)size);\n }\n if (*end == 0) {\n mi_option_set(desc->option, value);\n }\n else {\n // set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose.\n desc->init = DEFAULTED;\n if (desc->option == mi_option_verbose && desc->value == 0) {\n // if the 'mimalloc_verbose' env var has a bogus value we'd never know\n // (since the value defaults to 'off') so in that case briefly enable verbose\n desc->value = 1;\n _mi_warning_message(\"environment option mimalloc_%s has an invalid value.\\n\", desc->name);\n desc->value = 0;\n }\n else {\n _mi_warning_message(\"environment option mimalloc_%s has an invalid value.\\n\", desc->name);\n }\n }\n }\n mi_assert_internal(desc->init != UNINIT);\n }\n else if (!_mi_preloading()) {\n desc->init = DEFAULTED;\n }\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/os.c", "sha256": "1d4da5a55738e6cd324e73772ee663ab4b7f618ff1c54d0d5ed3eb0df575aa3f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\"\n\n#define mi_os_stat_increase(stat,amount) _mi_stat_increase(&_mi_stats_main.stat, amount)\n#define mi_os_stat_decrease(stat,amount) _mi_stat_decrease(&_mi_stats_main.stat, amount)\n#define mi_os_stat_counter_increase(stat,inc) _mi_stat_counter_increase(&_mi_stats_main.stat, inc)\n\n/* -----------------------------------------------------------\n Initialization.\n----------------------------------------------------------- */\n#ifndef MI_DEFAULT_VIRTUAL_ADDRESS_BITS\n#if MI_INTPTR_SIZE < 8\n#define MI_DEFAULT_VIRTUAL_ADDRESS_BITS 32\n#else\n#define MI_DEFAULT_VIRTUAL_ADDRESS_BITS 48\n#endif\n#endif\n\n#ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB\n#if MI_INTPTR_SIZE < 8\n#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB // 4 GiB\n#else\n#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB // 32 GiB\n#endif\n#endif\n\nstatic mi_os_mem_config_t mi_os_mem_config = {\n 4096, // page size\n 0, // large page size (usually 2MiB)\n 4096, // allocation granularity\n MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB,\n MI_DEFAULT_VIRTUAL_ADDRESS_BITS,\n true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems)\n false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)\n true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory)\n};\n\nbool _mi_os_has_overcommit(void) {\n return mi_os_mem_config.has_overcommit;\n}\n\nbool _mi_os_has_virtual_reserve(void) {\n return mi_os_mem_config.has_virtual_reserve;\n}\n\n\n// OS (small) page size\nsize_t _mi_os_page_size(void) {\n return mi_os_mem_config.page_size;\n}\n\n// if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB)\nsize_t _mi_os_large_page_size(void) {\n return (mi_os_mem_config.large_page_size != 0 ? mi_os_mem_config.large_page_size : _mi_os_page_size());\n}\n\nbool _mi_os_canuse_large_page(size_t size, size_t alignment) {\n // if we have access, check the size and alignment requirements\n if (mi_os_mem_config.large_page_size == 0) return false;\n return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0);\n}\n\n// round to a good OS allocation size (bounded by max 12.5% waste)\nsize_t _mi_os_good_alloc_size(size_t size) {\n size_t align_size;\n if (size < 512*MI_KiB) align_size = _mi_os_page_size();\n else if (size < 2*MI_MiB) align_size = 64*MI_KiB;\n else if (size < 8*MI_MiB) align_size = 256*MI_KiB;\n else if (size < 32*MI_MiB) align_size = 1*MI_MiB;\n else align_size = 4*MI_MiB;\n if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow?\n return _mi_align_up(size, align_size);\n}\n\nvoid _mi_os_init(void) {\n _mi_prim_mem_init(&mi_os_mem_config);\n}\n\n\n/* -----------------------------------------------------------\n Util\n-------------------------------------------------------------- */\nbool _mi_os_decommit(void* addr, size_t size);\nbool _mi_os_commit(void* addr, size_t size, bool* is_zero);\n\n\n/* -----------------------------------------------------------\n aligned hinting\n-------------------------------------------------------------- */\n\n// On systems with enough virtual address bits, we can do efficient aligned allocation by using\n// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address\n// space (64TiB) we use this technique. (but see issue #939)\n#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT)\nstatic mi_decl_cache_align _Atomic(uintptr_t)aligned_base;\n\n// Return a MI_SEGMENT_SIZE aligned address that is probably available.\n// If this returns NULL, the OS will determine the address but on some OS's that may not be\n// properly aligned which can be more costly as it needs to be adjusted afterwards.\n// For a size > 1GiB this always returns NULL in order to guarantee good ASLR randomization;\n// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses\n// in the middle of the 2TiB - 6TiB address range (see issue #372))\n\n#define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start\n#define MI_HINT_AREA ((uintptr_t)4 << 40) // upto (2+4) 6TiB (since before win8 there is \"only\" 8TiB available to processes)\n#define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages)\n\nvoid* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)\n{\n if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL;\n if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space\n size = _mi_align_up(size, MI_SEGMENT_SIZE);\n if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096.\n size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.\n \n uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);\n if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize\n uintptr_t init = MI_HINT_BASE;\n #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode\n uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap());\n init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB\n #endif\n uintptr_t expected = hint + size;\n mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);\n hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all\n }\n if (hint%try_alignment != 0) return NULL;\n return (void*)hint;\n}\n#else\nvoid* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {\n MI_UNUSED(try_alignment); MI_UNUSED(size);\n return NULL;\n}\n#endif\n\n/* -----------------------------------------------------------\n Free memory\n-------------------------------------------------------------- */\n\nstatic void mi_os_free_huge_os_pages(void* p, size_t size);\n\nstatic void mi_os_prim_free(void* addr, size_t size, size_t commit_size) {\n mi_assert_internal((size % _mi_os_page_size()) == 0);\n if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr)\n int err = _mi_prim_free(addr, size); // allow size==0 (issue #1041)\n if (err != 0) {\n _mi_warning_message(\"unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\\n\", err, err, size, addr);\n }\n if (commit_size > 0) {\n mi_os_stat_decrease(committed, commit_size);\n }\n mi_os_stat_decrease(reserved, size);\n}\n\nvoid _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid) {\n if (mi_memkind_is_os(memid.memkind)) {\n size_t csize = memid.mem.os.size;\n if (csize==0) { csize = _mi_os_good_alloc_size(size); }\n mi_assert_internal(csize >= size);\n size_t commit_size = (still_committed ? csize : 0);\n void* base = addr;\n // different base? (due to alignment)\n if (memid.mem.os.base != base) {\n mi_assert(memid.mem.os.base <= addr);\n base = memid.mem.os.base;\n const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base;\n if (memid.mem.os.size==0) {\n csize += diff;\n }\n if (still_committed) {\n commit_size -= diff; // the (addr-base) part was already un-committed\n }\n }\n // free it\n if (memid.memkind == MI_MEM_OS_HUGE) {\n mi_assert(memid.is_pinned);\n mi_os_free_huge_os_pages(base, csize);\n }\n else {\n mi_os_prim_free(base, csize, (still_committed ? commit_size : 0));\n }\n }\n else {\n // nothing to do\n mi_assert(memid.memkind < MI_MEM_OS);\n }\n}\n\nvoid _mi_os_free(void* p, size_t size, mi_memid_t memid) {\n _mi_os_free_ex(p, size, true, memid);\n}\n\n\n/* -----------------------------------------------------------\n Primitive allocation from the OS.\n-------------------------------------------------------------- */\n\n// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.\n// Also `hint_addr` is a hint and may be ignored.\nstatic void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {\n mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);\n mi_assert_internal(is_zero != NULL);\n mi_assert_internal(is_large != NULL);\n if (size == 0) return NULL;\n if (!commit) { allow_large = false; }\n if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning\n \n // try to align along large OS page size for larger allocations\n const size_t large_page_size = mi_os_mem_config.large_page_size;\n if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && _mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) {\n try_alignment = large_page_size; \n }\n\n *is_zero = false;\n void* p = NULL;\n int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p);\n if (err != 0) {\n _mi_warning_message(\"unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\\n\", err, err, hint_addr, size, try_alignment, commit, allow_large);\n }\n\n\n\n mi_os_stat_counter_increase(mmap_calls, 1);\n if (p != NULL) {\n mi_os_stat_increase(reserved, size);\n if (commit) {\n mi_os_stat_increase(committed, size);\n // seems needed for asan (or `mimalloc-test-api` fails)\n #ifdef MI_TRACK_ASAN\n if (*is_zero) { mi_track_mem_defined(p,size); }\n else { mi_track_mem_undefined(p,size); }\n #endif\n }\n }\n return p;\n}\n\nstatic void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {\n return mi_os_prim_alloc_at(NULL, size, try_alignment, commit, allow_large, is_large, is_zero);\n}\n\n\n// Primitive aligned allocation from the OS.\n// This function guarantees the allocated memory is aligned.\nstatic void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {\n mi_assert_internal(memid!=NULL);\n mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));\n mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);\n _mi_memzero(memid,sizeof(*memid));\n if (!commit) allow_large = false;\n if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;\n size = _mi_align_up(size, _mi_os_page_size());\n\n // try first with a requested alignment hint (this will usually be aligned directly on Win 10+ or BSD)\n bool os_is_large = false;\n bool os_is_zero = false;\n void* os_base = NULL;\n size_t os_size = size;\n void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, &os_is_large, &os_is_zero);\n if (p == NULL) return NULL;\n\n // aligned already?\n if (((uintptr_t)p % alignment) == 0) {\n os_base = p;\n }\n else {\n // if not aligned, free it, overallocate, and unmap around it\n #if !MI_TRACK_ASAN\n _mi_warning_message(\"unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\\n\", size, p, alignment, commit);\n #endif\n if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0)); }\n if (size >= (SIZE_MAX - alignment)) return NULL; // overflow\n const size_t over_size = size + alignment;\n\n if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block\n // over-allocate uncommitted (virtual) memory\n p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero);\n if (p == NULL) return NULL;\n\n // set p to the aligned part in the full region\n // note: this is dangerous on Windows as VirtualFree needs the actual base pointer\n // this is handled though by having the `base` field in the memid's\n os_base = p; // remember the base\n os_size = over_size;\n p = _mi_align_up_ptr(p, alignment);\n\n // explicitly commit only the aligned part\n if (commit) {\n if (!_mi_os_commit(p, size, NULL)) {\n mi_os_prim_free(os_base, over_size, 0);\n return NULL;\n }\n }\n }\n else { // mmap can free inside an allocation\n // overallocate...\n p = mi_os_prim_alloc(over_size, 1, commit, false, &os_is_large, &os_is_zero);\n if (p == NULL) return NULL;\n\n // and selectively unmap parts around the over-allocated area.\n void* const aligned_p = _mi_align_up_ptr(p, alignment);\n const size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;\n const size_t mid_size = _mi_align_up(size, _mi_os_page_size());\n const size_t post_size = over_size - pre_size - mid_size;\n mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);\n if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0)); }\n if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0)); }\n // we can return the aligned pointer on `mmap` systems\n p = aligned_p;\n os_base = aligned_p; // since we freed the pre part, `*base == p`.\n os_size = mid_size;\n }\n }\n\n mi_assert_internal(p != NULL && os_base != NULL && _mi_is_aligned(p,alignment));\n mi_assert_internal(os_base <= p && size <= os_size);\n *memid = _mi_memid_create_os(os_base,os_size,commit,os_is_zero,os_is_large);\n return p;\n}\n\n\n/* -----------------------------------------------------------\n OS API: alloc and alloc_aligned\n----------------------------------------------------------- */\n\nvoid* _mi_os_alloc(size_t size, mi_memid_t* memid) {\n *memid = _mi_memid_none();\n if (size == 0) return NULL;\n size = _mi_os_good_alloc_size(size);\n bool os_is_large = false;\n bool os_is_zero = false;\n void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero);\n if (p == NULL) return NULL;\n\n *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large);\n mi_assert_internal(memid->mem.os.size >= size);\n mi_assert_internal(memid->initially_committed);\n return p;\n}\n\nvoid* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)\n{\n MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings\n *memid = _mi_memid_none();\n if (size == 0) return NULL;\n size = _mi_os_good_alloc_size(size);\n alignment = _mi_align_up(alignment, _mi_os_page_size());\n\n void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, memid );\n if (p == NULL) return NULL;\n\n mi_assert_internal(memid->mem.os.size >= size);\n mi_assert_internal(_mi_is_aligned(p,alignment));\n if (commit) { mi_assert_internal(memid->initially_committed); }\n return p;\n}\n\n\nmi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_t* memid) {\n if (p==NULL || size==0) return p;\n // ensure committed\n if (!memid->initially_committed) {\n bool is_zero = false;\n if (!_mi_os_commit(p, size, &is_zero)) {\n _mi_os_free(p, size, *memid);\n return NULL;\n }\n memid->initially_committed = true;\n }\n // ensure zero'd\n if (memid->initially_zero) return p;\n _mi_memzero_aligned(p,size);\n memid->initially_zero = true;\n return p;\n}\n\nvoid* _mi_os_zalloc(size_t size, mi_memid_t* memid) {\n void* p = _mi_os_alloc(size,memid);\n return mi_os_ensure_zero(p, size, memid);\n}\n\n/* -----------------------------------------------------------\n OS aligned allocation with an offset. This is used\n for large alignments > MI_BLOCK_ALIGNMENT_MAX. We use a large mimalloc\n page where the object can be aligned at an offset from the start of the segment.\n As we may need to overallocate, we need to free such pointers using `mi_free_aligned`\n to use the actual start of the memory region.\n----------------------------------------------------------- */\n\nvoid* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {\n mi_assert(offset <= MI_SEGMENT_SIZE);\n mi_assert(offset <= size);\n mi_assert((alignment % _mi_os_page_size()) == 0);\n *memid = _mi_memid_none();\n if (offset > MI_SEGMENT_SIZE || offset > size) return NULL;\n if (offset == 0) {\n // regular aligned allocation\n return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);\n }\n else {\n // overallocate to align at an offset\n const size_t extra = _mi_align_up(offset, alignment) - offset;\n if (size >= SIZE_MAX - extra) return NULL; // too large\n const size_t oversize = size + extra;\n void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid);\n if (start == NULL) return NULL;\n\n void* const p = (uint8_t*)start + extra;\n mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment));\n // decommit the overallocation at the start\n if (commit && extra >= _mi_os_page_size()) {\n _mi_os_decommit(start, extra);\n }\n return p;\n }\n}\n\n/* -----------------------------------------------------------\n OS memory API: reset, commit, decommit, protect, unprotect.\n----------------------------------------------------------- */\n\n// OS page align within a given area, either conservative (pages inside the area only),\n// or not (straddling pages outside the area is possible)\nstatic void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {\n mi_assert(addr != NULL && size > 0);\n if (newsize != NULL) *newsize = 0;\n if (size == 0 || addr == NULL) return NULL;\n\n // page align conservatively within the range\n void* start = (conservative ? _mi_align_up_ptr(addr, _mi_os_page_size())\n : _mi_align_down_ptr(addr, _mi_os_page_size()));\n void* end = (conservative ? _mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())\n : _mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));\n ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;\n if (diff <= 0) return NULL;\n\n mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size));\n if (newsize != NULL) *newsize = (size_t)diff;\n return start;\n}\n\nstatic void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {\n return mi_os_page_align_areax(true, addr, size, newsize);\n}\n\nbool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) {\n if (is_zero != NULL) { *is_zero = false; }\n mi_os_stat_counter_increase(commit_calls, 1);\n\n // page align range\n size_t csize;\n void* start = mi_os_page_align_areax(false /* conservative? */, addr, size, &csize);\n if (csize == 0) return true;\n\n // commit\n bool os_is_zero = false;\n int err = _mi_prim_commit(start, csize, &os_is_zero);\n if (err != 0) {\n _mi_warning_message(\"cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\\n\", err, err, start, csize);\n return false;\n }\n if (os_is_zero && is_zero != NULL) {\n *is_zero = true;\n mi_assert_expensive(mi_mem_is_zero(start, csize));\n }\n // note: the following seems required for asan (otherwise `mimalloc-test-stress` fails)\n #ifdef MI_TRACK_ASAN\n if (os_is_zero) { mi_track_mem_defined(start,csize); }\n else { mi_track_mem_undefined(start,csize); }\n #endif\n mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit\n return true;\n}\n\nbool _mi_os_commit(void* addr, size_t size, bool* is_zero) {\n return _mi_os_commit_ex(addr, size, is_zero, size);\n}\n\nstatic bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) {\n mi_assert_internal(needs_recommit!=NULL);\n \n // page align\n size_t csize;\n void* start = mi_os_page_align_area_conservative(addr, size, &csize);\n if (csize == 0) return true;\n\n // decommit\n *needs_recommit = true;\n int err = _mi_prim_decommit(start,csize,needs_recommit);\n if (err != 0) {\n _mi_warning_message(\"cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\\n\", err, err, start, csize);\n }\n else if (*needs_recommit) { \n mi_os_stat_decrease(committed, stat_size);\n }\n mi_assert_internal(err == 0);\n return (err == 0);\n}\n\nbool _mi_os_decommit(void* addr, size_t size) {\n bool needs_recommit;\n return mi_os_decommit_ex(addr, size, &needs_recommit, size);\n}\n\n\n// Signal to the OS that the address range is no longer in use\n// but may be used later again. This will release physical memory\n// pages and reduce swapping while keeping the memory committed.\n// We page align to a conservative area inside the range to reset.\nbool _mi_os_reset(void* addr, size_t size) {\n // page align conservatively within the range\n size_t csize;\n void* start = mi_os_page_align_area_conservative(addr, size, &csize);\n if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr)\n mi_os_stat_counter_increase(reset, csize);\n mi_os_stat_counter_increase(reset_calls, 1);\n\n #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN\n memset(start, 0, csize); // pretend it is eagerly reset\n #endif\n\n int err = _mi_prim_reset(start, csize);\n if (err != 0) {\n _mi_warning_message(\"cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\\n\", err, err, start, csize);\n }\n return (err == 0);\n}\n\n\nvoid _mi_os_reuse( void* addr, size_t size ) {\n // page align conservatively within the range\n size_t csize = 0;\n void* const start = mi_os_page_align_area_conservative(addr, size, &csize);\n if (csize == 0) return;\n const int err = _mi_prim_reuse(start, csize);\n if (err != 0) {\n _mi_warning_message(\"cannot reuse OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\\n\", err, err, start, csize);\n }\n}\n\n// either resets or decommits memory, returns true if the memory needs\n// to be recommitted if it is to be re-used later on.\nbool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size)\n{\n if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed?\n mi_os_stat_counter_increase(purge_calls, 1);\n mi_os_stat_counter_increase(purged, size);\n\n if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit?\n !_mi_preloading()) // don't decommit during preloading (unsafe)\n {\n bool needs_recommit = true;\n mi_os_decommit_ex(p, size, &needs_recommit, stat_size);\n return needs_recommit;\n }\n else {\n if (allow_reset) { // this can sometimes be not allowed if the range is not fully committed\n _mi_os_reset(p, size);\n }\n return false; // needs no recommit\n }\n}\n\n// either resets or decommits memory, returns true if the memory needs\n// to be recommitted if it is to be re-used later on.\nbool _mi_os_purge(void* p, size_t size) {\n return _mi_os_purge_ex(p, size, true, size);\n}\n\n// Protect a region in memory to be not accessible.\nstatic bool mi_os_protectx(void* addr, size_t size, bool protect) {\n // page align conservatively within the range\n size_t csize = 0;\n void* start = mi_os_page_align_area_conservative(addr, size, &csize);\n if (csize == 0) return false;\n /*\n if (_mi_os_is_huge_reserved(addr)) {\n\t _mi_warning_message(\"cannot mprotect memory allocated in huge OS pages\\n\");\n }\n */\n int err = _mi_prim_protect(start,csize,protect);\n if (err != 0) {\n _mi_warning_message(\"cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\\n\", (protect ? \"protect\" : \"unprotect\"), err, err, start, csize);\n }\n return (err == 0);\n}\n\nbool _mi_os_protect(void* addr, size_t size) {\n return mi_os_protectx(addr, size, true);\n}\n\nbool _mi_os_unprotect(void* addr, size_t size) {\n return mi_os_protectx(addr, size, false);\n}\n\n\n\n/* ----------------------------------------------------------------------------\nSupport for allocating huge OS pages (1Gib) that are reserved up-front\nand possibly associated with a specific NUMA node. (use `numa_node>=0`)\n-----------------------------------------------------------------------------*/\n#define MI_HUGE_OS_PAGE_SIZE (MI_GiB)\n\n\n#if (MI_INTPTR_SIZE >= 8)\n// To ensure proper alignment, use our own area for huge OS pages\nstatic mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0\n\n// Claim an aligned address range for huge pages\nstatic uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {\n if (total_size != NULL) *total_size = 0;\n const size_t size = pages * MI_HUGE_OS_PAGE_SIZE;\n\n uintptr_t start = 0;\n uintptr_t end = 0;\n uintptr_t huge_start = mi_atomic_load_relaxed(&mi_huge_start);\n do {\n start = huge_start;\n if (start == 0) {\n // Initialize the start address after the 32TiB area\n start = ((uintptr_t)32 << 40); // 32TiB virtual start address\n #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode\n uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap());\n start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB\n #endif\n }\n end = start + size;\n mi_assert_internal(end % MI_SEGMENT_SIZE == 0);\n } while (!mi_atomic_cas_strong_acq_rel(&mi_huge_start, &huge_start, end));\n\n if (total_size != NULL) *total_size = size;\n return (uint8_t*)start;\n}\n#else\nstatic uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {\n MI_UNUSED(pages);\n if (total_size != NULL) *total_size = 0;\n return NULL;\n}\n#endif\n\n// Allocate MI_SEGMENT_SIZE aligned huge pages\nvoid* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {\n *memid = _mi_memid_none();\n if (psize != NULL) *psize = 0;\n if (pages_reserved != NULL) *pages_reserved = 0;\n size_t size = 0;\n uint8_t* const start = mi_os_claim_huge_pages(pages, &size);\n if (start == NULL) return NULL; // or 32-bit systems\n\n // Allocate one page at the time but try to place them contiguously\n // We allocate one page at the time to be able to abort if it takes too long\n // or to at least allocate as many as available on the system.\n mi_msecs_t start_t = _mi_clock_start();\n size_t page = 0;\n bool all_zero = true;\n while (page < pages) {\n // allocate a page\n bool is_zero = false;\n void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE);\n void* p = NULL;\n int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p);\n if (!is_zero) { all_zero = false; }\n if (err != 0) {\n _mi_warning_message(\"unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\\n\", err, err, addr, MI_HUGE_OS_PAGE_SIZE);\n break;\n }\n\n // Did we succeed at a contiguous address?\n if (p != addr) {\n // no success, issue a warning and break\n if (p != NULL) {\n _mi_warning_message(\"could not allocate contiguous huge OS page %zu at %p\\n\", page, addr);\n mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE);\n }\n break;\n }\n\n // success, record it\n page++; // increase before timeout check (see issue #711)\n mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE);\n mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE);\n\n // check for timeout\n if (max_msecs > 0) {\n mi_msecs_t elapsed = _mi_clock_end(start_t);\n if (page >= 1) {\n mi_msecs_t estimate = ((elapsed / (page+1)) * pages);\n if (estimate > 2*max_msecs) { // seems like we are going to timeout, break\n elapsed = max_msecs + 1;\n }\n }\n if (elapsed > max_msecs) {\n _mi_warning_message(\"huge OS page allocation timed out (after allocating %zu page(s))\\n\", page);\n break;\n }\n }\n }\n const size_t allocated = page * MI_HUGE_OS_PAGE_SIZE;\n mi_assert_internal(allocated <= size);\n if (pages_reserved != NULL) { *pages_reserved = page; }\n if (psize != NULL) { *psize = allocated; }\n if (page != 0) {\n mi_assert(start != NULL);\n *memid = _mi_memid_create_os(start, allocated, true /* is committed */, all_zero, true /* is_large */);\n memid->memkind = MI_MEM_OS_HUGE;\n mi_assert(memid->is_pinned);\n #ifdef MI_TRACK_ASAN\n if (all_zero) { mi_track_mem_defined(start,allocated); }\n #endif\n }\n return (page == 0 ? NULL : start);\n}\n\n// free every huge page in a range individually (as we allocated per page)\n// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.\nstatic void mi_os_free_huge_os_pages(void* p, size_t size) {\n if (p==NULL || size==0) return;\n uint8_t* base = (uint8_t*)p;\n while (size >= MI_HUGE_OS_PAGE_SIZE) {\n mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE);\n size -= MI_HUGE_OS_PAGE_SIZE;\n base += MI_HUGE_OS_PAGE_SIZE;\n }\n}\n\n\n/* ----------------------------------------------------------------------------\nSupport NUMA aware allocation\n-----------------------------------------------------------------------------*/\n\nstatic _Atomic(size_t) mi_numa_node_count; // = 0 // cache the node count\n\nint _mi_os_numa_node_count(void) {\n size_t count = mi_atomic_load_acquire(&mi_numa_node_count);\n if mi_unlikely(count == 0) {\n long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly?\n if (ncount > 0 && ncount < INT_MAX) {\n count = (size_t)ncount;\n }\n else {\n const size_t n = _mi_prim_numa_node_count(); // or detect dynamically\n if (n == 0 || n > INT_MAX) { count = 1; }\n else { count = n; }\n }\n mi_atomic_store_release(&mi_numa_node_count, count); // save it\n _mi_verbose_message(\"using %zd numa regions\\n\", count);\n }\n mi_assert_internal(count > 0 && count <= INT_MAX);\n return (int)count;\n}\n\nstatic int mi_os_numa_node_get(void) {\n int numa_count = _mi_os_numa_node_count();\n if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0\n // never more than the node count and >= 0\n const size_t n = _mi_prim_numa_node();\n int numa_node = (n < INT_MAX ? (int)n : 0);\n if (numa_node >= numa_count) { numa_node = numa_node % numa_count; }\n return numa_node;\n}\n\nint _mi_os_numa_node(void) {\n if mi_likely(mi_atomic_load_relaxed(&mi_numa_node_count) == 1) {\n return 0;\n }\n else {\n return mi_os_numa_node_get();\n }\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/page-queue.c", "sha256": "0e96bf06a55dbcf16008247025d0c9110fca6a41dd9b2c6c85d3d849af337377", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/*----------------------------------------------------------------------------\nCopyright (c) 2018-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* -----------------------------------------------------------\n Definition of page queues for each block size\n----------------------------------------------------------- */\n\n#ifndef MI_IN_PAGE_C\n#error \"this file should be included from 'page.c'\"\n// include to help an IDE\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#endif\n\n/* -----------------------------------------------------------\n Minimal alignment in machine words (i.e. `sizeof(void*)`)\n----------------------------------------------------------- */\n\n#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE)\n #error \"define alignment for more than 4x word size for this platform\"\n#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE)\n #define MI_ALIGN4W // 4 machine words minimal alignment\n#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE)\n #define MI_ALIGN2W // 2 machine words minimal alignment\n#else\n // ok, default alignment is 1 word\n#endif\n\n\n/* -----------------------------------------------------------\n Queue query\n----------------------------------------------------------- */\n\n\nstatic inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) {\n return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+sizeof(uintptr_t)));\n}\n\nstatic inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) {\n return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+(2*sizeof(uintptr_t))));\n}\n\nstatic inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) {\n return (pq->block_size > MI_MEDIUM_OBJ_SIZE_MAX);\n}\n\n/* -----------------------------------------------------------\n Bins\n----------------------------------------------------------- */\n\n// Return the bin for a given field size.\n// Returns MI_BIN_HUGE if the size is too large.\n// We use `wsize` for the size in \"machine word sizes\",\n// i.e. byte size == `wsize*sizeof(void*)`.\nstatic inline size_t mi_bin(size_t size) {\n size_t wsize = _mi_wsize_from_size(size);\n#if defined(MI_ALIGN4W)\n if mi_likely(wsize <= 4) {\n return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes\n }\n#elif defined(MI_ALIGN2W)\n if mi_likely(wsize <= 8) {\n return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes\n }\n#else\n if mi_likely(wsize <= 8) {\n return (wsize == 0 ? 1 : wsize);\n }\n#endif\n else if mi_unlikely(wsize > MI_MEDIUM_OBJ_WSIZE_MAX) {\n return MI_BIN_HUGE;\n }\n else {\n #if defined(MI_ALIGN4W)\n if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes\n #endif\n wsize--;\n // find the highest bit\n const size_t b = (MI_SIZE_BITS - 1 - mi_clz(wsize)); // note: wsize != 0\n // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation).\n // - adjust with 3 because we use do not round the first 8 sizes\n // which each get an exact bin\n const size_t bin = ((b << 2) + ((wsize >> (b - 2)) & 0x03)) - 3;\n mi_assert_internal(bin > 0 && bin < MI_BIN_HUGE);\n return bin;\n }\n}\n\n\n\n/* -----------------------------------------------------------\n Queue of pages with free blocks\n----------------------------------------------------------- */\n\nsize_t _mi_bin(size_t size) {\n return mi_bin(size);\n}\n\nsize_t _mi_bin_size(size_t bin) {\n return _mi_heap_empty.pages[bin].block_size;\n}\n\n// Good size for allocation\nsize_t mi_good_size(size_t size) mi_attr_noexcept {\n if (size <= MI_MEDIUM_OBJ_SIZE_MAX) {\n return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE));\n }\n else if (size <= MI_MAX_ALLOC_SIZE) {\n return _mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size());\n }\n else {\n return size;\n }\n}\n\n#if (MI_DEBUG>1)\nstatic bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_page_t* list = queue->first;\n while (list != NULL) {\n mi_assert_internal(list->next == NULL || list->next->prev == list);\n mi_assert_internal(list->prev == NULL || list->prev->next == list);\n if (list == page) break;\n list = list->next;\n }\n return (list == page);\n}\n\n#endif\n\n#if (MI_DEBUG>1)\nstatic bool mi_heap_contains_queue(const mi_heap_t* heap, const mi_page_queue_t* pq) {\n return (pq >= &heap->pages[0] && pq <= &heap->pages[MI_BIN_FULL]);\n}\n#endif\n\nstatic inline bool mi_page_is_large_or_huge(const mi_page_t* page) {\n return (mi_page_block_size(page) > MI_MEDIUM_OBJ_SIZE_MAX || mi_page_is_huge(page));\n}\n\nstatic size_t mi_page_bin(const mi_page_t* page) {\n const size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page))));\n mi_assert_internal(bin <= MI_BIN_FULL);\n return bin;\n}\n\n// returns the page bin without using MI_BIN_FULL for statistics\nsize_t _mi_page_stats_bin(const mi_page_t* page) {\n const size_t bin = (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)));\n mi_assert_internal(bin <= MI_BIN_HUGE);\n return bin;\n}\n\nstatic mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) {\n mi_assert_internal(heap!=NULL);\n const size_t bin = mi_page_bin(page);\n mi_page_queue_t* pq = &heap->pages[bin];\n mi_assert_internal((mi_page_block_size(page) == pq->block_size) ||\n (mi_page_is_large_or_huge(page) && mi_page_queue_is_huge(pq)) ||\n (mi_page_is_in_full(page) && mi_page_queue_is_full(pq)));\n return pq;\n}\n\nstatic mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) {\n mi_heap_t* heap = mi_page_heap(page);\n mi_page_queue_t* pq = mi_heap_page_queue_of(heap, page);\n mi_assert_expensive(mi_page_queue_contains(pq, page));\n return pq;\n}\n\n// The current small page array is for efficiency and for each\n// small size (up to 256) it points directly to the page for that\n// size without having to compute the bin. This means when the\n// current free page queue is updated for a small bin, we need to update a\n// range of entries in `_mi_page_small_free`.\nstatic inline void mi_heap_queue_first_update(mi_heap_t* heap, const mi_page_queue_t* pq) {\n mi_assert_internal(mi_heap_contains_queue(heap,pq));\n const size_t size = pq->block_size;\n if (size > MI_SMALL_SIZE_MAX) return;\n\n mi_page_t* page = pq->first;\n if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty;\n\n // find index in the right direct page array\n const size_t idx = _mi_wsize_from_size(size);\n mi_page_t** const pages_free = heap->pages_free_direct;\n if (pages_free[idx] == page) return; // already set\n\n // find start slot\n size_t start;\n if (idx<=1) {\n start = 0;\n }\n else {\n // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped\n mi_assert_internal(pq > &heap->pages[0]); // since idx > 1 \n size_t bin = mi_bin(size);\n const mi_page_queue_t* prev = pq - 1;\n while( bin == mi_bin(prev->block_size) && prev > &heap->pages[0]) {\n prev--;\n }\n start = 1 + _mi_wsize_from_size(prev->block_size);\n if (start > idx) start = idx;\n }\n\n // set size range to the right page\n mi_assert(start <= idx);\n for (size_t sz = start; sz <= idx; sz++) {\n pages_free[sz] = page;\n }\n}\n\n/*\nstatic bool mi_page_queue_is_empty(mi_page_queue_t* queue) {\n return (queue->first == NULL);\n}\n*/\n\nstatic void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_assert_expensive(mi_page_queue_contains(queue, page));\n mi_assert_internal(mi_page_block_size(page) == queue->block_size ||\n (mi_page_is_large_or_huge(page) && mi_page_queue_is_huge(queue)) ||\n (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));\n mi_heap_t* heap = mi_page_heap(page);\n\n if (page->prev != NULL) page->prev->next = page->next;\n if (page->next != NULL) page->next->prev = page->prev;\n if (page == queue->last) queue->last = page->prev;\n if (page == queue->first) {\n queue->first = page->next;\n // update first\n mi_assert_internal(mi_heap_contains_queue(heap, queue));\n mi_heap_queue_first_update(heap,queue);\n }\n heap->page_count--;\n page->next = NULL;\n page->prev = NULL;\n // mi_atomic_store_ptr_release(mi_atomic_cast(void*, &page->heap), NULL);\n mi_page_set_in_full(page,false);\n}\n\n\n#if MI_DEBUG >= 3\nstatic bool mi_page_queue_is_consistent(const mi_page_queue_t* queue);\n#endif\n\nstatic void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_t* page) {\n mi_assert_internal(mi_page_heap(page) == heap);\n mi_assert_internal(!mi_page_queue_contains(queue, page));\n #if MI_HUGE_PAGE_ABANDON\n mi_assert_internal(_mi_page_segment(page)->kind != MI_SEGMENT_HUGE);\n #endif\n mi_assert_internal(mi_page_block_size(page) == queue->block_size ||\n (mi_page_is_large_or_huge(page) && mi_page_queue_is_huge(queue)) ||\n (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));\n\n mi_page_set_in_full(page, mi_page_queue_is_full(queue));\n // mi_atomic_store_ptr_release(mi_atomic_cast(void*, &page->heap), heap);\n page->next = queue->first;\n page->prev = NULL;\n if (queue->first != NULL) {\n mi_assert_internal(queue->first->prev == NULL);\n queue->first->prev = page;\n queue->first = page;\n }\n else {\n queue->first = queue->last = page;\n }\n\n // update direct\n mi_heap_queue_first_update(heap, queue);\n heap->page_count++;\n\n // [specbot Q-NEW-1] first/last must be both null or both non-null (L1, O(1))\n mi_assert_internal((queue->first == NULL) == (queue->last == NULL));\n // [specbot Q-NEW-2] last element must have no next (L1, O(1))\n mi_assert_internal(queue->last == NULL || queue->last->next == NULL);\n mi_assert_expensive(mi_page_queue_is_consistent(queue));\n}\n\n#if MI_DEBUG >= 3\n// [specbot Q-NEW-3] Verify doubly-linked list forward/backward consistency (L2, O(n))\nstatic bool mi_page_queue_is_consistent(const mi_page_queue_t* queue) {\n if (queue->first == NULL) return (queue->last == NULL);\n if (queue->last == NULL) return false;\n // forward: first -> ... -> last\n const mi_page_t* p = queue->first;\n const mi_page_t* prev = NULL;\n size_t count = 0;\n while (p != NULL) {\n mi_assert_internal(p->prev == prev);\n prev = p;\n p = p->next;\n count++;\n if (count > 100000) return false; // cycle guard\n }\n mi_assert_internal(prev == queue->last);\n return true;\n}\n#endif\n\nstatic void mi_page_queue_move_to_front(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_t* page) {\n mi_assert_internal(mi_page_heap(page) == heap);\n mi_assert_internal(mi_page_queue_contains(queue, page));\n if (queue->first == page) return;\n mi_page_queue_remove(queue, page);\n mi_page_queue_push(heap, queue, page);\n mi_assert_internal(queue->first == page);\n}\n\nstatic void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_assert_expensive(mi_page_queue_contains(from, page));\n mi_assert_expensive(!mi_page_queue_contains(to, page));\n const size_t bsize = mi_page_block_size(page);\n MI_UNUSED(bsize);\n mi_assert_internal((bsize == to->block_size && bsize == from->block_size) ||\n (bsize == to->block_size && mi_page_queue_is_full(from)) ||\n (bsize == from->block_size && mi_page_queue_is_full(to)) ||\n (mi_page_is_large_or_huge(page) && mi_page_queue_is_huge(to)) ||\n (mi_page_is_large_or_huge(page) && mi_page_queue_is_full(to)));\n\n mi_heap_t* heap = mi_page_heap(page);\n\n // delete from `from`\n if (page->prev != NULL) page->prev->next = page->next;\n if (page->next != NULL) page->next->prev = page->prev;\n if (page == from->last) from->last = page->prev;\n if (page == from->first) {\n from->first = page->next;\n // update first\n mi_assert_internal(mi_heap_contains_queue(heap, from));\n mi_heap_queue_first_update(heap, from);\n }\n\n // insert into `to`\n if (enqueue_at_end) {\n // enqueue at the end\n page->prev = to->last;\n page->next = NULL;\n if (to->last != NULL) {\n mi_assert_internal(heap == mi_page_heap(to->last));\n to->last->next = page;\n to->last = page;\n }\n else {\n to->first = page;\n to->last = page;\n mi_heap_queue_first_update(heap, to);\n }\n }\n else {\n if (to->first != NULL) {\n // enqueue at 2nd place\n mi_assert_internal(heap == mi_page_heap(to->first));\n mi_page_t* next = to->first->next;\n page->prev = to->first;\n page->next = next;\n to->first->next = page;\n if (next != NULL) {\n next->prev = page;\n }\n else {\n to->last = page;\n }\n }\n else {\n // enqueue at the head (singleton list)\n page->prev = NULL;\n page->next = NULL;\n to->first = page;\n to->last = page;\n mi_heap_queue_first_update(heap, to);\n }\n }\n\n mi_page_set_in_full(page, mi_page_queue_is_full(to));\n}\n\nstatic void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {\n mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end */, page);\n}\n\nstatic void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {\n // note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`)\n mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page);\n}\n\n// Only called from `mi_heap_absorb`.\nsize_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append) {\n mi_assert_internal(mi_heap_contains_queue(heap,pq));\n mi_assert_internal(pq->block_size == append->block_size);\n\n if (append->first==NULL) return 0;\n\n // set append pages to new heap and count\n size_t count = 0;\n for (mi_page_t* page = append->first; page != NULL; page = page->next) {\n // inline `mi_page_set_heap` to avoid wrong assertion during absorption;\n // in this case it is ok to be delayed freeing since both \"to\" and \"from\" heap are still alive.\n mi_atomic_store_release(&page->xheap, (uintptr_t)heap);\n // set the flag to delayed free (not overriding NEVER_DELAYED_FREE) which has as a\n // side effect that it spins until any DELAYED_FREEING is finished. This ensures\n // that after appending only the new heap will be used for delayed free operations.\n _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, false);\n count++;\n }\n\n if (pq->last==NULL) {\n // take over afresh\n mi_assert_internal(pq->first==NULL);\n pq->first = append->first;\n pq->last = append->last;\n mi_heap_queue_first_update(heap, pq);\n }\n else {\n // append to end\n mi_assert_internal(pq->last!=NULL);\n mi_assert_internal(append->first!=NULL);\n pq->last->next = append->first;\n append->first->prev = pq->last;\n pq->last = append->last;\n }\n return count;\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/page.c", "sha256": "057eeb7d11821b41ff5bf63fa280795ff15b67403d50221918817ed1180bc485", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/*----------------------------------------------------------------------------\nCopyright (c) 2018-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* -----------------------------------------------------------\n The core of the allocator. Every segment contains\n pages of a certain block size. The main function\n exported is `mi_malloc_generic`.\n----------------------------------------------------------- */\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n\n/* -----------------------------------------------------------\n Definition of page queues for each block size\n----------------------------------------------------------- */\n\n#define MI_IN_PAGE_C\n#include \"page-queue.c\"\n#undef MI_IN_PAGE_C\n\n\n/* -----------------------------------------------------------\n Page helpers\n----------------------------------------------------------- */\n\n// Index a block in a page\nstatic inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t block_size, size_t i) {\n MI_UNUSED(page);\n mi_assert_internal(page != NULL);\n mi_assert_internal(i <= page->reserved);\n return (mi_block_t*)((uint8_t*)page_start + (i * block_size));\n}\n\nstatic void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t size, mi_tld_t* tld);\nstatic bool mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld);\n\n#if (MI_DEBUG>=3)\nstatic size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) {\n size_t count = 0;\n while (head != NULL) {\n mi_assert_internal(page == _mi_ptr_page(head));\n count++;\n head = mi_block_next(page, head);\n }\n return count;\n}\n\n/*\n// Start of the page available memory\nstatic inline uint8_t* mi_page_area(const mi_page_t* page) {\n return _mi_page_start(_mi_page_segment(page), page, NULL);\n}\n*/\n\nstatic bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) {\n size_t psize;\n uint8_t* page_area = _mi_segment_page_start(_mi_page_segment(page), page, &psize);\n mi_block_t* start = (mi_block_t*)page_area;\n mi_block_t* end = (mi_block_t*)(page_area + psize);\n while(p != NULL) {\n if (p < start || p >= end) return false;\n p = mi_block_next(page, p);\n }\n#if MI_DEBUG>3 // generally too expensive to check this\n if (page->free_is_zero) {\n const size_t ubsize = mi_page_usable_block_size(page);\n for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) {\n mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t)));\n }\n }\n#endif\n return true;\n}\n\nstatic bool mi_page_is_valid_init(mi_page_t* page) {\n mi_assert_internal(mi_page_block_size(page) > 0);\n \n mi_assert_internal(page->used <= page->capacity);\n mi_assert_internal(page->capacity <= page->reserved);\n\n // [specbot P-NEW-1] block_size_shift must be consistent with block_size (L1, O(1))\n mi_assert_internal(page->block_size_shift == 0 || (mi_page_block_size(page) == ((size_t)1 << page->block_size_shift)));\n // [specbot P-NEW-2] capacity must be nonzero when blocks are in use (L1, O(1))\n mi_assert_internal(page->used == 0 || page->capacity > 0);\n // [specbot P-NEW-4] page_start must be non-null when capacity > 0 (L1, O(1))\n mi_assert_internal(page->capacity == 0 || mi_page_start(page) != NULL);\n\n uint8_t* start = mi_page_start(page);\n mi_assert_internal(start == _mi_segment_page_start(_mi_page_segment(page), page, NULL));\n mi_assert_internal(page->is_huge == (_mi_page_segment(page)->kind == MI_SEGMENT_HUGE));\n //mi_assert_internal(start + page->capacity*page->block_size == page->top);\n\n mi_assert_internal(mi_page_list_is_valid(page,page->free));\n mi_assert_internal(mi_page_list_is_valid(page,page->local_free));\n\n // [specbot P-NEW-7] All free list blocks are aligned to block_size (L2, O(n))\n {\n size_t bsize = mi_page_block_size(page);\n uint8_t* pstart = mi_page_start(page);\n for (mi_block_t* b = page->free; b != NULL; b = mi_block_next(page, b)) {\n mi_assert_internal(((uint8_t*)b - pstart) % bsize == 0);\n }\n for (mi_block_t* b = page->local_free; b != NULL; b = mi_block_next(page, b)) {\n mi_assert_internal(((uint8_t*)b - pstart) % bsize == 0);\n }\n }\n\n #if MI_DEBUG>3 // generally too expensive to check this\n if (page->free_is_zero) {\n const size_t ubsize = mi_page_usable_block_size(page);\n for(mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) {\n mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t)));\n }\n }\n #endif\n\n #if !MI_TRACK_ENABLED && !MI_TSAN\n mi_block_t* tfree = mi_page_thread_free(page);\n mi_assert_internal(mi_page_list_is_valid(page, tfree));\n //size_t tfree_count = mi_page_list_count(page, tfree);\n //mi_assert_internal(tfree_count <= page->thread_freed + 1);\n #endif\n\n size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free);\n mi_assert_internal(page->used + free_count == page->capacity);\n\n return true;\n}\n\nextern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called?\n\nbool _mi_page_is_valid(mi_page_t* page) {\n mi_assert_internal(mi_page_is_valid_init(page));\n #if MI_SECURE\n mi_assert_internal(page->keys[0] != 0);\n #endif\n if (mi_page_heap(page)!=NULL) {\n mi_segment_t* segment = _mi_page_segment(page);\n\n // [specbot P-NEW-6] heap_tag must match owning heap's tag (L1, O(1))\n mi_assert_internal(page->heap_tag == mi_page_heap(page)->tag);\n\n mi_assert_internal(!_mi_process_is_initialized || segment->thread_id==0 || segment->thread_id == mi_page_heap(page)->thread_id);\n #if MI_HUGE_PAGE_ABANDON\n if (segment->kind != MI_SEGMENT_HUGE)\n #endif\n {\n mi_page_queue_t* pq = mi_page_queue_of(page);\n mi_assert_internal(mi_page_queue_contains(pq, page));\n mi_assert_internal(pq->block_size==mi_page_block_size(page) || mi_page_block_size(page) > MI_MEDIUM_OBJ_SIZE_MAX || mi_page_is_in_full(page));\n mi_assert_internal(mi_heap_contains_queue(mi_page_heap(page),pq));\n }\n }\n return true;\n}\n#endif\n\nvoid _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never) {\n while (!_mi_page_try_use_delayed_free(page, delay, override_never)) {\n mi_atomic_yield();\n }\n}\n\nbool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never) {\n mi_thread_free_t tfreex;\n mi_delayed_t old_delay;\n mi_thread_free_t tfree;\n size_t yield_count = 0;\n do {\n tfree = mi_atomic_load_acquire(&page->xthread_free); // note: must acquire as we can break/repeat this loop and not do a CAS;\n tfreex = mi_tf_set_delayed(tfree, delay);\n old_delay = mi_tf_delayed(tfree);\n if mi_unlikely(old_delay == MI_DELAYED_FREEING) {\n if (yield_count >= 4) return false; // give up after 4 tries\n yield_count++;\n mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done.\n // tfree = mi_tf_set_delayed(tfree, MI_NO_DELAYED_FREE); // will cause CAS to busy fail\n }\n else if (delay == old_delay) {\n break; // avoid atomic operation if already equal\n }\n else if (!override_never && old_delay == MI_NEVER_DELAYED_FREE) {\n break; // leave never-delayed flag set\n }\n } while ((old_delay == MI_DELAYED_FREEING) ||\n !mi_atomic_cas_weak_release(&page->xthread_free, &tfree, tfreex));\n\n return true; // success\n}\n\n/* -----------------------------------------------------------\n Page collect the `local_free` and `thread_free` lists\n----------------------------------------------------------- */\n\n// Collect the local `thread_free` list using an atomic exchange.\n// Note: The exchange must be done atomically as this is used right after\n// moving to the full list in `mi_page_collect_ex` and we need to\n// ensure that there was no race where the page became unfull just before the move.\nstatic void _mi_page_thread_free_collect(mi_page_t* page)\n{\n mi_block_t* head;\n mi_thread_free_t tfreex;\n mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free);\n do {\n head = mi_tf_block(tfree);\n tfreex = mi_tf_set_block(tfree,NULL);\n } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex));\n\n // return if the list is empty\n if (head == NULL) return;\n\n // find the tail -- also to get a proper count (without data races)\n size_t max_count = page->capacity; // cannot collect more than capacity\n size_t count = 1;\n mi_block_t* tail = head;\n mi_block_t* next;\n while ((next = mi_block_next(page,tail)) != NULL && count <= max_count) {\n count++;\n tail = next;\n }\n // if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free)\n if mi_unlikely(count > max_count) {\n _mi_error_message(EFAULT, \"corrupted thread-free list\\n\");\n return; // the thread-free items cannot be freed\n }\n // if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list)\n else if mi_unlikely(count > page->used) {\n _mi_error_message(EFAULT, \"corrupted meta-data in thread-free list\\n\");\n return; // the thread-free items cannot be freed\n }\n\n // and append the current local free list\n mi_block_set_next(page,tail, page->local_free);\n page->local_free = head;\n\n // update counts now\n mi_assert_internal(count <= UINT16_MAX);\n mi_assert_internal(page->used >= (uint16_t)count);\n page->used = page->used - (uint16_t)count;\n}\n\nvoid _mi_page_free_collect(mi_page_t* page, bool force) {\n mi_assert_internal(page!=NULL);\n\n // collect the thread free list\n if (force || mi_page_thread_free(page) != NULL) { // quick test to avoid an atomic operation\n _mi_page_thread_free_collect(page);\n }\n\n // and the local free list\n if (page->local_free != NULL) {\n if mi_likely(page->free == NULL) {\n // usual case\n page->free = page->local_free;\n page->local_free = NULL;\n page->free_is_zero = false;\n }\n else if (force) {\n // append -- only on shutdown (force) as this is a linear operation\n mi_block_t* tail = page->local_free;\n mi_block_t* next;\n while ((next = mi_block_next(page, tail)) != NULL) {\n tail = next;\n }\n mi_block_set_next(page, tail, page->free);\n page->free = page->local_free;\n page->local_free = NULL;\n page->free_is_zero = false;\n }\n }\n\n mi_assert_internal(!force || page->local_free == NULL);\n}\n\n\n\n/* -----------------------------------------------------------\n Page fresh and retire\n----------------------------------------------------------- */\n\n// called from segments when reclaiming abandoned pages\nvoid _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) {\n mi_assert_expensive(mi_page_is_valid_init(page));\n\n mi_assert_internal(mi_page_heap(page) == heap);\n mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE);\n #if MI_HUGE_PAGE_ABANDON\n mi_assert_internal(_mi_page_segment(page)->kind != MI_SEGMENT_HUGE);\n #endif\n\n // TODO: push on full queue immediately if it is full?\n mi_page_queue_t* pq = mi_page_queue(heap, mi_page_block_size(page));\n mi_page_queue_push(heap, pq, page);\n mi_assert_expensive(_mi_page_is_valid(page));\n}\n\n// allocate a fresh page from a segment\nstatic mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size_t block_size, size_t page_alignment) {\n #if !MI_HUGE_PAGE_ABANDON\n mi_assert_internal(pq != NULL);\n mi_assert_internal(mi_heap_contains_queue(heap, pq));\n mi_assert_internal(page_alignment > 0 || block_size > MI_MEDIUM_OBJ_SIZE_MAX || block_size == pq->block_size);\n #endif\n mi_page_t* page = _mi_segment_page_alloc(heap, block_size, page_alignment, &heap->tld->segments);\n if (page == NULL) {\n // this may be out-of-memory, or an abandoned page was reclaimed (and in our queue)\n return NULL;\n }\n #if MI_HUGE_PAGE_ABANDON\n mi_assert_internal(pq==NULL || _mi_page_segment(page)->page_kind != MI_PAGE_HUGE);\n #endif\n mi_assert_internal(page_alignment >0 || block_size > MI_MEDIUM_OBJ_SIZE_MAX || _mi_page_segment(page)->kind != MI_SEGMENT_HUGE);\n mi_assert_internal(pq!=NULL || mi_page_block_size(page) >= block_size);\n // a fresh page was found, initialize it\n const size_t full_block_size = (pq == NULL || mi_page_is_huge(page) ? mi_page_block_size(page) : block_size); // see also: mi_segment_huge_page_alloc\n mi_assert_internal(full_block_size >= block_size);\n mi_page_init(heap, page, full_block_size, heap->tld);\n mi_heap_stat_increase(heap, pages, 1);\n mi_heap_stat_increase(heap, page_bins[_mi_page_stats_bin(page)], 1);\n if (pq != NULL) { mi_page_queue_push(heap, pq, page); }\n mi_assert_expensive(_mi_page_is_valid(page));\n return page;\n}\n\n// Get a fresh page to use\nstatic mi_page_t* mi_page_fresh(mi_heap_t* heap, mi_page_queue_t* pq) {\n mi_assert_internal(mi_heap_contains_queue(heap, pq));\n mi_page_t* page = mi_page_fresh_alloc(heap, pq, pq->block_size, 0);\n if (page==NULL) return NULL;\n mi_assert_internal(pq->block_size==mi_page_block_size(page));\n mi_assert_internal(pq==mi_page_queue(heap, mi_page_block_size(page)));\n return page;\n}\n\n/* -----------------------------------------------------------\n Do any delayed frees\n (put there by other threads if they deallocated in a full page)\n----------------------------------------------------------- */\nvoid _mi_heap_delayed_free_all(mi_heap_t* heap) {\n while (!_mi_heap_delayed_free_partial(heap)) {\n mi_atomic_yield();\n }\n}\n\n// returns true if all delayed frees were processed\nbool _mi_heap_delayed_free_partial(mi_heap_t* heap) {\n // take over the list (note: no atomic exchange since it is often NULL)\n mi_block_t* block = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free);\n while (block != NULL && !mi_atomic_cas_ptr_weak_acq_rel(mi_block_t, &heap->thread_delayed_free, &block, NULL)) { /* nothing */ };\n bool all_freed = true;\n\n // and free them all\n while(block != NULL) {\n mi_block_t* next = mi_block_nextx(heap,block, heap->keys);\n // use internal free instead of regular one to keep stats etc correct\n if (!_mi_free_delayed_block(block)) {\n // we might already start delayed freeing while another thread has not yet\n // reset the delayed_freeing flag; in that case delay it further by reinserting the current block\n // into the delayed free list\n all_freed = false;\n mi_block_t* dfree = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free);\n do {\n mi_block_set_nextx(heap, block, dfree, heap->keys);\n } while (!mi_atomic_cas_ptr_weak_release(mi_block_t,&heap->thread_delayed_free, &dfree, block));\n }\n block = next;\n }\n return all_freed;\n}\n\n/* -----------------------------------------------------------\n Unfull, abandon, free and retire\n----------------------------------------------------------- */\n\n// Move a page from the full list back to a regular list\nvoid _mi_page_unfull(mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_assert_expensive(_mi_page_is_valid(page));\n mi_assert_internal(mi_page_is_in_full(page));\n if (!mi_page_is_in_full(page)) return;\n\n mi_heap_t* heap = mi_page_heap(page);\n mi_page_queue_t* pqfull = &heap->pages[MI_BIN_FULL];\n mi_page_set_in_full(page, false); // to get the right queue\n mi_page_queue_t* pq = mi_heap_page_queue_of(heap, page);\n mi_page_set_in_full(page, true);\n mi_page_queue_enqueue_from_full(pq, pqfull, page);\n}\n\nstatic void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {\n mi_assert_internal(pq == mi_page_queue_of(page));\n mi_assert_internal(!mi_page_immediate_available(page));\n mi_assert_internal(!mi_page_is_in_full(page));\n\n if (mi_page_is_in_full(page)) return;\n mi_page_queue_enqueue_from(&mi_page_heap(page)->pages[MI_BIN_FULL], pq, page);\n _mi_page_free_collect(page,false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set\n}\n\n\n// Abandon a page with used blocks at the end of a thread.\n// Note: only call if it is ensured that no references exist from\n// the `page->heap->thread_delayed_free` into this page.\n// Currently only called through `mi_heap_collect_ex` which ensures this.\nvoid _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {\n mi_assert_internal(page != NULL);\n mi_assert_expensive(_mi_page_is_valid(page));\n mi_assert_internal(pq == mi_page_queue_of(page));\n mi_assert_internal(mi_page_heap(page) != NULL);\n\n mi_heap_t* pheap = mi_page_heap(page);\n\n // remove from our page list\n mi_segments_tld_t* segments_tld = &pheap->tld->segments;\n mi_page_queue_remove(pq, page);\n\n // page is no longer associated with our heap\n mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE);\n mi_page_set_heap(page, NULL);\n\n#if (MI_DEBUG>1) && !MI_TRACK_ENABLED\n // check there are no references left..\n for (mi_block_t* block = (mi_block_t*)pheap->thread_delayed_free; block != NULL; block = mi_block_nextx(pheap, block, pheap->keys)) {\n mi_assert_internal(_mi_ptr_page(block) != page);\n }\n#endif\n\n // and abandon it\n mi_assert_internal(mi_page_heap(page) == NULL);\n _mi_segment_page_abandon(page,segments_tld);\n}\n\n// force abandon a page\nvoid _mi_page_force_abandon(mi_page_t* page) {\n mi_heap_t* heap = mi_page_heap(page);\n // mark page as not using delayed free\n _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false);\n\n // ensure this page is no longer in the heap delayed free list\n _mi_heap_delayed_free_all(heap);\n // We can still access the page meta-info even if it is freed as we ensure\n // in `mi_segment_force_abandon` that the segment is not freed (yet)\n if (page->capacity == 0) return; // it may have been freed now\n\n // and now unlink it from the page queue and abandon (or free)\n mi_page_queue_t* pq = mi_heap_page_queue_of(heap, page);\n if (mi_page_all_free(page)) {\n _mi_page_free(page, pq, false);\n }\n else {\n _mi_page_abandon(page, pq);\n }\n}\n\n\n// Free a page with no more free blocks\nvoid _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {\n mi_assert_internal(page != NULL);\n mi_assert_expensive(_mi_page_is_valid(page));\n mi_assert_internal(pq == mi_page_queue_of(page));\n mi_assert_internal(mi_page_all_free(page));\n mi_assert_internal(mi_page_thread_free_flag(page)!=MI_DELAYED_FREEING);\n\n // no more aligned blocks in here\n mi_page_set_has_aligned(page, false);\n\n // remove from the page list\n // (no need to do _mi_heap_delayed_free first as all blocks are already free)\n mi_heap_t* heap = mi_page_heap(page);\n mi_segments_tld_t* segments_tld = &heap->tld->segments;\n mi_page_queue_remove(pq, page);\n\n // and free it \n mi_page_set_heap(page,NULL);\n _mi_segment_page_free(page, force, segments_tld);\n}\n\n#define MI_MAX_RETIRE_SIZE MI_MEDIUM_OBJ_SIZE_MAX // should be less than size for MI_BIN_HUGE\n#define MI_RETIRE_CYCLES (16)\n\n// Retire a page with no more used blocks\n// Important to not retire too quickly though as new\n// allocations might coming.\n// Note: called from `mi_free` and benchmarks often\n// trigger this due to freeing everything and then\n// allocating again so careful when changing this.\nvoid _mi_page_retire(mi_page_t* page) mi_attr_noexcept {\n mi_assert_internal(page != NULL);\n mi_assert_expensive(_mi_page_is_valid(page));\n mi_assert_internal(mi_page_all_free(page));\n\n mi_page_set_has_aligned(page, false);\n\n // don't retire too often..\n // (or we end up retiring and re-allocating most of the time)\n // NOTE: refine this more: we should not retire if this\n // is the only page left with free blocks. It is not clear\n // how to check this efficiently though...\n // for now, we don't retire if it is the only page left of this size class.\n mi_page_queue_t* pq = mi_page_queue_of(page);\n #if MI_RETIRE_CYCLES > 0\n const size_t bsize = mi_page_block_size(page);\n if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue?\n if (pq->last==page && pq->first==page) { // the only page in the queue?\n mi_stat_counter_increase(_mi_stats_main.pages_retire,1);\n page->retire_expire = (bsize <= MI_SMALL_OBJ_SIZE_MAX ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4);\n mi_heap_t* heap = mi_page_heap(page);\n mi_assert_internal(pq >= heap->pages);\n const size_t index = pq - heap->pages;\n mi_assert_internal(index < MI_BIN_FULL && index < MI_BIN_HUGE);\n if (index < heap->page_retired_min) heap->page_retired_min = index;\n if (index > heap->page_retired_max) heap->page_retired_max = index;\n mi_assert_internal(mi_page_all_free(page));\n return; // don't free after all\n }\n }\n #endif\n _mi_page_free(page, pq, false);\n}\n\n// free retired pages: we don't need to look at the entire queues\n// since we only retire pages that are at the head position in a queue.\nvoid _mi_heap_collect_retired(mi_heap_t* heap, bool force) {\n size_t min = MI_BIN_FULL;\n size_t max = 0;\n for(size_t bin = heap->page_retired_min; bin <= heap->page_retired_max; bin++) {\n mi_page_queue_t* pq = &heap->pages[bin];\n mi_page_t* page = pq->first;\n if (page != NULL && page->retire_expire != 0) {\n if (mi_page_all_free(page)) {\n page->retire_expire--;\n if (force || page->retire_expire == 0) {\n _mi_page_free(pq->first, pq, force);\n }\n else {\n // keep retired, update min/max\n if (bin < min) min = bin;\n if (bin > max) max = bin;\n }\n }\n else {\n page->retire_expire = 0;\n }\n }\n }\n heap->page_retired_min = min;\n heap->page_retired_max = max;\n}\n\n\n/* -----------------------------------------------------------\n Initialize the initial free list in a page.\n In secure mode we initialize a randomized list by\n alternating between slices.\n----------------------------------------------------------- */\n\n#define MI_MAX_SLICE_SHIFT (6) // at most 64 slices\n#define MI_MAX_SLICES (1UL << MI_MAX_SLICE_SHIFT)\n#define MI_MIN_SLICES (2)\n\nstatic void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats) {\n MI_UNUSED(stats);\n #if (MI_SECURE<=2)\n mi_assert_internal(page->free == NULL);\n mi_assert_internal(page->local_free == NULL);\n #endif\n mi_assert_internal(page->capacity + extend <= page->reserved);\n mi_assert_internal(bsize == mi_page_block_size(page));\n void* const page_area = mi_page_start(page);\n\n // initialize a randomized free list\n // set up `slice_count` slices to alternate between\n size_t shift = MI_MAX_SLICE_SHIFT;\n while ((extend >> shift) == 0) {\n shift--;\n }\n const size_t slice_count = (size_t)1U << shift;\n const size_t slice_extend = extend / slice_count;\n mi_assert_internal(slice_extend >= 1);\n mi_block_t* blocks[MI_MAX_SLICES]; // current start of the slice\n size_t counts[MI_MAX_SLICES]; // available objects in the slice\n for (size_t i = 0; i < slice_count; i++) {\n blocks[i] = mi_page_block_at(page, page_area, bsize, page->capacity + i*slice_extend);\n counts[i] = slice_extend;\n }\n counts[slice_count-1] += (extend % slice_count); // final slice holds the modulus too (todo: distribute evenly?)\n\n // and initialize the free list by randomly threading through them\n // set up first element\n const uintptr_t r = _mi_heap_random_next(heap);\n size_t current = r % slice_count;\n counts[current]--;\n mi_block_t* const free_start = blocks[current];\n // and iterate through the rest; use `random_shuffle` for performance\n uintptr_t rnd = _mi_random_shuffle(r|1); // ensure not 0\n for (size_t i = 1; i < extend; i++) {\n // call random_shuffle only every INTPTR_SIZE rounds\n const size_t round = i%MI_INTPTR_SIZE;\n if (round == 0) rnd = _mi_random_shuffle(rnd);\n // select a random next slice index\n size_t next = ((rnd >> 8*round) & (slice_count-1));\n while (counts[next]==0) { // ensure it still has space\n next++;\n if (next==slice_count) next = 0;\n }\n // and link the current block to it\n counts[next]--;\n mi_block_t* const block = blocks[current];\n blocks[current] = (mi_block_t*)((uint8_t*)block + bsize); // bump to the following block\n mi_block_set_next(page, block, blocks[next]); // and set next; note: we may have `current == next`\n current = next;\n }\n // prepend to the free list (usually NULL)\n mi_block_set_next(page, blocks[current], page->free); // end of the list\n page->free = free_start;\n}\n\nstatic mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats)\n{\n MI_UNUSED(stats);\n #if (MI_SECURE <= 2)\n mi_assert_internal(page->free == NULL);\n mi_assert_internal(page->local_free == NULL);\n #endif\n mi_assert_internal(page->capacity + extend <= page->reserved);\n mi_assert_internal(bsize == mi_page_block_size(page));\n void* const page_area = mi_page_start(page);\n\n mi_block_t* const start = mi_page_block_at(page, page_area, bsize, page->capacity);\n\n // initialize a sequential free list\n mi_block_t* const last = mi_page_block_at(page, page_area, bsize, page->capacity + extend - 1);\n mi_block_t* block = start;\n while(block <= last) {\n mi_block_t* next = (mi_block_t*)((uint8_t*)block + bsize);\n mi_block_set_next(page,block,next);\n block = next;\n }\n // prepend to free list (usually `NULL`)\n mi_block_set_next(page, last, page->free);\n page->free = start;\n}\n\n/* -----------------------------------------------------------\n Page initialize and extend the capacity\n----------------------------------------------------------- */\n\n#define MI_MAX_EXTEND_SIZE (4*1024) // heuristic, one OS page seems to work well.\n#if (MI_SECURE>=2)\n#define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many blocks\n#else\n#define MI_MIN_EXTEND (4)\n#endif\n\n// Extend the capacity (up to reserved) by initializing a free list\n// We do at most `MI_MAX_EXTEND` to avoid touching too much memory\n// Note: we also experimented with \"bump\" allocation on the first\n// allocations but this did not speed up any benchmark (due to an\n// extra test in malloc? or cache effects?)\nstatic bool mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) {\n mi_assert_expensive(mi_page_is_valid_init(page));\n #if (MI_SECURE<=2)\n mi_assert(page->free == NULL);\n mi_assert(page->local_free == NULL);\n if (page->free != NULL) return true;\n #endif\n if (page->capacity >= page->reserved) return true;\n\n mi_stat_counter_increase(tld->stats.pages_extended, 1);\n\n // calculate the extend count\n const size_t bsize = mi_page_block_size(page);\n size_t extend = page->reserved - page->capacity;\n mi_assert_internal(extend > 0);\n\n size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE/bsize);\n if (max_extend < MI_MIN_EXTEND) { max_extend = MI_MIN_EXTEND; }\n mi_assert_internal(max_extend > 0);\n\n if (extend > max_extend) {\n // ensure we don't touch memory beyond the page to reduce page commit.\n // the `lean` benchmark tests this. Going from 1 to 8 increases rss by 50%.\n extend = max_extend;\n }\n\n mi_assert_internal(extend > 0 && extend + page->capacity <= page->reserved);\n mi_assert_internal(extend < (1UL<<16));\n\n // and append the extend the free list\n if (extend < MI_MIN_SLICES || MI_SECURE<2) { //!mi_option_is_enabled(mi_option_secure)) {\n mi_page_free_list_extend(page, bsize, extend, &tld->stats );\n }\n else {\n mi_page_free_list_extend_secure(heap, page, bsize, extend, &tld->stats);\n }\n // enable the new free list\n page->capacity += (uint16_t)extend;\n mi_stat_increase(tld->stats.page_committed, extend * bsize);\n mi_assert_expensive(mi_page_is_valid_init(page));\n return true;\n}\n\n// Initialize a fresh page\nstatic void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi_tld_t* tld) {\n mi_assert(page != NULL);\n mi_segment_t* segment = _mi_page_segment(page);\n mi_assert(segment != NULL);\n mi_assert_internal(block_size > 0);\n // set fields\n mi_page_set_heap(page, heap);\n page->block_size = block_size;\n size_t page_size;\n page->page_start = _mi_segment_page_start(segment, page, &page_size);\n mi_track_mem_noaccess(page->page_start,page_size);\n mi_assert_internal(mi_page_block_size(page) <= page_size);\n mi_assert_internal(page_size <= page->slice_count*MI_SEGMENT_SLICE_SIZE);\n mi_assert_internal(page_size / block_size < (1L<<16));\n page->reserved = (uint16_t)(page_size / block_size);\n mi_assert_internal(page->reserved > 0);\n #if (MI_PADDING || MI_ENCODE_FREELIST)\n page->keys[0] = _mi_heap_random_next(heap);\n page->keys[1] = _mi_heap_random_next(heap);\n #endif\n page->free_is_zero = page->is_zero_init;\n #if MI_DEBUG>2\n if (page->is_zero_init) {\n mi_track_mem_defined(page->page_start, page_size);\n mi_assert_expensive(mi_mem_is_zero(page->page_start, page_size));\n }\n #endif\n mi_assert_internal(page->is_committed);\n if (block_size > 0 && _mi_is_power_of_two(block_size)) {\n page->block_size_shift = (uint8_t)(mi_ctz((uintptr_t)block_size));\n }\n else {\n page->block_size_shift = 0;\n }\n\n mi_assert_internal(page->capacity == 0);\n mi_assert_internal(page->free == NULL);\n mi_assert_internal(page->used == 0);\n mi_assert_internal(page->xthread_free == 0);\n mi_assert_internal(page->next == NULL);\n mi_assert_internal(page->prev == NULL);\n mi_assert_internal(page->retire_expire == 0);\n mi_assert_internal(!mi_page_has_aligned(page));\n #if (MI_PADDING || MI_ENCODE_FREELIST)\n mi_assert_internal(page->keys[0] != 0);\n mi_assert_internal(page->keys[1] != 0);\n #endif\n mi_assert_internal(page->block_size_shift == 0 || (block_size == ((size_t)1 << page->block_size_shift)));\n mi_assert_expensive(mi_page_is_valid_init(page));\n\n // initialize an initial free list\n if (mi_page_extend_free(heap,page,tld)) {\n mi_assert(mi_page_immediate_available(page));\n }\n return;\n}\n\n\n/* -----------------------------------------------------------\n Find pages with free blocks\n-------------------------------------------------------------*/\n\n// search for a best next page to use for at most N pages (often cut short if immediate blocks are available)\n#define MI_MAX_CANDIDATE_SEARCH (4)\n\n// is the page not yet used up to its reserved space?\nstatic bool mi_page_is_expandable(const mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_assert_internal(page->capacity <= page->reserved);\n return (page->capacity < page->reserved);\n}\n\n\n// Find a page with free blocks of `page->block_size`.\nstatic mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* pq, bool first_try)\n{\n // search through the pages in \"next fit\" order\n #if MI_STAT\n size_t count = 0;\n #endif\n size_t candidate_count = 0; // we reset this on the first candidate to limit the search\n mi_page_t* page_candidate = NULL; // a page with free space\n mi_page_t* page = pq->first;\n\n while (page != NULL)\n {\n mi_page_t* next = page->next; // remember next\n #if MI_STAT\n count++;\n #endif\n candidate_count++;\n\n // collect freed blocks by us and other threads\n _mi_page_free_collect(page, false);\n\n #if MI_MAX_CANDIDATE_SEARCH > 1\n // search up to N pages for a best candidate\n\n // is the local free list non-empty?\n const bool immediate_available = mi_page_immediate_available(page);\n\n // if the page is completely full, move it to the `mi_pages_full`\n // queue so we don't visit long-lived pages too often.\n if (!immediate_available && !mi_page_is_expandable(page)) {\n mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));\n mi_page_to_full(page, pq);\n }\n else {\n // the page has free space, make it a candidate\n // we prefer non-expandable pages with high usage as candidates (to reduce commit, and increase chances of free-ing up pages)\n if (page_candidate == NULL) {\n page_candidate = page;\n candidate_count = 0;\n }\n // prefer to reuse fuller pages (in the hope the less used page gets freed)\n else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page) && !mi_page_is_expandable(page)) {\n page_candidate = page;\n }\n // if we find a non-expandable candidate, or searched for N pages, return with the best candidate\n if (immediate_available || candidate_count > MI_MAX_CANDIDATE_SEARCH) {\n mi_assert_internal(page_candidate!=NULL);\n break;\n }\n }\n #else\n // first-fit algorithm\n // If the page contains free blocks, we are done\n if (mi_page_immediate_available(page) || mi_page_is_expandable(page)) {\n break; // pick this one\n }\n\n // If the page is completely full, move it to the `mi_pages_full`\n // queue so we don't visit long-lived pages too often.\n mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));\n mi_page_to_full(page, pq);\n #endif\n\n page = next;\n } // for each page\n\n mi_heap_stat_counter_increase(heap, page_searches, count);\n mi_heap_stat_counter_increase(heap, page_searches_count, 1);\n\n // set the page to the best candidate\n if (page_candidate != NULL) {\n page = page_candidate;\n }\n if (page != NULL) {\n if (!mi_page_immediate_available(page)) {\n mi_assert_internal(mi_page_is_expandable(page));\n if (!mi_page_extend_free(heap, page, heap->tld)) {\n page = NULL; // failed to extend\n }\n }\n mi_assert_internal(page == NULL || mi_page_immediate_available(page));\n }\n\n if (page == NULL) {\n _mi_heap_collect_retired(heap, false); // perhaps make a page available?\n page = mi_page_fresh(heap, pq);\n if (page == NULL && first_try) {\n // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again\n page = mi_page_queue_find_free_ex(heap, pq, false);\n }\n }\n else {\n // move the page to the front of the queue\n mi_page_queue_move_to_front(heap, pq, page);\n page->retire_expire = 0;\n // _mi_heap_collect_retired(heap, false); // update retire counts; note: increases rss on MemoryLoad bench so don't do this\n }\n mi_assert_internal(page == NULL || mi_page_immediate_available(page));\n\n\n return page;\n}\n\n\n\n// Find a page with free blocks of `size`.\nstatic inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {\n mi_page_queue_t* pq = mi_page_queue(heap, size);\n\n // check the first page: we even do this with candidate search or otherwise we re-search every time\n mi_page_t* page = pq->first;\n if (page != NULL) {\n #if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness\n if (page->capacity < page->reserved && ((_mi_heap_random_next(heap) & 1) == 1)) {\n mi_page_extend_free(heap, page, heap->tld);\n mi_assert_internal(mi_page_immediate_available(page));\n }\n else\n #endif\n {\n _mi_page_free_collect(page,false);\n }\n\n if (mi_page_immediate_available(page)) {\n page->retire_expire = 0;\n return page; // fast path\n }\n }\n\n return mi_page_queue_find_free_ex(heap, pq, true);\n}\n\n\n/* -----------------------------------------------------------\n Users can register a deferred free function called\n when the `free` list is empty. Since the `local_free`\n is separate this is deterministically called after\n a certain number of allocations.\n----------------------------------------------------------- */\n\n// The program should only install a single deferred free handler before doing allocation.\nstatic _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers)\nstatic _Atomic(void*) deferred_arg; \n\nvoid _mi_deferred_free(mi_heap_t* heap, bool force) {\n heap->tld->heartbeat++;\n mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free);\n if (fun != NULL && !heap->tld->recurse) {\n heap->tld->recurse = true;\n void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg); \n fun(force, heap->tld->heartbeat, arg);\n heap->tld->recurse = false;\n }\n}\n\nvoid mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept {\n mi_atomic_store_ptr_release(void,&deferred_arg, arg);\n mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn); \n}\n\n\n/* -----------------------------------------------------------\n General allocation\n----------------------------------------------------------- */\n\n// Large and huge page allocation.\n// Huge pages contain just one block, and the segment contains just that page (as `MI_SEGMENT_HUGE`).\n// Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX)\n// so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`.\nstatic mi_page_t* mi_large_huge_page_alloc(mi_heap_t* heap, size_t size, size_t page_alignment) {\n size_t block_size = _mi_os_good_alloc_size(size);\n mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0);\n bool is_huge = (block_size > MI_LARGE_OBJ_SIZE_MAX || page_alignment > 0);\n #if MI_HUGE_PAGE_ABANDON\n mi_page_queue_t* pq = (is_huge ? NULL : mi_page_queue(heap, block_size));\n #else\n mi_page_queue_t* pq = mi_page_queue(heap, is_huge ? MI_LARGE_OBJ_SIZE_MAX+1 : block_size);\n mi_assert_internal(!is_huge || mi_page_queue_is_huge(pq));\n #endif\n mi_page_t* page = mi_page_fresh_alloc(heap, pq, block_size, page_alignment);\n if (page != NULL) {\n mi_assert_internal(mi_page_immediate_available(page));\n\n if (is_huge) {\n mi_assert_internal(mi_page_is_huge(page));\n mi_assert_internal(_mi_page_segment(page)->kind == MI_SEGMENT_HUGE);\n mi_assert_internal(_mi_page_segment(page)->used==1);\n #if MI_HUGE_PAGE_ABANDON\n mi_assert_internal(_mi_page_segment(page)->thread_id==0); // abandoned, not in the huge queue\n mi_page_set_heap(page, NULL);\n #endif\n }\n else {\n mi_assert_internal(!mi_page_is_huge(page));\n }\n\n const size_t bsize = mi_page_usable_block_size(page); // note: not `mi_page_block_size` to account for padding\n /*if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {\n mi_heap_stat_increase(heap, malloc_large, bsize);\n mi_heap_stat_counter_increase(heap, malloc_large_count, 1);\n }\n else */\n {\n _mi_stat_increase(&heap->tld->stats.malloc_huge, bsize);\n _mi_stat_counter_increase(&heap->tld->stats.malloc_huge_count, 1);\n }\n }\n return page;\n}\n\n\n// Allocate a page\n// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.\nstatic mi_page_t* mi_find_page(mi_heap_t* heap, size_t size, size_t huge_alignment) mi_attr_noexcept {\n // huge allocation?\n const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`\n if mi_unlikely(req_size > (MI_MEDIUM_OBJ_SIZE_MAX - MI_PADDING_SIZE) || huge_alignment > 0) {\n if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) {\n _mi_error_message(EOVERFLOW, \"allocation request is too large (%zu bytes)\\n\", req_size);\n return NULL;\n }\n else {\n return mi_large_huge_page_alloc(heap,size,huge_alignment);\n }\n }\n else {\n // otherwise find a page with free blocks in our size segregated queues\n #if MI_PADDING\n mi_assert_internal(size >= MI_PADDING_SIZE);\n #endif\n return mi_find_free_page(heap, size);\n }\n}\n\n// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed.\n// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.\n// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for\n// very large requested alignments in which case we use a huge singleton page.\nvoid* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept\n{\n mi_assert_internal(heap != NULL);\n\n // initialize if necessary\n if mi_unlikely(!mi_heap_is_initialized(heap)) {\n heap = mi_heap_get_default(); // calls mi_thread_init\n if mi_unlikely(!mi_heap_is_initialized(heap)) { return NULL; }\n }\n mi_assert_internal(mi_heap_is_initialized(heap));\n\n // do administrative tasks every N generic mallocs\n if mi_unlikely(++heap->generic_count >= 100) {\n heap->generic_collect_count += heap->generic_count;\n heap->generic_count = 0;\n // call potential deferred free routines\n _mi_deferred_free(heap, false);\n\n // free delayed frees from other threads (but skip contended ones)\n _mi_heap_delayed_free_partial(heap);\n\n // collect every once in a while (10000 by default)\n const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L);\n if (heap->generic_collect_count >= generic_collect) {\n heap->generic_collect_count = 0;\n mi_heap_collect(heap, false /* force? */);\n }\n }\n\n // find (or allocate) a page of the right size\n mi_page_t* page = mi_find_page(heap, size, huge_alignment);\n if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more\n mi_heap_collect(heap, true /* force */);\n page = mi_find_page(heap, size, huge_alignment);\n }\n\n if mi_unlikely(page == NULL) { // out of memory\n const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`\n _mi_error_message(ENOMEM, \"unable to allocate memory (%zu bytes)\\n\", req_size);\n return NULL;\n }\n\n mi_assert_internal(mi_page_immediate_available(page));\n mi_assert_internal(mi_page_block_size(page) >= size);\n\n // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc)\n void* const p = _mi_page_malloc_zero(heap, page, size, zero, usable);\n mi_assert_internal(p != NULL);\n\n // move singleton pages to the full queue\n if (page->reserved == page->used) {\n mi_page_to_full(page, mi_page_queue_of(page));\n }\n return p;\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/prim/emscripten/prim.c", "sha256": "fb236c93d69f68ee57afead03b874aa02fbf2efc25ae483fab0d4ff7d71543ad", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen, Alon Zakai\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// This file is included in `src/prim/prim.c`\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\"\n\n#include // getentropy\n\n// Design\n// ======\n//\n// mimalloc is built on top of emmalloc. emmalloc is a minimal allocator on top\n// of sbrk. The reason for having three layers here is that we want mimalloc to\n// be able to allocate and release system memory properly, the same way it would\n// when using VirtualAlloc on Windows or mmap on POSIX, and sbrk is too limited.\n// Specifically, sbrk can only go up and down, and not \"skip\" over regions, and\n// so we end up either never freeing memory to the system, or we can get stuck\n// with holes.\n//\n// Atm wasm generally does *not* free memory back the system: once grown, we do\n// not shrink back down (https://github.com/WebAssembly/design/issues/1397).\n// However, that is expected to improve\n// (https://github.com/WebAssembly/memory-control/blob/main/proposals/memory-control/Overview.md)\n// and so we do not want to bake those limitations in here.\n//\n// Even without that issue, we want our system allocator to handle holes, that\n// is, it should merge freed regions and allow allocating new content there of\n// the full size, etc., so that we do not waste space. That means that the\n// system allocator really does need to handle the general problem of allocating\n// and freeing variable-sized chunks of memory in a random order, like malloc/\n// free do. And so it makes sense to layer mimalloc on top of such an\n// implementation.\n//\n// emmalloc makes sense for the lower level because it is small and simple while\n// still fully handling merging of holes etc. It is not the most efficient\n// allocator, but our assumption is that mimalloc needs to be fast while the\n// system allocator underneath it is called much less frequently.\n//\n\n//---------------------------------------------\n// init\n//---------------------------------------------\n\nvoid _mi_prim_mem_init( mi_os_mem_config_t* config) {\n config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB\n config->alloc_granularity = 16;\n config->has_overcommit = false;\n config->has_partial_free = false;\n config->has_virtual_reserve = false;\n}\n\nextern void emmalloc_free(void*);\n\nint _mi_prim_free(void* addr, size_t size) {\n if (size==0) return 0;\n emmalloc_free(addr);\n return 0;\n}\n\n\n//---------------------------------------------\n// Allocation\n//---------------------------------------------\n\nextern void* emmalloc_memalign(size_t alignment, size_t size);\n\n// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.\nint _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {\n MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);\n *is_large = false;\n // TODO: Track the highest address ever seen; first uses of it are zeroes.\n // That assumes no one else uses sbrk but us (they could go up,\n // scribble, and then down), but we could assert on that perhaps.\n *is_zero = false;\n // emmalloc has a minimum alignment size.\n #define MIN_EMMALLOC_ALIGN 8\n if (try_alignment < MIN_EMMALLOC_ALIGN) {\n try_alignment = MIN_EMMALLOC_ALIGN;\n }\n void* p = emmalloc_memalign(try_alignment, size);\n *addr = p;\n if (p == 0) {\n return ENOMEM;\n }\n return 0;\n}\n\n\n//---------------------------------------------\n// Commit/Reset\n//---------------------------------------------\n\nint _mi_prim_commit(void* addr, size_t size, bool* is_zero) {\n MI_UNUSED(addr); MI_UNUSED(size);\n // See TODO above.\n *is_zero = false;\n return 0;\n}\n\nint _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {\n MI_UNUSED(addr); MI_UNUSED(size);\n *needs_recommit = false;\n return 0;\n}\n\nint _mi_prim_reset(void* addr, size_t size) {\n MI_UNUSED(addr); MI_UNUSED(size);\n return 0;\n}\n\nint _mi_prim_reuse(void* addr, size_t size) {\n MI_UNUSED(addr); MI_UNUSED(size);\n return 0;\n}\n\nint _mi_prim_protect(void* addr, size_t size, bool protect) {\n MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect);\n return 0;\n}\n\n\n//---------------------------------------------\n// Huge pages and NUMA nodes\n//---------------------------------------------\n\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {\n MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);\n *is_zero = true;\n *addr = NULL;\n return ENOSYS;\n}\n\nsize_t _mi_prim_numa_node(void) {\n return 0;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n return 1;\n}\n\n\n//----------------------------------------------------------------\n// Clock\n//----------------------------------------------------------------\n\n#include \n\nmi_msecs_t _mi_prim_clock_now(void) {\n return emscripten_date_now();\n}\n\n\n//----------------------------------------------------------------\n// Process info\n//----------------------------------------------------------------\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo)\n{\n // use defaults\n MI_UNUSED(pinfo);\n}\n\n\n//----------------------------------------------------------------\n// Output\n//----------------------------------------------------------------\n\n#include \n\nvoid _mi_prim_out_stderr( const char* msg) {\n emscripten_console_error(msg);\n}\n\n\n//----------------------------------------------------------------\n// Environment\n//----------------------------------------------------------------\n\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size) {\n // For code size reasons, do not support environ customization for now.\n MI_UNUSED(name);\n MI_UNUSED(result);\n MI_UNUSED(result_size);\n return false;\n}\n\n\n//----------------------------------------------------------------\n// Random\n//----------------------------------------------------------------\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n int err = getentropy(buf, buf_len);\n return !err;\n}\n\n\n//----------------------------------------------------------------\n// Thread init/done\n//----------------------------------------------------------------\n\n#if defined(MI_USE_PTHREADS)\n\n// use pthread local storage keys to detect thread ending\n// (and used with MI_TLS_PTHREADS for the default heap)\npthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);\n\nstatic void mi_pthread_done(void* value) {\n if (value!=NULL) {\n _mi_thread_done((mi_heap_t*)value);\n }\n}\n\nvoid _mi_prim_thread_init_auto_done(void) {\n mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));\n pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);\n}\n\nvoid _mi_prim_thread_done_auto_done(void) {\n if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809\n pthread_key_delete(_mi_heap_default_key);\n }\n}\n\nvoid _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {\n if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD\n pthread_setspecific(_mi_heap_default_key, heap);\n }\n}\n\n#else\n\nvoid _mi_prim_thread_init_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_done_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {\n MI_UNUSED(heap);\n\n}\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/prim/osx/alloc-override-zone.c", "sha256": "1d0a5f40f7ab1cf8614cfd71c49d885fd129b9bf17b5ce6da2dcd4bae5880bf0", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2022, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n\n#if defined(MI_MALLOC_OVERRIDE)\n\n#if !defined(__APPLE__)\n#error \"this file should only be included on macOS\"\n#endif\n\n/* ------------------------------------------------------\n Override system malloc on macOS\n This is done through the malloc zone interface.\n It seems to be most robust in combination with interposing\n though or otherwise we may get zone errors as there are could\n be allocations done by the time we take over the\n zone.\n------------------------------------------------------ */\n\n#include \n#include \n#include // memset\n#include \n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)\n// only available from OSX 10.6\nextern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import));\n#endif\n\n/* ------------------------------------------------------\n malloc zone members\n------------------------------------------------------ */\n\nstatic size_t zone_size(malloc_zone_t* zone, const void* p) {\n MI_UNUSED(zone);\n if (!mi_is_in_heap_region(p)){ return 0; } // not our pointer, bail out\n return mi_usable_size(p);\n}\n\nstatic void* zone_malloc(malloc_zone_t* zone, size_t size) {\n MI_UNUSED(zone);\n return mi_malloc(size);\n}\n\nstatic void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) {\n MI_UNUSED(zone);\n return mi_calloc(count, size);\n}\n\nstatic void* zone_valloc(malloc_zone_t* zone, size_t size) {\n MI_UNUSED(zone);\n return mi_malloc_aligned(size, _mi_os_page_size());\n}\n\nstatic void zone_free(malloc_zone_t* zone, void* p) {\n MI_UNUSED(zone);\n mi_cfree(p);\n}\n\nstatic void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) {\n MI_UNUSED(zone);\n return mi_realloc(p, newsize);\n}\n\nstatic void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) {\n MI_UNUSED(zone);\n return mi_malloc_aligned(size,alignment);\n}\n\nstatic void zone_destroy(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n // todo: ignore for now?\n}\n\nstatic unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) {\n unsigned i;\n for (i = 0; i < count; i++) {\n ps[i] = zone_malloc(zone, size);\n if (ps[i] == NULL) break;\n }\n return i;\n}\n\nstatic void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) {\n for(size_t i = 0; i < count; i++) {\n zone_free(zone, ps[i]);\n ps[i] = NULL;\n }\n}\n\nstatic size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) {\n MI_UNUSED(zone); MI_UNUSED(size);\n mi_collect(false);\n return 0;\n}\n\nstatic void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) {\n MI_UNUSED(size);\n zone_free(zone,p);\n}\n\nstatic boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) {\n MI_UNUSED(zone);\n return mi_is_in_heap_region(p);\n}\n\n\n/* ------------------------------------------------------\n Introspection members\n------------------------------------------------------ */\n\nstatic kern_return_t intro_enumerator(task_t task, void* p,\n unsigned type_mask, vm_address_t zone_address,\n memory_reader_t reader,\n vm_range_recorder_t recorder)\n{\n // todo: enumerate all memory\n MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address);\n MI_UNUSED(reader); MI_UNUSED(recorder);\n return KERN_SUCCESS;\n}\n\nstatic size_t intro_good_size(malloc_zone_t* zone, size_t size) {\n MI_UNUSED(zone);\n return mi_good_size(size);\n}\n\nstatic boolean_t intro_check(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n return true;\n}\n\nstatic void intro_print(malloc_zone_t* zone, boolean_t verbose) {\n MI_UNUSED(zone); MI_UNUSED(verbose);\n mi_stats_print(NULL);\n}\n\nstatic void intro_log(malloc_zone_t* zone, void* p) {\n MI_UNUSED(zone); MI_UNUSED(p);\n // todo?\n}\n\nstatic void intro_force_lock(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n // todo?\n}\n\nstatic void intro_force_unlock(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n // todo?\n}\n\nstatic void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) {\n MI_UNUSED(zone);\n // todo...\n stats->blocks_in_use = 0;\n stats->size_in_use = 0;\n stats->max_size_in_use = 0;\n stats->size_allocated = 0;\n}\n\nstatic boolean_t intro_zone_locked(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n return false;\n}\n\n\n/* ------------------------------------------------------\n At process start, override the default allocator\n------------------------------------------------------ */\n\n#if defined(__GNUC__) && !defined(__clang__)\n#pragma GCC diagnostic ignored \"-Wmissing-field-initializers\"\n#endif\n\n#if defined(__clang__)\n#pragma clang diagnostic ignored \"-Wc99-extensions\"\n#endif\n\nstatic malloc_introspection_t mi_introspect = {\n .enumerator = &intro_enumerator,\n .good_size = &intro_good_size,\n .check = &intro_check,\n .print = &intro_print,\n .log = &intro_log,\n .force_lock = &intro_force_lock,\n .force_unlock = &intro_force_unlock,\n#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)\n .statistics = &intro_statistics,\n .zone_locked = &intro_zone_locked,\n#endif\n};\n\nstatic malloc_zone_t mi_malloc_zone = {\n // note: even with designators, the order is important for C++ compilation\n //.reserved1 = NULL,\n //.reserved2 = NULL,\n .size = &zone_size,\n .malloc = &zone_malloc,\n .calloc = &zone_calloc,\n .valloc = &zone_valloc,\n .free = &zone_free,\n .realloc = &zone_realloc,\n .destroy = &zone_destroy,\n .zone_name = \"mimalloc\",\n .batch_malloc = &zone_batch_malloc,\n .batch_free = &zone_batch_free,\n .introspect = &mi_introspect,\n#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)\n #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)\n .version = 10,\n #else\n .version = 9,\n #endif\n // switch to version 9+ on OSX 10.6 to support memalign.\n .memalign = &zone_memalign,\n .free_definite_size = &zone_free_definite_size,\n #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)\n .pressure_relief = &zone_pressure_relief,\n #endif\n #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)\n .claimed_address = &zone_claimed_address,\n #endif\n#else\n .version = 4,\n#endif\n};\n\n#ifdef __cplusplus\n}\n#endif\n\n\n#if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT)\n\n// ------------------------------------------------------\n// Override malloc_xxx and malloc_zone_xxx api's to use only\n// our mimalloc zone. Since even the loader uses malloc\n// on macOS, this ensures that all allocations go through\n// mimalloc (as all calls are interposed).\n// The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`,\n// Here, we also override macOS specific API's like\n// `malloc_zone_calloc` etc. see \n// ------------------------------------------------------\n\nstatic inline malloc_zone_t* mi_get_default_zone(void) {\n mi_atomic_do_once {\n malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see )\n }\n return &mi_malloc_zone;\n}\n\nmi_decl_externc int malloc_jumpstart(uintptr_t cookie);\nmi_decl_externc void _malloc_fork_prepare(void);\nmi_decl_externc void _malloc_fork_parent(void);\nmi_decl_externc void _malloc_fork_child(void);\n\n\nstatic malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) {\n MI_UNUSED(size); MI_UNUSED(flags);\n return mi_get_default_zone();\n}\n\nstatic malloc_zone_t* mi_malloc_default_zone (void) {\n return mi_get_default_zone();\n}\n\nstatic malloc_zone_t* mi_malloc_default_purgeable_zone(void) {\n return mi_get_default_zone();\n}\n\nstatic void mi_malloc_destroy_zone(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n // nothing.\n}\n\nstatic kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) {\n MI_UNUSED(task); MI_UNUSED(mr);\n if (addresses != NULL) *addresses = NULL;\n if (count != NULL) *count = 0;\n return KERN_SUCCESS;\n}\n\nstatic const char* mi_malloc_get_zone_name(malloc_zone_t* zone) {\n return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name);\n}\n\nstatic void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) {\n MI_UNUSED(zone); MI_UNUSED(name);\n}\n\nstatic int mi_malloc_jumpstart(uintptr_t cookie) {\n MI_UNUSED(cookie);\n return 1; // or 0 for no error?\n}\n\nstatic void mi__malloc_fork_prepare(void) {\n // nothing\n}\nstatic void mi__malloc_fork_parent(void) {\n // nothing\n}\nstatic void mi__malloc_fork_child(void) {\n // nothing\n}\n\nstatic void mi_malloc_printf(const char* fmt, ...) {\n MI_UNUSED(fmt);\n}\n\nstatic bool zone_check(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n return true;\n}\n\nstatic malloc_zone_t* zone_from_ptr(const void* p) {\n MI_UNUSED(p);\n return (mi_is_in_heap_region(p) ? mi_get_default_zone() : NULL);\n}\n\nstatic void zone_log(malloc_zone_t* zone, void* p) {\n MI_UNUSED(zone); MI_UNUSED(p);\n}\n\nstatic void zone_print(malloc_zone_t* zone, bool b) {\n MI_UNUSED(zone); MI_UNUSED(b);\n}\n\nstatic void zone_print_ptr_info(void* p) {\n MI_UNUSED(p);\n}\n\nstatic void zone_register(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n}\n\nstatic void zone_unregister(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n}\n\n// use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1`\n// See: \nstruct mi_interpose_s {\n const void* replacement;\n const void* target;\n};\n#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun }\n#define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun)\n#define MI_INTERPOSE_ZONE(fun) MI_INTERPOSE_FUN(malloc_##fun,fun)\n__attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] __attribute__((section(\"__DATA, __interpose\"))) =\n{\n\n MI_INTERPOSE_MI(malloc_create_zone),\n MI_INTERPOSE_MI(malloc_default_purgeable_zone),\n MI_INTERPOSE_MI(malloc_default_zone),\n MI_INTERPOSE_MI(malloc_destroy_zone),\n MI_INTERPOSE_MI(malloc_get_all_zones),\n MI_INTERPOSE_MI(malloc_get_zone_name),\n MI_INTERPOSE_MI(malloc_jumpstart),\n MI_INTERPOSE_MI(malloc_printf),\n MI_INTERPOSE_MI(malloc_set_zone_name),\n MI_INTERPOSE_MI(_malloc_fork_child),\n MI_INTERPOSE_MI(_malloc_fork_parent),\n MI_INTERPOSE_MI(_malloc_fork_prepare),\n\n MI_INTERPOSE_ZONE(zone_batch_free),\n MI_INTERPOSE_ZONE(zone_batch_malloc),\n MI_INTERPOSE_ZONE(zone_calloc),\n MI_INTERPOSE_ZONE(zone_check),\n MI_INTERPOSE_ZONE(zone_free),\n MI_INTERPOSE_ZONE(zone_from_ptr),\n MI_INTERPOSE_ZONE(zone_log),\n MI_INTERPOSE_ZONE(zone_malloc),\n MI_INTERPOSE_ZONE(zone_memalign),\n MI_INTERPOSE_ZONE(zone_print),\n MI_INTERPOSE_ZONE(zone_print_ptr_info),\n MI_INTERPOSE_ZONE(zone_realloc),\n MI_INTERPOSE_ZONE(zone_register),\n MI_INTERPOSE_ZONE(zone_unregister),\n MI_INTERPOSE_ZONE(zone_valloc)\n};\n\n\n#else\n\n// ------------------------------------------------------\n// hook into the zone api's without interposing\n// This is the official way of adding an allocator but\n// it seems less robust than using interpose.\n// ------------------------------------------------------\n\nstatic inline malloc_zone_t* mi_get_default_zone(void)\n{\n // The first returned zone is the real default\n malloc_zone_t** zones = NULL;\n unsigned count = 0;\n kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count);\n if (ret == KERN_SUCCESS && count > 0) {\n return zones[0];\n }\n else {\n // fallback\n return malloc_default_zone();\n }\n}\n\n#if defined(__clang__)\n__attribute__((constructor(101))) // highest priority\n#else\n__attribute__((constructor)) // priority level is not supported by gcc\n#endif\n__attribute__((used))\nstatic void _mi_macos_override_malloc(void) {\n malloc_zone_t* purgeable_zone = NULL;\n\n #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)\n // force the purgeable zone to exist to avoid strange bugs\n if (malloc_default_purgeable_zone) {\n purgeable_zone = malloc_default_purgeable_zone();\n }\n #endif\n\n // Register our zone.\n // thomcc: I think this is still needed to put us in the zone list.\n malloc_zone_register(&mi_malloc_zone);\n // Unregister the default zone, this makes our zone the new default\n // as that was the last registered.\n malloc_zone_t *default_zone = mi_get_default_zone();\n // thomcc: Unsure if the next test is *always* false or just false in the\n // cases I've tried. I'm also unsure if the code inside is needed. at all\n if (default_zone != &mi_malloc_zone) {\n malloc_zone_unregister(default_zone);\n\n // Reregister the default zone so free and realloc in that zone keep working.\n malloc_zone_register(default_zone);\n }\n\n // Unregister, and re-register the purgeable_zone to avoid bugs if it occurs\n // earlier than the default zone.\n if (purgeable_zone != NULL) {\n malloc_zone_unregister(purgeable_zone);\n malloc_zone_register(purgeable_zone);\n }\n\n}\n#endif // MI_OSX_INTERPOSE\n\n#endif // MI_MALLOC_OVERRIDE\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/prim/osx/prim.c", "sha256": "247a9952465eb105be03a9962e922b085ce5d52034775551a61cd8594275be73", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// We use the unix/prim.c with the mmap API on macOSX\n#include \"../unix/prim.c\"\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/prim/prim.c", "sha256": "cc771788e5ba591efb681829a8724f8662a1f50f3db39cf5ebe7738b8ed57e44", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// Select the implementation of the primitives\n// depending on the OS.\n\n#if defined(_WIN32)\n#include \"windows/prim.c\" // VirtualAlloc (Windows)\n\n#elif defined(__APPLE__)\n#include \"osx/prim.c\" // macOSX (actually defers to mmap in unix/prim.c)\n\n#elif defined(__wasi__)\n#define MI_USE_SBRK\n#include \"wasi/prim.c\" // memory-grow or sbrk (Wasm)\n\n#elif defined(__EMSCRIPTEN__)\n#include \"emscripten/prim.c\" // emmalloc_*, + pthread support\n\n#else\n#include \"unix/prim.c\" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.)\n\n#endif\n\n// Generic process initialization\n#ifndef MI_PRIM_HAS_PROCESS_ATTACH\n#if defined(__GNUC__) || defined(__clang__)\n // gcc,clang: use the constructor/destructor attribute\n // which for both seem to run before regular constructors/destructors\n #if defined(__clang__)\n #define mi_attr_constructor __attribute__((constructor(101)))\n #define mi_attr_destructor __attribute__((destructor(101)))\n #else\n #define mi_attr_constructor __attribute__((constructor))\n #define mi_attr_destructor __attribute__((destructor))\n #endif\n static void mi_attr_constructor mi_process_attach(void) {\n _mi_auto_process_init();\n }\n static void mi_attr_destructor mi_process_detach(void) {\n _mi_auto_process_done();\n }\n#elif defined(__cplusplus)\n // C++: use static initialization to detect process start/end\n // This is not guaranteed to be first/last but the best we can generally do?\n struct mi_init_done_t {\n mi_init_done_t() {\n _mi_auto_process_init();\n }\n ~mi_init_done_t() {\n _mi_auto_process_done();\n }\n };\n static mi_init_done_t mi_init_done;\n #else\n #pragma message(\"define a way to call _mi_auto_process_init/done on your platform\")\n#endif\n#endif\n\n// Generic allocator init/done callback\n#ifndef MI_PRIM_HAS_ALLOCATOR_INIT\nbool _mi_is_redirected(void) {\n return false;\n}\nbool _mi_allocator_init(const char** message) {\n if (message != NULL) { *message = NULL; }\n return true;\n}\nvoid _mi_allocator_done(void) {\n // nothing to do\n}\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/prim/unix/prim.c", "sha256": "0ebab4f3b989fbb347f776a14183d55ee09d05df711815468a7bfb75cc0a5881", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// This file is included in `src/prim/prim.c`\n\n#ifndef _DEFAULT_SOURCE\n#define _DEFAULT_SOURCE // ensure mmap flags and syscall are defined\n#endif\n\n#if defined(__sun)\n// illumos provides new mman.h api when any of these are defined\n// otherwise the old api based on caddr_t which predates the void pointers one.\n// stock solaris provides only the former, chose to atomically to discard those\n// flags only here rather than project wide tough.\n#undef _XOPEN_SOURCE\n#undef _POSIX_C_SOURCE\n#endif\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\"\n\n#include // mmap\n#include // sysconf\n#include // open, close, read, access\n#include // getenv, arc4random_buf\n\n#if defined(__linux__)\n #include \n #include // THP disable, PR_SET_VMA\n #include // sysinfo\n #if defined(__GLIBC__) && !defined(PR_SET_VMA)\n #include \n #endif\n #if defined(__GLIBC__)\n #include // linux mmap flags\n #else\n #include \n #endif\n#elif defined(__APPLE__)\n #include \n #include \n #if !defined(TARGET_OS_OSX) || TARGET_OS_OSX // see issue #879, used to be (!TARGET_IOS_IPHONE && !TARGET_IOS_SIMULATOR)\n #include // VM_MAKE_TAG, VM_FLAGS_SUPERPAGE_SIZE_2MB, etc.\n #endif\n #if !defined(MAC_OS_X_VERSION_10_7)\n #define MAC_OS_X_VERSION_10_7 1070\n #endif\n #include \n#elif defined(__FreeBSD__) || defined(__DragonFly__)\n #include \n #if __FreeBSD_version >= 1200000\n #include \n #include \n #endif\n #include \n#endif\n\n#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__FreeBSD__)\n #define MI_HAS_SYSCALL_H\n #include \n#endif\n\n#if !defined(MADV_DONTNEED) && defined(POSIX_MADV_DONTNEED) // QNX\n#define MADV_DONTNEED POSIX_MADV_DONTNEED\n#endif\n#if !defined(MADV_FREE) && defined(POSIX_MADV_FREE) // QNX\n#define MADV_FREE POSIX_MADV_FREE\n#endif\n\n#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // TODO: can we query the OS for this?\n\n//------------------------------------------------------------------------------------\n// Use syscalls for some primitives to allow for libraries that override open/read/close etc.\n// and do allocation themselves; using syscalls prevents recursion when mimalloc is\n// still initializing (issue #713)\n// Declare inline to avoid unused function warnings.\n//------------------------------------------------------------------------------------\n\n#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access)\n\nstatic inline int mi_prim_open(const char* fpath, int open_flags) {\n return syscall(SYS_open,fpath,open_flags,0);\n}\nstatic inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {\n return syscall(SYS_read,fd,buf,bufsize);\n}\nstatic inline int mi_prim_close(int fd) {\n return syscall(SYS_close,fd);\n}\nstatic inline int mi_prim_access(const char *fpath, int mode) {\n return syscall(SYS_access,fpath,mode);\n}\n\n#else\n\nstatic inline int mi_prim_open(const char* fpath, int open_flags) {\n return open(fpath,open_flags);\n}\nstatic inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {\n return read(fd,buf,bufsize);\n}\nstatic inline int mi_prim_close(int fd) {\n return close(fd);\n}\nstatic inline int mi_prim_access(const char *fpath, int mode) {\n return access(fpath,mode);\n}\n\n#endif\n\n\n\n//---------------------------------------------\n// init\n//---------------------------------------------\n\nstatic bool unix_detect_overcommit(void) {\n bool os_overcommit = true;\n #if defined(__linux__)\n int fd = mi_prim_open(\"/proc/sys/vm/overcommit_memory\", O_RDONLY);\n if (fd >= 0) {\n char buf[32];\n ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));\n mi_prim_close(fd);\n // \n // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE)\n if (nread >= 1) {\n os_overcommit = (buf[0] == '0' || buf[0] == '1');\n }\n }\n #elif defined(__FreeBSD__)\n int val = 0;\n size_t olen = sizeof(val);\n if (sysctlbyname(\"vm.overcommit\", &val, &olen, NULL, 0) == 0) {\n os_overcommit = (val != 0);\n }\n #else\n // default: overcommit is true\n #endif\n return os_overcommit;\n}\n\n// try to detect the physical memory dynamically (if possible)\nstatic void unix_detect_physical_memory( size_t page_size, size_t* physical_memory_in_kib ) {\n #if defined(CTL_HW) && (defined(HW_PHYSMEM64) || defined(HW_MEMSIZE)) // freeBSD, macOS\n MI_UNUSED(page_size);\n int64_t physical_memory = 0;\n size_t length = sizeof(int64_t);\n #if defined(HW_PHYSMEM64)\n int mib[2] = { CTL_HW, HW_PHYSMEM64 };\n #else\n int mib[2] = { CTL_HW, HW_MEMSIZE };\n #endif\n const int err = sysctl(mib, 2, &physical_memory, &length, NULL, 0);\n if (err==0 && physical_memory > 0) {\n const int64_t phys_in_kib = physical_memory / MI_KiB;\n if (phys_in_kib > 0 && (uint64_t)phys_in_kib <= SIZE_MAX) {\n *physical_memory_in_kib = (size_t)phys_in_kib;\n }\n }\n #elif defined(__linux__)\n MI_UNUSED(page_size);\n struct sysinfo info; _mi_memzero_var(info);\n const int err = sysinfo(&info);\n if (err==0 && info.mem_unit > 0 && info.totalram <= SIZE_MAX) {\n if (info.mem_unit==MI_KiB) {\n *physical_memory_in_kib = (size_t)info.totalram;\n }\n else {\n size_t total = 0;\n if (!mi_mul_overflow((size_t)info.totalram, (size_t)info.mem_unit, &total)) {\n *physical_memory_in_kib = (total / MI_KiB);\n }\n }\n }\n #elif defined(_SC_PHYS_PAGES) // do not use by default as it might cause allocation (by using `fopen` to parse /proc/meminfo) (issue #1100)\n const long pphys = sysconf(_SC_PHYS_PAGES);\n const size_t psize_in_kib = page_size / MI_KiB;\n if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys <= SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) {\n *physical_memory_in_kib = (size_t)pphys * psize_in_kib;\n }\n #endif\n}\n\nvoid _mi_prim_mem_init( mi_os_mem_config_t* config )\n{\n long psize = sysconf(_SC_PAGESIZE);\n if (psize > 0 && (unsigned long)psize < SIZE_MAX) {\n config->page_size = (size_t)psize;\n config->alloc_granularity = (size_t)psize;\n unix_detect_physical_memory(config->page_size, &config->physical_memory_in_kib);\n }\n config->large_page_size = MI_UNIX_LARGE_PAGE_SIZE;\n config->has_overcommit = unix_detect_overcommit();\n config->has_partial_free = true; // mmap can free in parts\n config->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE)\n\n // disable transparent huge pages for this process?\n #if (defined(__linux__) || defined(__ANDROID__)) && defined(PR_GET_THP_DISABLE)\n if (!mi_option_is_enabled(mi_option_allow_thp)) // disable THP if requested through an option\n {\n if (prctl(PR_GET_THP_DISABLE, 0, 0, 0, 0) == 0) { // -1 on error, 1 if already disabled\n // Most likely since distros often come with always/madvise settings.\n // Disabling only for mimalloc process rather than touching system wide settings\n (void)prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0);\n }\n }\n #endif\n}\n\n\n//---------------------------------------------\n// free\n//---------------------------------------------\n\nint _mi_prim_free(void* addr, size_t size ) {\n if (size==0) return 0;\n bool err = (munmap(addr, size) == -1);\n return (err ? errno : 0);\n}\n\n\n//---------------------------------------------\n// mmap\n//---------------------------------------------\n\n// return errno on failure\nstatic int unix_madvise(void* addr, size_t size, int advice) {\n #if defined(__sun)\n const int res = madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520)\n return (res==0 ? 0 : errno);\n #elif defined(__QNX__)\n return posix_madvise(addr, size, advice); // posix returns errno\n #else\n const int res = madvise(addr, size, advice); // linux returns -1 on failure and sets errno\n return (res==0 ? 0 : errno);\n #endif\n}\n\nstatic void* unix_mmap_prim(void* addr, size_t size, int protect_flags, int flags, int fd) {\n void* p = mmap(addr, size, protect_flags, flags, fd, 0 /* offset */);\n #if defined(__linux__) && defined(PR_SET_VMA)\n if (p!=MAP_FAILED && p!=NULL) {\n prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, p, size, \"mimalloc\");\n }\n #endif\n return p;\n}\n\nstatic void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {\n MI_UNUSED(try_alignment);\n void* p = NULL;\n #if defined(MAP_ALIGNED) // BSD\n if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {\n size_t n = mi_bsr(try_alignment);\n if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB\n p = unix_mmap_prim(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd);\n if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {\n int err = errno;\n _mi_trace_message(\"unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\\n\", err, err, size, try_alignment, addr);\n }\n if (p!=MAP_FAILED) return p;\n // fall back to regular mmap\n }\n }\n #elif defined(MAP_ALIGN) // Solaris\n if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {\n p = unix_mmap_prim((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd); // addr parameter is the required alignment\n if (p!=MAP_FAILED) return p;\n // fall back to regular mmap\n }\n #endif\n #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)\n // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations\n if (addr == NULL) {\n void* hint = _mi_os_get_aligned_hint(try_alignment, size);\n if (hint != NULL) {\n p = unix_mmap_prim(hint, size, protect_flags, flags, fd);\n if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {\n #if MI_TRACK_ENABLED // asan sometimes does not instrument errno correctly?\n int err = 0;\n #else\n int err = errno;\n #endif\n _mi_trace_message(\"unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\\n\", err, err, size, try_alignment, hint);\n }\n if (p!=MAP_FAILED) return p;\n // fall back to regular mmap\n }\n }\n #endif\n // regular mmap\n p = unix_mmap_prim(addr, size, protect_flags, flags, fd);\n if (p!=MAP_FAILED) return p;\n // failed to allocate\n return NULL;\n}\n\nstatic int unix_mmap_fd(void) {\n #if defined(VM_MAKE_TAG)\n // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)\n int os_tag = (int)mi_option_get(mi_option_os_tag);\n if (os_tag < 100 || os_tag > 255) { os_tag = 254; }\n return VM_MAKE_TAG(os_tag);\n #else\n return -1;\n #endif\n}\n\nstatic void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {\n #if !defined(MAP_ANONYMOUS)\n #define MAP_ANONYMOUS MAP_ANON\n #endif\n #if !defined(MAP_NORESERVE)\n #define MAP_NORESERVE 0\n #endif\n void* p = NULL;\n const int fd = unix_mmap_fd();\n int flags = MAP_PRIVATE | MAP_ANONYMOUS;\n if (_mi_os_has_overcommit()) {\n flags |= MAP_NORESERVE;\n }\n #if defined(PROT_MAX)\n protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD\n #endif\n // huge page allocation\n if (allow_large && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) {\n static _Atomic(size_t) large_page_try_ok; // = 0;\n size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);\n if (!large_only && try_ok > 0) {\n // If the OS is not configured for large OS pages, or the user does not have\n // enough permission, the `mmap` will always fail (but it might also fail for other reasons).\n // Therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times\n // to avoid too many failing calls to mmap.\n mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1);\n }\n else {\n int lflags = flags & ~MAP_NORESERVE; // using NORESERVE on huge pages seems to fail on Linux\n int lfd = fd;\n #ifdef MAP_ALIGNED_SUPER\n lflags |= MAP_ALIGNED_SUPER;\n #endif\n #ifdef MAP_HUGETLB\n lflags |= MAP_HUGETLB;\n #endif\n #ifdef MAP_HUGE_1GB\n static bool mi_huge_pages_available = true;\n if (large_only && (size % MI_GiB) == 0 && mi_huge_pages_available) {\n lflags |= MAP_HUGE_1GB;\n }\n else\n #endif\n {\n #ifdef MAP_HUGE_2MB\n lflags |= MAP_HUGE_2MB;\n #endif\n }\n #ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB\n lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;\n #endif\n if (large_only || lflags != flags) {\n // try large OS page allocation\n *is_large = true;\n p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);\n #ifdef MAP_HUGE_1GB\n if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) {\n mi_huge_pages_available = false; // don't try huge 1GiB pages again\n if (large_only) {\n _mi_warning_message(\"unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\\n\", errno);\n }\n lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);\n p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);\n }\n #endif\n if (large_only) return p;\n if (p == NULL) {\n mi_atomic_store_release(&large_page_try_ok, (size_t)8); // on error, don't try again for the next N allocations\n }\n }\n }\n }\n // regular allocation\n if (p == NULL) {\n *is_large = false;\n p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd);\n #if !defined(MI_NO_THP)\n if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) {\n #if defined(MADV_HUGEPAGE)\n // Many Linux systems don't allow MAP_HUGETLB but they support instead\n // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE\n // though since properly aligned allocations will already use large pages if available\n // in that case -- in particular for our large regions (in `memory.c`).\n // However, some systems only allow THP if called with explicit `madvise`, so\n // when large OS pages are enabled for mimalloc, we call `madvise` anyways.\n if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {\n // *is_large = true; // possibly\n };\n #elif defined(__sun)\n struct memcntl_mha cmd = {0};\n cmd.mha_pagesize = _mi_os_large_page_size();\n cmd.mha_cmd = MHA_MAPSIZE_VA;\n if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {\n // *is_large = true; // possibly\n }\n #endif\n }\n #endif\n }\n return p;\n}\n\n// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.\nint _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {\n mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);\n mi_assert_internal(commit || !allow_large);\n mi_assert_internal(try_alignment > 0);\n *is_zero = true;\n int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);\n *addr = unix_mmap(hint_addr, size, try_alignment, protect_flags, false, allow_large, is_large);\n return (*addr != NULL ? 0 : errno);\n}\n\n\n//---------------------------------------------\n// Commit/Reset\n//---------------------------------------------\n\nstatic void unix_mprotect_hint(int err) {\n #if defined(__linux__) && (MI_SECURE>=5) // guard page around every mimalloc page\n if (err == ENOMEM) {\n _mi_warning_message(\"The next warning may be caused by a low memory map limit.\\n\"\n \" On Linux this is controlled by the vm.max_map_count -- maybe increase it?\\n\"\n \" For example: sudo sysctl -w vm.max_map_count=262144\\n\");\n }\n #else\n MI_UNUSED(err);\n #endif\n}\n\nint _mi_prim_commit(void* start, size_t size, bool* is_zero) {\n // commit: ensure we can access the area\n // note: we may think that *is_zero can be true since the memory\n // was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but\n // we sometimes call commit on a range with still partially committed\n // memory and `mprotect` does not zero the range.\n *is_zero = false;\n int err = mprotect(start, size, (PROT_READ | PROT_WRITE));\n if (err != 0) {\n err = errno;\n unix_mprotect_hint(err);\n }\n return err;\n}\n\nint _mi_prim_reuse(void* start, size_t size) {\n MI_UNUSED(start); MI_UNUSED(size);\n #if defined(__APPLE__) && defined(MADV_FREE_REUSE)\n return unix_madvise(start, size, MADV_FREE_REUSE);\n #endif\n return 0;\n}\n\nint _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {\n int err = 0;\n #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)\n // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097)\n err = unix_madvise(start, size, MADV_FREE_REUSABLE);\n if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }\n #else\n // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)\n err = unix_madvise(start, size, MADV_DONTNEED);\n #endif\n #if !MI_DEBUG && MI_SECURE<=2\n *needs_recommit = false;\n #else\n *needs_recommit = true;\n mprotect(start, size, PROT_NONE);\n #endif\n /*\n // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss)\n *needs_recommit = true;\n const int fd = unix_mmap_fd();\n void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);\n if (p != start) { err = errno; }\n */\n return err;\n}\n\nint _mi_prim_reset(void* start, size_t size) {\n int err = 0;\n\n // on macOS can use MADV_FREE_REUSABLE (but we disable this for now as it seems slower)\n #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE)\n err = unix_madvise(start, size, MADV_FREE_REUSABLE);\n if (err==0) return 0;\n // fall through\n #endif\n\n #if defined(MADV_FREE)\n // Otherwise, we try to use `MADV_FREE` as that is the fastest. A drawback though is that it\n // will not reduce the `rss` stats in tools like `top` even though the memory is available\n // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by\n // default `MADV_DONTNEED` is used though.\n static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE);\n int oadvice = (int)mi_atomic_load_relaxed(&advice);\n while ((err = unix_madvise(start, size, oadvice)) != 0 && err == EAGAIN) { /* try again */ };\n if (err == EINVAL && oadvice == MADV_FREE) {\n // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on\n mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED);\n err = unix_madvise(start, size, MADV_DONTNEED);\n }\n #else\n err = unix_madvise(start, size, MADV_DONTNEED);\n #endif\n return err;\n}\n\nint _mi_prim_protect(void* start, size_t size, bool protect) {\n int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE));\n if (err != 0) { err = errno; }\n unix_mprotect_hint(err);\n return err;\n}\n\n\n\n//---------------------------------------------\n// Huge page allocation\n//---------------------------------------------\n\n#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__)\n\n#ifndef MPOL_PREFERRED\n#define MPOL_PREFERRED 1\n#endif\n\n#if defined(MI_HAS_SYSCALL_H) && defined(SYS_mbind)\nstatic long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {\n return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags);\n}\n#else\nstatic long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {\n MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags);\n return 0;\n}\n#endif\n\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {\n bool is_large = true;\n *is_zero = true;\n *addr = unix_mmap(hint_addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large);\n if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes\n unsigned long numa_mask = (1UL << numa_node);\n // TODO: does `mbind` work correctly for huge OS pages? should we\n // use `set_mempolicy` before calling mmap instead?\n // see: \n long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0);\n if (err != 0) {\n err = errno;\n _mi_warning_message(\"failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\\n\", numa_node, err, err);\n }\n }\n return (*addr != NULL ? 0 : errno);\n}\n\n#else\n\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {\n MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);\n *is_zero = false;\n *addr = NULL;\n return ENOMEM;\n}\n\n#endif\n\n//---------------------------------------------\n// NUMA nodes\n//---------------------------------------------\n\n#if defined(__linux__)\n\nsize_t _mi_prim_numa_node(void) {\n #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu)\n unsigned long node = 0;\n unsigned long ncpu = 0;\n long err = syscall(SYS_getcpu, &ncpu, &node, NULL);\n if (err != 0) return 0;\n return node;\n #else\n return 0;\n #endif\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n char buf[128];\n unsigned node = 0;\n for(node = 0; node < 256; node++) {\n // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)\n _mi_snprintf(buf, 127, \"/sys/devices/system/node/node%u\", node + 1);\n if (mi_prim_access(buf,R_OK) != 0) break;\n }\n return (node+1);\n}\n\n#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000\n\nsize_t _mi_prim_numa_node(void) {\n domainset_t dom;\n size_t node;\n int policy;\n if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul;\n for (node = 0; node < MAXMEMDOM; node++) {\n if (DOMAINSET_ISSET(node, &dom)) return node;\n }\n return 0ul;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n size_t ndomains = 0;\n size_t len = sizeof(ndomains);\n if (sysctlbyname(\"vm.ndomains\", &ndomains, &len, NULL, 0) == -1) return 0ul;\n return ndomains;\n}\n\n#elif defined(__DragonFly__)\n\nsize_t _mi_prim_numa_node(void) {\n // TODO: DragonFly does not seem to provide any userland means to get this information.\n return 0ul;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n size_t ncpus = 0, nvirtcoresperphys = 0;\n size_t len = sizeof(size_t);\n if (sysctlbyname(\"hw.ncpu\", &ncpus, &len, NULL, 0) == -1) return 0ul;\n if (sysctlbyname(\"hw.cpu_topology_ht_ids\", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul;\n return nvirtcoresperphys * ncpus;\n}\n\n#else\n\nsize_t _mi_prim_numa_node(void) {\n return 0;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n return 1;\n}\n\n#endif\n\n// ----------------------------------------------------------------\n// Clock\n// ----------------------------------------------------------------\n\n#include \n\n#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)\n\nmi_msecs_t _mi_prim_clock_now(void) {\n struct timespec t;\n #ifdef CLOCK_MONOTONIC\n clock_gettime(CLOCK_MONOTONIC, &t);\n #else\n clock_gettime(CLOCK_REALTIME, &t);\n #endif\n return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);\n}\n\n#else\n\n// low resolution timer\nmi_msecs_t _mi_prim_clock_now(void) {\n #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)\n return (mi_msecs_t)clock();\n #elif (CLOCKS_PER_SEC < 1000)\n return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);\n #else\n return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);\n #endif\n}\n\n#endif\n\n\n\n\n//----------------------------------------------------------------\n// Process info\n//----------------------------------------------------------------\n\n#if defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__)\n#include \n#include \n#include \n\n#if defined(__APPLE__)\n#include \n#endif\n\n#if defined(__HAIKU__)\n#include \n#endif\n\nstatic mi_msecs_t timeval_secs(const struct timeval* tv) {\n return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L);\n}\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo)\n{\n struct rusage rusage;\n getrusage(RUSAGE_SELF, &rusage);\n pinfo->utime = timeval_secs(&rusage.ru_utime);\n pinfo->stime = timeval_secs(&rusage.ru_stime);\n#if !defined(__HAIKU__)\n pinfo->page_faults = rusage.ru_majflt;\n#endif\n#if defined(__HAIKU__)\n // Haiku does not have (yet?) a way to\n // get these stats per process\n thread_info tid;\n area_info mem;\n ssize_t c;\n get_thread_info(find_thread(0), &tid);\n while (get_next_area_info(tid.team, &c, &mem) == B_OK) {\n pinfo->peak_rss += mem.ram_size;\n }\n pinfo->page_faults = 0;\n#elif defined(__APPLE__)\n pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes\n #ifdef MACH_TASK_BASIC_INFO\n struct mach_task_basic_info info;\n mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;\n if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {\n pinfo->current_rss = (size_t)info.resident_size;\n }\n #else\n struct task_basic_info info;\n mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT;\n if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {\n pinfo->current_rss = (size_t)info.resident_size;\n }\n #endif\n#else\n pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB\n#endif\n // use defaults for commit\n}\n\n#else\n\n#ifndef __wasi__\n// WebAssembly instances are not processes\n#pragma message(\"define a way to get process info\")\n#endif\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo)\n{\n // use defaults\n MI_UNUSED(pinfo);\n}\n\n#endif\n\n\n//----------------------------------------------------------------\n// Output\n//----------------------------------------------------------------\n\nvoid _mi_prim_out_stderr( const char* msg ) {\n fputs(msg,stderr);\n}\n\n\n//----------------------------------------------------------------\n// Environment\n//----------------------------------------------------------------\n\n#if !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0)\n// On Posix systemsr use `environ` to access environment variables\n// even before the C runtime is initialized.\n#if defined(__APPLE__) && defined(__has_include) && __has_include()\n#include \nstatic char** mi_get_environ(void) {\n return (*_NSGetEnviron());\n}\n#else\nextern char** environ;\nstatic char** mi_get_environ(void) {\n return environ;\n}\n#endif\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size) {\n if (name==NULL) return false;\n const size_t len = _mi_strlen(name);\n if (len == 0) return false;\n char** env = mi_get_environ();\n if (env == NULL) return false;\n // compare up to 10000 entries\n for (int i = 0; i < 10000 && env[i] != NULL; i++) {\n const char* s = env[i];\n if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive\n // found it\n _mi_strlcpy(result, s + len + 1, result_size);\n return true;\n }\n }\n return false;\n}\n#else\n// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size) {\n // cannot call getenv() when still initializing the C runtime.\n if (_mi_preloading()) return false;\n const char* s = getenv(name);\n if (s == NULL) {\n // we check the upper case name too.\n char buf[64+1];\n size_t len = _mi_strnlen(name,sizeof(buf)-1);\n for (size_t i = 0; i < len; i++) {\n buf[i] = _mi_toupper(name[i]);\n }\n buf[len] = 0;\n s = getenv(buf);\n }\n if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false;\n _mi_strlcpy(result, s, result_size);\n return true;\n}\n#endif // !MI_USE_ENVIRON\n\n\n//----------------------------------------------------------------\n// Random\n//----------------------------------------------------------------\n\n#if defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_15)\n#include \n#include \n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n // We prefer CCRandomGenerateBytes as it returns an error code while arc4random_buf\n // may fail silently on macOS. See PR #390, and \n return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess);\n}\n\n#elif defined(__ANDROID__) || defined(__DragonFly__) || \\\n defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \\\n defined(__sun) || \\\n (defined(__APPLE__) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_7))\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n arc4random_buf(buf, buf_len);\n return true;\n}\n\n#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) // also for old apple versions < 10.7 (issue #829)\n\n#include \n#include \n#include \n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n // Modern Linux provides `getrandom` but different distributions either use `sys/random.h` or `linux/random.h`\n // and for the latter the actual `getrandom` call is not always defined.\n // (see )\n // We therefore use a syscall directly and fall back dynamically to /dev/urandom when needed.\n #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getrandom)\n #ifndef GRND_NONBLOCK\n #define GRND_NONBLOCK (1)\n #endif\n static _Atomic(uintptr_t) no_getrandom; // = 0\n if (mi_atomic_load_acquire(&no_getrandom)==0) {\n ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK);\n if (ret >= 0) return (buf_len == (size_t)ret);\n if (errno != ENOSYS) return false;\n mi_atomic_store_release(&no_getrandom, (uintptr_t)1); // don't call again, and fall back to /dev/urandom\n }\n #endif\n int flags = O_RDONLY;\n #if defined(O_CLOEXEC)\n flags |= O_CLOEXEC;\n #endif\n int fd = mi_prim_open(\"/dev/urandom\", flags);\n if (fd < 0) return false;\n size_t count = 0;\n while(count < buf_len) {\n ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count);\n if (ret<=0) {\n if (errno!=EAGAIN && errno!=EINTR) break;\n }\n else {\n count += ret;\n }\n }\n mi_prim_close(fd);\n return (count==buf_len);\n}\n\n#else\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n return false;\n}\n\n#endif\n\n\n//----------------------------------------------------------------\n// Thread init/done\n//----------------------------------------------------------------\n\n#if defined(MI_USE_PTHREADS)\n\n// use pthread local storage keys to detect thread ending\n// (and used with MI_TLS_PTHREADS for the default heap)\npthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);\n\nstatic void mi_pthread_done(void* value) {\n if (value!=NULL) {\n _mi_thread_done((mi_heap_t*)value);\n }\n}\n\nvoid _mi_prim_thread_init_auto_done(void) {\n mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));\n pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);\n}\n\nvoid _mi_prim_thread_done_auto_done(void) {\n if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809\n pthread_key_delete(_mi_heap_default_key);\n }\n}\n\nvoid _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {\n if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD\n pthread_setspecific(_mi_heap_default_key, heap);\n }\n}\n\n#else\n\nvoid _mi_prim_thread_init_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_done_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {\n MI_UNUSED(heap);\n}\n\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/prim/wasi/prim.c", "sha256": "05a23fbcf2d96c693aaa4d9222780f3551c968356ee19d6d7d5dfdc6242cf049", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// This file is included in `src/prim/prim.c`\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\"\n\n#include // fputs\n#include // getenv\n\n//---------------------------------------------\n// Initialize\n//---------------------------------------------\n\nvoid _mi_prim_mem_init( mi_os_mem_config_t* config ) {\n config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB\n config->alloc_granularity = 16;\n config->has_overcommit = false;\n config->has_partial_free = false;\n config->has_virtual_reserve = false;\n}\n\n//---------------------------------------------\n// Free\n//---------------------------------------------\n\nint _mi_prim_free(void* addr, size_t size ) {\n MI_UNUSED(addr); MI_UNUSED(size);\n // wasi heap cannot be shrunk\n return 0;\n}\n\n\n//---------------------------------------------\n// Allocation: sbrk or memory_grow\n//---------------------------------------------\n\n#if defined(MI_USE_SBRK)\n #include // for sbrk\n\n static void* mi_memory_grow( size_t size ) {\n void* p = sbrk(size);\n if (p == (void*)(-1)) return NULL;\n #if !defined(__wasi__) // on wasi this is always zero initialized already (?)\n memset(p,0,size);\n #endif\n return p;\n }\n#elif defined(__wasi__)\n static void* mi_memory_grow( size_t size ) {\n size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size()))\n : __builtin_wasm_memory_size(0));\n if (base == SIZE_MAX) return NULL;\n return (void*)(base * _mi_os_page_size());\n }\n#endif\n\n#if defined(MI_USE_PTHREADS)\nstatic pthread_mutex_t mi_heap_grow_mutex = PTHREAD_MUTEX_INITIALIZER;\n#endif\n\nstatic void* mi_prim_mem_grow(size_t size, size_t try_alignment) {\n void* p = NULL;\n if (try_alignment <= 1) {\n // `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now)\n #if defined(MI_USE_PTHREADS)\n pthread_mutex_lock(&mi_heap_grow_mutex);\n #endif\n p = mi_memory_grow(size);\n #if defined(MI_USE_PTHREADS)\n pthread_mutex_unlock(&mi_heap_grow_mutex);\n #endif\n }\n else {\n void* base = NULL;\n size_t alloc_size = 0;\n // to allocate aligned use a lock to try to avoid thread interaction\n // between getting the current size and actual allocation\n // (also, `sbrk` is not thread safe in general)\n #if defined(MI_USE_PTHREADS)\n pthread_mutex_lock(&mi_heap_grow_mutex);\n #endif\n {\n void* current = mi_memory_grow(0); // get current size\n if (current != NULL) {\n void* aligned_current = _mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space\n alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size());\n base = mi_memory_grow(alloc_size);\n }\n }\n #if defined(MI_USE_PTHREADS)\n pthread_mutex_unlock(&mi_heap_grow_mutex);\n #endif\n if (base != NULL) {\n p = _mi_align_up_ptr(base, try_alignment);\n if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) {\n // another thread used wasm_memory_grow/sbrk in-between and we do not have enough\n // space after alignment. Give up (and waste the space as we cannot shrink :-( )\n // (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align)\n p = NULL;\n }\n }\n }\n /*\n if (p == NULL) {\n _mi_warning_message(\"unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\\n\", size, try_alignment);\n errno = ENOMEM;\n return NULL;\n }\n */\n mi_assert_internal( p == NULL || try_alignment == 0 || (uintptr_t)p % try_alignment == 0 );\n return p;\n}\n\n// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.\nint _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {\n MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);\n *is_large = false;\n *is_zero = false;\n *addr = mi_prim_mem_grow(size, try_alignment);\n return (*addr != NULL ? 0 : ENOMEM);\n}\n\n\n//---------------------------------------------\n// Commit/Reset/Protect\n//---------------------------------------------\n\nint _mi_prim_commit(void* addr, size_t size, bool* is_zero) {\n MI_UNUSED(addr); MI_UNUSED(size);\n *is_zero = false;\n return 0;\n}\n\nint _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {\n MI_UNUSED(addr); MI_UNUSED(size);\n *needs_recommit = false;\n return 0;\n}\n\nint _mi_prim_reset(void* addr, size_t size) {\n MI_UNUSED(addr); MI_UNUSED(size);\n return 0;\n}\n\nint _mi_prim_reuse(void* addr, size_t size) {\n MI_UNUSED(addr); MI_UNUSED(size);\n return 0;\n}\n\nint _mi_prim_protect(void* addr, size_t size, bool protect) {\n MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect);\n return 0;\n}\n\n\n//---------------------------------------------\n// Huge pages and NUMA nodes\n//---------------------------------------------\n\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {\n MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);\n *is_zero = true;\n *addr = NULL;\n return ENOSYS;\n}\n\nsize_t _mi_prim_numa_node(void) {\n return 0;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n return 1;\n}\n\n\n//----------------------------------------------------------------\n// Clock\n//----------------------------------------------------------------\n\n#include \n\n#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)\n\nmi_msecs_t _mi_prim_clock_now(void) {\n struct timespec t;\n #ifdef CLOCK_MONOTONIC\n clock_gettime(CLOCK_MONOTONIC, &t);\n #else\n clock_gettime(CLOCK_REALTIME, &t);\n #endif\n return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);\n}\n\n#else\n\n// low resolution timer\nmi_msecs_t _mi_prim_clock_now(void) {\n #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)\n return (mi_msecs_t)clock();\n #elif (CLOCKS_PER_SEC < 1000)\n return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);\n #else\n return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);\n #endif\n}\n\n#endif\n\n\n//----------------------------------------------------------------\n// Process info\n//----------------------------------------------------------------\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo)\n{\n // use defaults\n MI_UNUSED(pinfo);\n}\n\n\n//----------------------------------------------------------------\n// Output\n//----------------------------------------------------------------\n\nvoid _mi_prim_out_stderr( const char* msg ) {\n fputs(msg,stderr);\n}\n\n\n//----------------------------------------------------------------\n// Environment\n//----------------------------------------------------------------\n\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size) {\n // cannot call getenv() when still initializing the C runtime.\n if (_mi_preloading()) return false;\n const char* s = getenv(name);\n if (s == NULL) {\n // we check the upper case name too.\n char buf[64+1];\n size_t len = _mi_strnlen(name,sizeof(buf)-1);\n for (size_t i = 0; i < len; i++) {\n buf[i] = _mi_toupper(name[i]);\n }\n buf[len] = 0;\n s = getenv(buf);\n }\n if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false;\n _mi_strlcpy(result, s, result_size);\n return true;\n}\n\n\n//----------------------------------------------------------------\n// Random\n//----------------------------------------------------------------\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n return false;\n}\n\n\n//----------------------------------------------------------------\n// Thread init/done\n//----------------------------------------------------------------\n\nvoid _mi_prim_thread_init_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_done_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {\n MI_UNUSED(heap);\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/prim/windows/etw.h", "sha256": "4f3110ef2054c95cb275be96cb279224d3d46a728c5117bd1435425f382de778", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "//**********************************************************************`\n//* This is an include file generated by Message Compiler. *`\n//* *`\n//* Copyright (c) Microsoft Corporation. All Rights Reserved. *`\n//**********************************************************************`\n#pragma once\n\n//*****************************************************************************\n//\n// Notes on the ETW event code generated by MC:\n//\n// - Structures and arrays of structures are treated as an opaque binary blob.\n// The caller is responsible for packing the data for the structure into a\n// single region of memory, with no padding between values. The macro will\n// have an extra parameter for the length of the blob.\n// - Arrays of nul-terminated strings must be packed by the caller into a\n// single binary blob containing the correct number of strings, with a nul\n// after each string. The size of the blob is specified in characters, and\n// includes the final nul.\n// - Arrays of SID are treated as a single binary blob. The caller is\n// responsible for packing the SID values into a single region of memory with\n// no padding.\n// - The length attribute on the data element in the manifest is significant\n// for values with intype win:UnicodeString, win:AnsiString, or win:Binary.\n// The length attribute must be specified for win:Binary, and is optional for\n// win:UnicodeString and win:AnsiString (if no length is given, the strings\n// are assumed to be nul-terminated). For win:UnicodeString, the length is\n// measured in characters, not bytes.\n// - For an array of win:UnicodeString, win:AnsiString, or win:Binary, the\n// length attribute applies to every value in the array, so every value in\n// the array must have the same length. The values in the array are provided\n// to the macro via a single pointer -- the caller is responsible for packing\n// all of the values into a single region of memory with no padding between\n// values.\n// - Values of type win:CountedUnicodeString, win:CountedAnsiString, and\n// win:CountedBinary can be generated and collected on Vista or later.\n// However, they may not decode properly without the Windows 10 2018 Fall\n// Update.\n// - Arrays of type win:CountedUnicodeString, win:CountedAnsiString, and\n// win:CountedBinary must be packed by the caller into a single region of\n// memory. The format for each item is a UINT16 byte-count followed by that\n// many bytes of data. When providing the array to the generated macro, you\n// must provide the total size of the packed array data, including the UINT16\n// sizes for each item. In the case of win:CountedUnicodeString, the data\n// size is specified in WCHAR (16-bit) units. In the case of\n// win:CountedAnsiString and win:CountedBinary, the data size is specified in\n// bytes.\n//\n//*****************************************************************************\n\n#include \n#include \n#include \n\n#ifndef ETW_INLINE\n #ifdef _ETW_KM_\n // In kernel mode, save stack space by never inlining templates.\n #define ETW_INLINE DECLSPEC_NOINLINE __inline\n #else\n // In user mode, save code size by inlining templates as appropriate.\n #define ETW_INLINE __inline\n #endif\n#endif // ETW_INLINE\n\n#if defined(__cplusplus)\nextern \"C\" {\n#endif\n\n//\n// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:\n// Define this macro to have the compiler skip the generated functions in this\n// header.\n//\n#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n//\n// MCGEN_USE_KERNEL_MODE_APIS macro:\n// Controls whether the generated code uses kernel-mode or user-mode APIs.\n// - Set to 0 to use Windows user-mode APIs such as EventRegister.\n// - Set to 1 to use Windows kernel-mode APIs such as EtwRegister.\n// Default is based on whether the _ETW_KM_ macro is defined (i.e. by wdm.h).\n// Note that the APIs can also be overridden directly, e.g. by setting the\n// MCGEN_EVENTWRITETRANSFER or MCGEN_EVENTREGISTER macros.\n//\n#ifndef MCGEN_USE_KERNEL_MODE_APIS\n #ifdef _ETW_KM_\n #define MCGEN_USE_KERNEL_MODE_APIS 1\n #else\n #define MCGEN_USE_KERNEL_MODE_APIS 0\n #endif\n#endif // MCGEN_USE_KERNEL_MODE_APIS\n\n//\n// MCGEN_HAVE_EVENTSETINFORMATION macro:\n// Controls how McGenEventSetInformation uses the EventSetInformation API.\n// - Set to 0 to disable the use of EventSetInformation\n// (McGenEventSetInformation will always return an error).\n// - Set to 1 to directly invoke MCGEN_EVENTSETINFORMATION.\n// - Set to 2 to to locate EventSetInformation at runtime via GetProcAddress\n// (user-mode) or MmGetSystemRoutineAddress (kernel-mode).\n// Default is determined as follows:\n// - If MCGEN_EVENTSETINFORMATION has been customized, set to 1\n// (i.e. use MCGEN_EVENTSETINFORMATION).\n// - Else if the target OS version has EventSetInformation, set to 1\n// (i.e. use MCGEN_EVENTSETINFORMATION).\n// - Else set to 2 (i.e. try to dynamically locate EventSetInformation).\n// Note that an McGenEventSetInformation function will only be generated if one\n// or more provider in a manifest has provider traits.\n//\n#ifndef MCGEN_HAVE_EVENTSETINFORMATION\n #ifdef MCGEN_EVENTSETINFORMATION // if MCGEN_EVENTSETINFORMATION has been customized,\n #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...).\n #elif MCGEN_USE_KERNEL_MODE_APIS // else if using kernel-mode APIs,\n #if NTDDI_VERSION >= 0x06040000 // if target OS is Windows 10 or later,\n #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...).\n #else // else\n #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find \"EtwSetInformation\" via MmGetSystemRoutineAddress.\n #endif // else (using user-mode APIs)\n #else // if target OS and SDK is Windows 8 or later,\n #if WINVER >= 0x0602 && defined(EVENT_FILTER_TYPE_SCHEMATIZED)\n #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...).\n #else // else\n #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find \"EventSetInformation\" via GetModuleHandleExW/GetProcAddress.\n #endif\n #endif\n#endif // MCGEN_HAVE_EVENTSETINFORMATION\n\n//\n// MCGEN Override Macros\n//\n// The following override macros may be defined before including this header\n// to control the APIs used by this header:\n//\n// - MCGEN_EVENTREGISTER\n// - MCGEN_EVENTUNREGISTER\n// - MCGEN_EVENTSETINFORMATION\n// - MCGEN_EVENTWRITETRANSFER\n//\n// If the the macro is undefined, the MC implementation will default to the\n// corresponding ETW APIs. For example, if the MCGEN_EVENTREGISTER macro is\n// undefined, the EventRegister[MyProviderName] macro will use EventRegister\n// in user mode and will use EtwRegister in kernel mode.\n//\n// To prevent issues from conflicting definitions of these macros, the value\n// of the override macro will be used as a suffix in certain internal function\n// names. Because of this, the override macros must follow certain rules:\n//\n// - The macro must be defined before any MC-generated header is included and\n// must not be undefined or redefined after any MC-generated header is\n// included. Different translation units (i.e. different .c or .cpp files)\n// may set the macros to different values, but within a translation unit\n// (within a single .c or .cpp file), the macro must be set once and not\n// changed.\n// - The override must be an object-like macro, not a function-like macro\n// (i.e. the override macro must not have a parameter list).\n// - The override macro's value must be a simple identifier, i.e. must be\n// something that starts with a letter or '_' and contains only letters,\n// numbers, and '_' characters.\n// - If the override macro's value is the name of a second object-like macro,\n// the second object-like macro must follow the same rules. (The override\n// macro's value can also be the name of a function-like macro, in which\n// case the function-like macro does not need to follow the same rules.)\n//\n// For example, the following will cause compile errors:\n//\n// #define MCGEN_EVENTWRITETRANSFER MyNamespace::MyClass::MyFunction // Value has non-identifier characters (colon).\n// #define MCGEN_EVENTWRITETRANSFER GetEventWriteFunctionPointer(7) // Value has non-identifier characters (parentheses).\n// #define MCGEN_EVENTWRITETRANSFER(h,e,a,r,c,d) EventWrite(h,e,c,d) // Override is defined as a function-like macro.\n// #define MY_OBJECT_LIKE_MACRO MyNamespace::MyClass::MyEventWriteFunction\n// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // Evaluates to something with non-identifier characters (colon).\n//\n// The following would be ok:\n//\n// #define MCGEN_EVENTWRITETRANSFER MyEventWriteFunction1 // OK, suffix will be \"MyEventWriteFunction1\".\n// #define MY_OBJECT_LIKE_MACRO MyEventWriteFunction2\n// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // OK, suffix will be \"MyEventWriteFunction2\".\n// #define MY_FUNCTION_LIKE_MACRO(h,e,a,r,c,d) MyNamespace::MyClass::MyEventWriteFunction3(h,e,c,d)\n// #define MCGEN_EVENTWRITETRANSFER MY_FUNCTION_LIKE_MACRO // OK, suffix will be \"MY_FUNCTION_LIKE_MACRO\".\n//\n#ifndef MCGEN_EVENTREGISTER\n #if MCGEN_USE_KERNEL_MODE_APIS\n #define MCGEN_EVENTREGISTER EtwRegister\n #else\n #define MCGEN_EVENTREGISTER EventRegister\n #endif\n#endif // MCGEN_EVENTREGISTER\n#ifndef MCGEN_EVENTUNREGISTER\n #if MCGEN_USE_KERNEL_MODE_APIS\n #define MCGEN_EVENTUNREGISTER EtwUnregister\n #else\n #define MCGEN_EVENTUNREGISTER EventUnregister\n #endif\n#endif // MCGEN_EVENTUNREGISTER\n#ifndef MCGEN_EVENTSETINFORMATION\n #if MCGEN_USE_KERNEL_MODE_APIS\n #define MCGEN_EVENTSETINFORMATION EtwSetInformation\n #else\n #define MCGEN_EVENTSETINFORMATION EventSetInformation\n #endif\n#endif // MCGEN_EVENTSETINFORMATION\n#ifndef MCGEN_EVENTWRITETRANSFER\n #if MCGEN_USE_KERNEL_MODE_APIS\n #define MCGEN_EVENTWRITETRANSFER EtwWriteTransfer\n #else\n #define MCGEN_EVENTWRITETRANSFER EventWriteTransfer\n #endif\n#endif // MCGEN_EVENTWRITETRANSFER\n\n//\n// MCGEN_EVENT_ENABLED macro:\n// Override to control how the EventWrite[EventName] macros determine whether\n// an event is enabled. The default behavior is for EventWrite[EventName] to\n// use the EventEnabled[EventName] macros.\n//\n#ifndef MCGEN_EVENT_ENABLED\n#define MCGEN_EVENT_ENABLED(EventName) EventEnabled##EventName()\n#endif\n\n//\n// MCGEN_EVENT_ENABLED_FORCONTEXT macro:\n// Override to control how the EventWrite[EventName]_ForContext macros\n// determine whether an event is enabled. The default behavior is for\n// EventWrite[EventName]_ForContext to use the\n// EventEnabled[EventName]_ForContext macros.\n//\n#ifndef MCGEN_EVENT_ENABLED_FORCONTEXT\n#define MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, EventName) EventEnabled##EventName##_ForContext(pContext)\n#endif\n\n//\n// MCGEN_ENABLE_CHECK macro:\n// Determines whether the specified event would be considered as enabled\n// based on the state of the specified context. Slightly faster than calling\n// McGenEventEnabled directly.\n//\n#ifndef MCGEN_ENABLE_CHECK\n#define MCGEN_ENABLE_CHECK(Context, Descriptor) (Context.IsEnabled && McGenEventEnabled(&Context, &Descriptor))\n#endif\n\n#if !defined(MCGEN_TRACE_CONTEXT_DEF)\n#define MCGEN_TRACE_CONTEXT_DEF\n// This structure is for use by MC-generated code and should not be used directly.\ntypedef struct _MCGEN_TRACE_CONTEXT\n{\n TRACEHANDLE RegistrationHandle;\n TRACEHANDLE Logger; // Used as pointer to provider traits.\n ULONGLONG MatchAnyKeyword;\n ULONGLONG MatchAllKeyword;\n ULONG Flags;\n ULONG IsEnabled;\n UCHAR Level;\n UCHAR Reserve;\n USHORT EnableBitsCount;\n PULONG EnableBitMask;\n const ULONGLONG* EnableKeyWords;\n const UCHAR* EnableLevel;\n} MCGEN_TRACE_CONTEXT, *PMCGEN_TRACE_CONTEXT;\n#endif // MCGEN_TRACE_CONTEXT_DEF\n\n#if !defined(MCGEN_LEVEL_KEYWORD_ENABLED_DEF)\n#define MCGEN_LEVEL_KEYWORD_ENABLED_DEF\n//\n// Determines whether an event with a given Level and Keyword would be\n// considered as enabled based on the state of the specified context.\n// Note that you may want to use MCGEN_ENABLE_CHECK instead of calling this\n// function directly.\n//\nFORCEINLINE\nBOOLEAN\nMcGenLevelKeywordEnabled(\n _In_ PMCGEN_TRACE_CONTEXT EnableInfo,\n _In_ UCHAR Level,\n _In_ ULONGLONG Keyword\n )\n{\n //\n // Check if the event Level is lower than the level at which\n // the channel is enabled.\n // If the event Level is 0 or the channel is enabled at level 0,\n // all levels are enabled.\n //\n\n if ((Level <= EnableInfo->Level) || // This also covers the case of Level == 0.\n (EnableInfo->Level == 0)) {\n\n //\n // Check if Keyword is enabled\n //\n\n if ((Keyword == (ULONGLONG)0) ||\n ((Keyword & EnableInfo->MatchAnyKeyword) &&\n ((Keyword & EnableInfo->MatchAllKeyword) == EnableInfo->MatchAllKeyword))) {\n return TRUE;\n }\n }\n\n return FALSE;\n}\n#endif // MCGEN_LEVEL_KEYWORD_ENABLED_DEF\n\n#if !defined(MCGEN_EVENT_ENABLED_DEF)\n#define MCGEN_EVENT_ENABLED_DEF\n//\n// Determines whether the specified event would be considered as enabled based\n// on the state of the specified context. Note that you may want to use\n// MCGEN_ENABLE_CHECK instead of calling this function directly.\n//\nFORCEINLINE\nBOOLEAN\nMcGenEventEnabled(\n _In_ PMCGEN_TRACE_CONTEXT EnableInfo,\n _In_ PCEVENT_DESCRIPTOR EventDescriptor\n )\n{\n return McGenLevelKeywordEnabled(EnableInfo, EventDescriptor->Level, EventDescriptor->Keyword);\n}\n#endif // MCGEN_EVENT_ENABLED_DEF\n\n#if !defined(MCGEN_CONTROL_CALLBACK)\n#define MCGEN_CONTROL_CALLBACK\n\n// This function is for use by MC-generated code and should not be used directly.\nDECLSPEC_NOINLINE __inline\nVOID\n__stdcall\nMcGenControlCallbackV2(\n _In_ LPCGUID SourceId,\n _In_ ULONG ControlCode,\n _In_ UCHAR Level,\n _In_ ULONGLONG MatchAnyKeyword,\n _In_ ULONGLONG MatchAllKeyword,\n _In_opt_ PEVENT_FILTER_DESCRIPTOR FilterData,\n _Inout_opt_ PVOID CallbackContext\n )\n/*++\n\nRoutine Description:\n\n This is the notification callback for Windows Vista and later.\n\nArguments:\n\n SourceId - The GUID that identifies the session that enabled the provider.\n\n ControlCode - The parameter indicates whether the provider\n is being enabled or disabled.\n\n Level - The level at which the event is enabled.\n\n MatchAnyKeyword - The bitmask of keywords that the provider uses to\n determine the category of events that it writes.\n\n MatchAllKeyword - This bitmask additionally restricts the category\n of events that the provider writes.\n\n FilterData - The provider-defined data.\n\n CallbackContext - The context of the callback that is defined when the provider\n called EtwRegister to register itself.\n\nRemarks:\n\n ETW calls this function to notify provider of enable/disable\n\n--*/\n{\n PMCGEN_TRACE_CONTEXT Ctx = (PMCGEN_TRACE_CONTEXT)CallbackContext;\n ULONG Ix;\n#ifndef MCGEN_PRIVATE_ENABLE_CALLBACK_V2\n UNREFERENCED_PARAMETER(SourceId);\n UNREFERENCED_PARAMETER(FilterData);\n#endif\n\n if (Ctx == NULL) {\n return;\n }\n\n switch (ControlCode) {\n\n case EVENT_CONTROL_CODE_ENABLE_PROVIDER:\n Ctx->Level = Level;\n Ctx->MatchAnyKeyword = MatchAnyKeyword;\n Ctx->MatchAllKeyword = MatchAllKeyword;\n Ctx->IsEnabled = EVENT_CONTROL_CODE_ENABLE_PROVIDER;\n\n for (Ix = 0; Ix < Ctx->EnableBitsCount; Ix += 1) {\n if (McGenLevelKeywordEnabled(Ctx, Ctx->EnableLevel[Ix], Ctx->EnableKeyWords[Ix]) != FALSE) {\n Ctx->EnableBitMask[Ix >> 5] |= (1 << (Ix % 32));\n } else {\n Ctx->EnableBitMask[Ix >> 5] &= ~(1 << (Ix % 32));\n }\n }\n break;\n\n case EVENT_CONTROL_CODE_DISABLE_PROVIDER:\n Ctx->IsEnabled = EVENT_CONTROL_CODE_DISABLE_PROVIDER;\n Ctx->Level = 0;\n Ctx->MatchAnyKeyword = 0;\n Ctx->MatchAllKeyword = 0;\n if (Ctx->EnableBitsCount > 0) {\n#pragma warning(suppress: 26451) // Arithmetic overflow cannot occur, no matter the value of EnableBitCount\n RtlZeroMemory(Ctx->EnableBitMask, (((Ctx->EnableBitsCount - 1) / 32) + 1) * sizeof(ULONG));\n }\n break;\n\n default:\n break;\n }\n\n#ifdef MCGEN_PRIVATE_ENABLE_CALLBACK_V2\n //\n // Call user defined callback\n //\n MCGEN_PRIVATE_ENABLE_CALLBACK_V2(\n SourceId,\n ControlCode,\n Level,\n MatchAnyKeyword,\n MatchAllKeyword,\n FilterData,\n CallbackContext\n );\n#endif // MCGEN_PRIVATE_ENABLE_CALLBACK_V2\n\n return;\n}\n\n#endif // MCGEN_CONTROL_CALLBACK\n\n#ifndef _mcgen_PENABLECALLBACK\n #if MCGEN_USE_KERNEL_MODE_APIS\n #define _mcgen_PENABLECALLBACK PETWENABLECALLBACK\n #else\n #define _mcgen_PENABLECALLBACK PENABLECALLBACK\n #endif\n#endif // _mcgen_PENABLECALLBACK\n\n#if !defined(_mcgen_PASTE2)\n// This macro is for use by MC-generated code and should not be used directly.\n#define _mcgen_PASTE2(a, b) _mcgen_PASTE2_imp(a, b)\n#define _mcgen_PASTE2_imp(a, b) a##b\n#endif // _mcgen_PASTE2\n\n#if !defined(_mcgen_PASTE3)\n// This macro is for use by MC-generated code and should not be used directly.\n#define _mcgen_PASTE3(a, b, c) _mcgen_PASTE3_imp(a, b, c)\n#define _mcgen_PASTE3_imp(a, b, c) a##b##_##c\n#endif // _mcgen_PASTE3\n\n//\n// Macro validation\n//\n\n// Validate MCGEN_EVENTREGISTER:\n\n// Trigger an error if MCGEN_EVENTREGISTER is not an unqualified (simple) identifier:\nstruct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER);\n\n// Trigger an error if MCGEN_EVENTREGISTER is redefined:\ntypedef struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER)\n MCGEN_EVENTREGISTER_must_not_be_redefined_between_headers;\n\n// Trigger an error if MCGEN_EVENTREGISTER is defined as a function-like macro:\ntypedef void MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTREGISTER;\ntypedef int _mcgen_PASTE2(MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTREGISTER);\n\n// Validate MCGEN_EVENTUNREGISTER:\n\n// Trigger an error if MCGEN_EVENTUNREGISTER is not an unqualified (simple) identifier:\nstruct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER);\n\n// Trigger an error if MCGEN_EVENTUNREGISTER is redefined:\ntypedef struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER)\n MCGEN_EVENTUNREGISTER_must_not_be_redefined_between_headers;\n\n// Trigger an error if MCGEN_EVENTUNREGISTER is defined as a function-like macro:\ntypedef void MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTUNREGISTER;\ntypedef int _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTUNREGISTER);\n\n// Validate MCGEN_EVENTSETINFORMATION:\n\n// Trigger an error if MCGEN_EVENTSETINFORMATION is not an unqualified (simple) identifier:\nstruct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION);\n\n// Trigger an error if MCGEN_EVENTSETINFORMATION is redefined:\ntypedef struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION)\n MCGEN_EVENTSETINFORMATION_must_not_be_redefined_between_headers;\n\n// Trigger an error if MCGEN_EVENTSETINFORMATION is defined as a function-like macro:\ntypedef void MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_MCGEN_EVENTSETINFORMATION;\ntypedef int _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_, MCGEN_EVENTSETINFORMATION);\n\n// Validate MCGEN_EVENTWRITETRANSFER:\n\n// Trigger an error if MCGEN_EVENTWRITETRANSFER is not an unqualified (simple) identifier:\nstruct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER);\n\n// Trigger an error if MCGEN_EVENTWRITETRANSFER is redefined:\ntypedef struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER)\n MCGEN_EVENTWRITETRANSFER_must_not_be_redefined_between_headers;;\n\n// Trigger an error if MCGEN_EVENTWRITETRANSFER is defined as a function-like macro:\ntypedef void MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_MCGEN_EVENTWRITETRANSFER;\ntypedef int _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_, MCGEN_EVENTWRITETRANSFER);\n\n#ifndef McGenEventWrite_def\n#define McGenEventWrite_def\n\n// This macro is for use by MC-generated code and should not be used directly.\n#define McGenEventWrite _mcgen_PASTE2(McGenEventWrite_, MCGEN_EVENTWRITETRANSFER)\n\n// This function is for use by MC-generated code and should not be used directly.\nDECLSPEC_NOINLINE __inline\nULONG __stdcall\nMcGenEventWrite(\n _In_ PMCGEN_TRACE_CONTEXT Context,\n _In_ PCEVENT_DESCRIPTOR Descriptor,\n _In_opt_ LPCGUID ActivityId,\n _In_range_(1, 128) ULONG EventDataCount,\n _Pre_cap_(EventDataCount) EVENT_DATA_DESCRIPTOR* EventData\n )\n{\n const USHORT UNALIGNED* Traits;\n\n // Some customized MCGEN_EVENTWRITETRANSFER macros might ignore ActivityId.\n UNREFERENCED_PARAMETER(ActivityId);\n\n Traits = (const USHORT UNALIGNED*)(UINT_PTR)Context->Logger;\n\n if (Traits == NULL) {\n EventData[0].Ptr = 0;\n EventData[0].Size = 0;\n EventData[0].Reserved = 0;\n } else {\n EventData[0].Ptr = (ULONG_PTR)Traits;\n EventData[0].Size = *Traits;\n EventData[0].Reserved = 2; // EVENT_DATA_DESCRIPTOR_TYPE_PROVIDER_METADATA\n }\n\n return MCGEN_EVENTWRITETRANSFER(\n Context->RegistrationHandle,\n Descriptor,\n ActivityId,\n NULL,\n EventDataCount,\n EventData);\n}\n#endif // McGenEventWrite_def\n\n#if !defined(McGenEventRegisterUnregister)\n#define McGenEventRegisterUnregister\n\n// This macro is for use by MC-generated code and should not be used directly.\n#define McGenEventRegister _mcgen_PASTE2(McGenEventRegister_, MCGEN_EVENTREGISTER)\n\n#pragma warning(push)\n#pragma warning(disable:6103)\n// This function is for use by MC-generated code and should not be used directly.\nDECLSPEC_NOINLINE __inline\nULONG __stdcall\nMcGenEventRegister(\n _In_ LPCGUID ProviderId,\n _In_opt_ _mcgen_PENABLECALLBACK EnableCallback,\n _In_opt_ PVOID CallbackContext,\n _Inout_ PREGHANDLE RegHandle\n )\n/*++\n\nRoutine Description:\n\n This function registers the provider with ETW.\n\nArguments:\n\n ProviderId - Provider ID to register with ETW.\n\n EnableCallback - Callback to be used.\n\n CallbackContext - Context for the callback.\n\n RegHandle - Pointer to registration handle.\n\nRemarks:\n\n Should not be called if the provider is already registered (i.e. should not\n be called if *RegHandle != 0). Repeatedly registering a provider is a bug\n and may indicate a race condition. However, for compatibility with previous\n behavior, this function will return SUCCESS in this case.\n\n--*/\n{\n ULONG Error;\n\n if (*RegHandle != 0)\n {\n Error = 0; // ERROR_SUCCESS\n }\n else\n {\n Error = MCGEN_EVENTREGISTER(ProviderId, EnableCallback, CallbackContext, RegHandle);\n }\n\n return Error;\n}\n#pragma warning(pop)\n\n// This macro is for use by MC-generated code and should not be used directly.\n#define McGenEventUnregister _mcgen_PASTE2(McGenEventUnregister_, MCGEN_EVENTUNREGISTER)\n\n// This function is for use by MC-generated code and should not be used directly.\nDECLSPEC_NOINLINE __inline\nULONG __stdcall\nMcGenEventUnregister(_Inout_ PREGHANDLE RegHandle)\n/*++\n\nRoutine Description:\n\n Unregister from ETW and set *RegHandle = 0.\n\nArguments:\n\n RegHandle - the pointer to the provider registration handle\n\nRemarks:\n\n If provider has not been registered (i.e. if *RegHandle == 0),\n return SUCCESS. It is safe to call McGenEventUnregister even if the\n call to McGenEventRegister returned an error.\n\n--*/\n{\n ULONG Error;\n\n if(*RegHandle == 0)\n {\n Error = 0; // ERROR_SUCCESS\n }\n else\n {\n Error = MCGEN_EVENTUNREGISTER(*RegHandle);\n *RegHandle = (REGHANDLE)0;\n }\n\n return Error;\n}\n\n#endif // McGenEventRegisterUnregister\n\n#ifndef _mcgen_EVENT_BIT_SET\n #if defined(_M_IX86) || defined(_M_X64)\n // This macro is for use by MC-generated code and should not be used directly.\n #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((((const unsigned char*)EnableBits)[BitPosition >> 3] & (1u << (BitPosition & 7))) != 0)\n #else // CPU type\n // This macro is for use by MC-generated code and should not be used directly.\n #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((EnableBits[BitPosition >> 5] & (1u << (BitPosition & 31))) != 0)\n #endif // CPU type\n#endif // _mcgen_EVENT_BIT_SET\n\n#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n// Provider \"microsoft-windows-mimalloc\" event count 2\n//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n// Provider GUID = 138f4dbb-ee04-4899-aa0a-572ad4475779\nEXTERN_C __declspec(selectany) const GUID ETW_MI_Provider = {0x138f4dbb, 0xee04, 0x4899, {0xaa, 0x0a, 0x57, 0x2a, 0xd4, 0x47, 0x57, 0x79}};\n\n#ifndef ETW_MI_Provider_Traits\n#define ETW_MI_Provider_Traits NULL\n#endif // ETW_MI_Provider_Traits\n\n//\n// Event Descriptors\n//\nEXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_ALLOC = {0x64, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0};\n#define ETW_MI_ALLOC_value 0x64\nEXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_FREE = {0x65, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0};\n#define ETW_MI_FREE_value 0x65\n\n//\n// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:\n// Define this macro to have the compiler skip the generated functions in this\n// header.\n//\n#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n//\n// Event Enablement Bits\n// These variables are for use by MC-generated code and should not be used directly.\n//\nEXTERN_C __declspec(selectany) DECLSPEC_CACHEALIGN ULONG microsoft_windows_mimallocEnableBits[1];\nEXTERN_C __declspec(selectany) const ULONGLONG microsoft_windows_mimallocKeywords[1] = {0x0};\nEXTERN_C __declspec(selectany) const unsigned char microsoft_windows_mimallocLevels[1] = {4};\n\n//\n// Provider context\n//\nEXTERN_C __declspec(selectany) MCGEN_TRACE_CONTEXT ETW_MI_Provider_Context = {0, (ULONG_PTR)ETW_MI_Provider_Traits, 0, 0, 0, 0, 0, 0, 1, microsoft_windows_mimallocEnableBits, microsoft_windows_mimallocKeywords, microsoft_windows_mimallocLevels};\n\n//\n// Provider REGHANDLE\n//\n#define microsoft_windows_mimallocHandle (ETW_MI_Provider_Context.RegistrationHandle)\n\n//\n// This macro is set to 0, indicating that the EventWrite[Name] macros do not\n// have an Activity parameter. This is controlled by the -km and -um options.\n//\n#define ETW_MI_Provider_EventWriteActivity 0\n\n//\n// Register with ETW using the control GUID specified in the manifest.\n// Invoke this macro during module initialization (i.e. program startup,\n// DLL process attach, or driver load) to initialize the provider.\n// Note that if this function returns an error, the error means that\n// will not work, but no action needs to be taken -- even if EventRegister\n// returns an error, it is generally safe to use EventWrite and\n// EventUnregister macros (they will be no-ops if EventRegister failed).\n//\n#ifndef EventRegistermicrosoft_windows_mimalloc\n#define EventRegistermicrosoft_windows_mimalloc() McGenEventRegister(&ETW_MI_Provider, McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle)\n#endif\n\n//\n// Register with ETW using a specific control GUID (i.e. a GUID other than what\n// is specified in the manifest). Advanced scenarios only.\n//\n#ifndef EventRegisterByGuidmicrosoft_windows_mimalloc\n#define EventRegisterByGuidmicrosoft_windows_mimalloc(Guid) McGenEventRegister(&(Guid), McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle)\n#endif\n\n//\n// Unregister with ETW and close the provider.\n// Invoke this macro during module shutdown (i.e. program exit, DLL process\n// detach, or driver unload) to unregister the provider.\n// Note that you MUST call EventUnregister before DLL or driver unload\n// (not optional): failure to unregister a provider before DLL or driver unload\n// will result in crashes.\n//\n#ifndef EventUnregistermicrosoft_windows_mimalloc\n#define EventUnregistermicrosoft_windows_mimalloc() McGenEventUnregister(µsoft_windows_mimallocHandle)\n#endif\n\n//\n// MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION macro:\n// Define this macro to enable support for caller-allocated provider context.\n//\n#ifdef MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION\n\n//\n// Advanced scenarios: Caller-allocated provider context.\n// Use when multiple differently-configured provider handles are needed,\n// e.g. for container-aware drivers, one context per container.\n//\n// Usage:\n//\n// - Caller enables the feature before including this header, e.g.\n// #define MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION 1\n// - Caller allocates memory, e.g. pContext = malloc(sizeof(McGenContext_microsoft_windows_mimalloc));\n// - Caller registers the provider, e.g. EventRegistermicrosoft_windows_mimalloc_ForContext(pContext);\n// - Caller writes events, e.g. EventWriteMyEvent_ForContext(pContext, ...);\n// - Caller unregisters, e.g. EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext);\n// - Caller frees memory, e.g. free(pContext);\n//\n\ntypedef struct tagMcGenContext_microsoft_windows_mimalloc {\n // The fields of this structure are subject to change and should\n // not be accessed directly. To access the provider's REGHANDLE,\n // use microsoft_windows_mimallocHandle_ForContext(pContext).\n MCGEN_TRACE_CONTEXT Context;\n ULONG EnableBits[1];\n} McGenContext_microsoft_windows_mimalloc;\n\n#define EventRegistermicrosoft_windows_mimalloc_ForContext(pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&ETW_MI_Provider, pContext)\n#define EventRegisterByGuidmicrosoft_windows_mimalloc_ForContext(Guid, pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&(Guid), pContext)\n#define EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext) McGenEventUnregister(&(pContext)->Context.RegistrationHandle)\n\n//\n// Provider REGHANDLE for caller-allocated context.\n//\n#define microsoft_windows_mimallocHandle_ForContext(pContext) ((pContext)->Context.RegistrationHandle)\n\n// This function is for use by MC-generated code and should not be used directly.\n// Initialize and register the caller-allocated context.\n__inline\nULONG __stdcall\n_mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(\n _In_ LPCGUID pProviderId,\n _Out_ McGenContext_microsoft_windows_mimalloc* pContext)\n{\n RtlZeroMemory(pContext, sizeof(*pContext));\n pContext->Context.Logger = (ULONG_PTR)ETW_MI_Provider_Traits;\n pContext->Context.EnableBitsCount = 1;\n pContext->Context.EnableBitMask = pContext->EnableBits;\n pContext->Context.EnableKeyWords = microsoft_windows_mimallocKeywords;\n pContext->Context.EnableLevel = microsoft_windows_mimallocLevels;\n return McGenEventRegister(\n pProviderId,\n McGenControlCallbackV2,\n &pContext->Context,\n &pContext->Context.RegistrationHandle);\n}\n\n// This function is for use by MC-generated code and should not be used directly.\n// Trigger a compile error if called with the wrong parameter type.\nFORCEINLINE\n_Ret_ McGenContext_microsoft_windows_mimalloc*\n_mcgen_CheckContextType_microsoft_windows_mimalloc(_In_ McGenContext_microsoft_windows_mimalloc* pContext)\n{\n return pContext;\n}\n\n#endif // MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION\n\n//\n// Enablement check macro for event \"ETW_MI_ALLOC\"\n//\n#define EventEnabledETW_MI_ALLOC() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0)\n#define EventEnabledETW_MI_ALLOC_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0)\n\n//\n// Event write macros for event \"ETW_MI_ALLOC\"\n//\n#define EventWriteETW_MI_ALLOC(Address, Size) \\\n MCGEN_EVENT_ENABLED(ETW_MI_ALLOC) \\\n ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size) : 0\n#define EventWriteETW_MI_ALLOC_AssumeEnabled(Address, Size) \\\n _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size)\n#define EventWriteETW_MI_ALLOC_ForContext(pContext, Address, Size) \\\n MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_ALLOC) \\\n ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&(pContext)->Context, &ETW_MI_ALLOC, Address, Size) : 0\n#define EventWriteETW_MI_ALLOC_ForContextAssumeEnabled(pContext, Address, Size) \\\n _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_ALLOC, Address, Size)\n\n// This macro is for use by MC-generated code and should not be used directly.\n#define _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)\n\n//\n// Enablement check macro for event \"ETW_MI_FREE\"\n//\n#define EventEnabledETW_MI_FREE() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0)\n#define EventEnabledETW_MI_FREE_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0)\n\n//\n// Event write macros for event \"ETW_MI_FREE\"\n//\n#define EventWriteETW_MI_FREE(Address, Size) \\\n MCGEN_EVENT_ENABLED(ETW_MI_FREE) \\\n ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size) : 0\n#define EventWriteETW_MI_FREE_AssumeEnabled(Address, Size) \\\n _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size)\n#define EventWriteETW_MI_FREE_ForContext(pContext, Address, Size) \\\n MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_FREE) \\\n ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&(pContext)->Context, &ETW_MI_FREE, Address, Size) : 0\n#define EventWriteETW_MI_FREE_ForContextAssumeEnabled(pContext, Address, Size) \\\n _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_FREE, Address, Size)\n\n// This macro is for use by MC-generated code and should not be used directly.\n#define _mcgen_TEMPLATE_FOR_ETW_MI_FREE _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)\n\n#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n//\n// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:\n// Define this macro to have the compiler skip the generated functions in this\n// header.\n//\n#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n//\n// Template Functions\n//\n\n//\n// Function for template \"ETW_CUSTOM_HEAP_ALLOC_DATA\" (and possibly others).\n// This function is for use by MC-generated code and should not be used directly.\n//\n#ifndef McTemplateU0xx_def\n#define McTemplateU0xx_def\nETW_INLINE\nULONG\n_mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)(\n _In_ PMCGEN_TRACE_CONTEXT Context,\n _In_ PCEVENT_DESCRIPTOR Descriptor,\n _In_ const unsigned __int64 _Arg0,\n _In_ const unsigned __int64 _Arg1\n )\n{\n#define McTemplateU0xx_ARGCOUNT 2\n\n EVENT_DATA_DESCRIPTOR EventData[McTemplateU0xx_ARGCOUNT + 1];\n\n EventDataDescCreate(&EventData[1],&_Arg0, sizeof(const unsigned __int64) );\n\n EventDataDescCreate(&EventData[2],&_Arg1, sizeof(const unsigned __int64) );\n\n return McGenEventWrite(Context, Descriptor, NULL, McTemplateU0xx_ARGCOUNT + 1, EventData);\n}\n#endif // McTemplateU0xx_def\n\n#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n#if defined(__cplusplus)\n}\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/prim/windows/prim.c", "sha256": "6de43c6aeb5238017885fa57f96ed0bbc71098503ce8ab5667c73dac1a63454e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// This file is included in `src/prim/prim.c`\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\"\n#include // fputs, stderr\n#include // atexit\n\n// xbox has no console IO\n#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP | WINAPI_PARTITION_SYSTEM)\n#define MI_HAS_CONSOLE_IO\n#endif\n\n//---------------------------------------------\n// Dynamically bind Windows API points for portability\n//---------------------------------------------\n\n// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.\n// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)\n// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB)\n// We define a minimal MEM_EXTENDED_PARAMETER ourselves in order to be able to compile with older SDK's.\ntypedef enum MI_MEM_EXTENDED_PARAMETER_TYPE_E {\n MiMemExtendedParameterInvalidType = 0,\n MiMemExtendedParameterAddressRequirements,\n MiMemExtendedParameterNumaNode,\n MiMemExtendedParameterPartitionHandle,\n MiMemExtendedParameterUserPhysicalHandle,\n MiMemExtendedParameterAttributeFlags,\n MiMemExtendedParameterMax\n} MI_MEM_EXTENDED_PARAMETER_TYPE;\n\ntypedef struct DECLSPEC_ALIGN(8) MI_MEM_EXTENDED_PARAMETER_S {\n struct { DWORD64 Type : 8; DWORD64 Reserved : 56; } Type;\n union { DWORD64 ULong64; PVOID Pointer; SIZE_T Size; HANDLE Handle; DWORD ULong; } Arg;\n} MI_MEM_EXTENDED_PARAMETER;\n\ntypedef struct MI_MEM_ADDRESS_REQUIREMENTS_S {\n PVOID LowestStartingAddress;\n PVOID HighestEndingAddress;\n SIZE_T Alignment;\n} MI_MEM_ADDRESS_REQUIREMENTS;\n\n#define MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE 0x00000010\n\n#include \ntypedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG);\ntypedef LONG (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG); // avoid NTSTATUS as it is not defined on xbox (pr #1084)\nstatic PVirtualAlloc2 pVirtualAlloc2 = NULL;\nstatic PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL;\n\n// Similarly, GetNumaProcessorNodeEx is only supported since Windows 7 (and GetNumaNodeProcessorMask is not supported on xbox)\ntypedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER;\n\ntypedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber);\ntypedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(MI_PROCESSOR_NUMBER* Processor, PUSHORT NodeNumber);\ntypedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINITY ProcessorMask);\ntypedef BOOL (__stdcall *PGetNumaProcessorNode)(UCHAR Processor, PUCHAR NodeNumber);\ntypedef BOOL (__stdcall* PGetNumaNodeProcessorMask)(UCHAR Node, PULONGLONG ProcessorMask);\ntypedef BOOL (__stdcall* PGetNumaHighestNodeNumber)(PULONG Node);\nstatic PGetCurrentProcessorNumberEx pGetCurrentProcessorNumberEx = NULL;\nstatic PGetNumaProcessorNodeEx pGetNumaProcessorNodeEx = NULL;\nstatic PGetNumaNodeProcessorMaskEx pGetNumaNodeProcessorMaskEx = NULL;\nstatic PGetNumaProcessorNode pGetNumaProcessorNode = NULL;\nstatic PGetNumaNodeProcessorMask pGetNumaNodeProcessorMask = NULL;\nstatic PGetNumaHighestNodeNumber pGetNumaHighestNodeNumber = NULL;\n\n// Not available on xbox\ntypedef SIZE_T(__stdcall* PGetLargePageMinimum)(VOID);\nstatic PGetLargePageMinimum pGetLargePageMinimum = NULL;\n\n// Available after Windows XP\ntypedef BOOL (__stdcall *PGetPhysicallyInstalledSystemMemory)( PULONGLONG TotalMemoryInKilobytes );\n\n\n//---------------------------------------------\n// Enable large page support dynamically (if possible)\n//---------------------------------------------\n\nstatic bool win_enable_large_os_pages_once(size_t* large_page_size)\n{\n if (pGetLargePageMinimum==NULL) return false; // no large page support (xbox etc.)\n\n // Try to see if large OS pages are supported\n // To use large pages on Windows, we first need access permission\n // Set \"Lock pages in memory\" permission in the group policy editor\n // \n unsigned long err = 0;\n HANDLE token = NULL;\n BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);\n if (ok) {\n TOKEN_PRIVILEGES tp;\n ok = LookupPrivilegeValue(NULL, TEXT(\"SeLockMemoryPrivilege\"), &tp.Privileges[0].Luid);\n if (ok) {\n tp.PrivilegeCount = 1;\n tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;\n ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0);\n if (ok) {\n err = GetLastError();\n ok = (err == ERROR_SUCCESS);\n if (ok && large_page_size != NULL && pGetLargePageMinimum != NULL) {\n *large_page_size = (*pGetLargePageMinimum)();\n }\n }\n }\n CloseHandle(token);\n }\n if (!ok) {\n if (err == 0) err = GetLastError();\n _mi_warning_message(\"cannot enable large OS page support, error %lu\\n\", err);\n }\n return (ok!=0);\n}\n\nstatic bool win_enable_large_os_pages(size_t* large_page_size) {\n mi_atomic_do_once {\n win_enable_large_os_pages_once(large_page_size);\n }\n return (_mi_os_large_page_size() > 0);\n}\n\n\n//---------------------------------------------\n// Initialize\n//---------------------------------------------\n\nvoid _mi_prim_mem_init( mi_os_mem_config_t* config )\n{\n config->has_overcommit = false;\n config->has_partial_free = false;\n config->has_virtual_reserve = true;\n // get the page size\n SYSTEM_INFO si;\n GetSystemInfo(&si);\n if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; }\n if (si.dwAllocationGranularity > 0) { config->alloc_granularity = si.dwAllocationGranularity; }\n // get virtual address bits\n if ((uintptr_t)si.lpMaximumApplicationAddress > 0) {\n const size_t vbits = MI_SIZE_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress);\n config->virtual_address_bits = vbits;\n }\n\n // get the VirtualAlloc2 function\n HINSTANCE hDll;\n hDll = LoadLibrary(TEXT(\"kernelbase.dll\"));\n if (hDll != NULL) {\n // use VirtualAlloc2FromApp if possible as it is available to Windows store apps\n pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, \"VirtualAlloc2FromApp\");\n if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, \"VirtualAlloc2\");\n FreeLibrary(hDll);\n }\n // NtAllocateVirtualMemoryEx is used for huge page allocation\n hDll = LoadLibrary(TEXT(\"ntdll.dll\"));\n if (hDll != NULL) {\n pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, \"NtAllocateVirtualMemoryEx\");\n FreeLibrary(hDll);\n }\n // Try to use Win7+ numa API\n hDll = LoadLibrary(TEXT(\"kernel32.dll\"));\n if (hDll != NULL) {\n pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, \"GetCurrentProcessorNumberEx\");\n pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, \"GetNumaProcessorNodeEx\");\n pGetNumaNodeProcessorMaskEx = (PGetNumaNodeProcessorMaskEx)(void (*)(void))GetProcAddress(hDll, \"GetNumaNodeProcessorMaskEx\");\n pGetNumaProcessorNode = (PGetNumaProcessorNode)(void (*)(void))GetProcAddress(hDll, \"GetNumaProcessorNode\");\n pGetNumaNodeProcessorMask = (PGetNumaNodeProcessorMask)(void (*)(void))GetProcAddress(hDll, \"GetNumaNodeProcessorMask\");\n pGetNumaHighestNodeNumber = (PGetNumaHighestNodeNumber)(void (*)(void))GetProcAddress(hDll, \"GetNumaHighestNodeNumber\");\n pGetLargePageMinimum = (PGetLargePageMinimum)(void (*)(void))GetProcAddress(hDll, \"GetLargePageMinimum\");\n // Get physical memory (not available on XP, so check dynamically)\n PGetPhysicallyInstalledSystemMemory pGetPhysicallyInstalledSystemMemory = (PGetPhysicallyInstalledSystemMemory)(void (*)(void))GetProcAddress(hDll,\"GetPhysicallyInstalledSystemMemory\");\n if (pGetPhysicallyInstalledSystemMemory != NULL) {\n ULONGLONG memInKiB = 0;\n if ((*pGetPhysicallyInstalledSystemMemory)(&memInKiB)) {\n if (memInKiB > 0 && memInKiB <= SIZE_MAX) {\n config->physical_memory_in_kib = (size_t)memInKiB;\n }\n }\n }\n FreeLibrary(hDll);\n }\n // Enable large/huge OS page support?\n if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {\n win_enable_large_os_pages(&config->large_page_size);\n }\n}\n\n\n//---------------------------------------------\n// Free\n//---------------------------------------------\n\nint _mi_prim_free(void* addr, size_t size ) {\n MI_UNUSED(size);\n DWORD errcode = 0;\n bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0);\n if (err) { errcode = GetLastError(); }\n if (errcode == ERROR_INVALID_ADDRESS) {\n // In mi_os_mem_alloc_aligned the fallback path may have returned a pointer inside\n // the memory region returned by VirtualAlloc; in that case we need to free using\n // the start of the region.\n MEMORY_BASIC_INFORMATION info; _mi_memzero_var(info);\n VirtualQuery(addr, &info, sizeof(info));\n if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)MI_SEGMENT_SIZE) {\n errcode = 0;\n err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0);\n if (err) { errcode = GetLastError(); }\n }\n }\n return (int)errcode;\n}\n\n\n//---------------------------------------------\n// VirtualAlloc\n//---------------------------------------------\n\nstatic void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) {\n #if (MI_INTPTR_SIZE >= 8)\n // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations\n if (addr == NULL) {\n void* hint = _mi_os_get_aligned_hint(try_alignment,size);\n if (hint != NULL) {\n void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE);\n if (p != NULL) return p;\n _mi_verbose_message(\"warning: unable to allocate hinted aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\\n\", size, GetLastError(), hint, try_alignment, flags);\n // fall through on error\n }\n }\n #endif\n // on modern Windows try use VirtualAlloc2 for aligned allocation\n if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {\n MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 };\n reqs.Alignment = try_alignment;\n MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} };\n param.Type.Type = MiMemExtendedParameterAddressRequirements;\n param.Arg.Pointer = &reqs;\n void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1);\n if (p != NULL) return p;\n _mi_warning_message(\"unable to allocate aligned OS memory (0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\\n\", size, GetLastError(), addr, try_alignment, flags);\n // fall through on error\n }\n // last resort\n return VirtualAlloc(addr, size, flags, PAGE_READWRITE);\n}\n\nstatic bool win_is_out_of_memory_error(DWORD err) {\n switch (err) {\n case ERROR_COMMITMENT_MINIMUM:\n case ERROR_COMMITMENT_LIMIT:\n case ERROR_PAGEFILE_QUOTA:\n case ERROR_NOT_ENOUGH_MEMORY:\n return true;\n default:\n return false;\n }\n}\n\nstatic void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) {\n long max_retry_msecs = mi_option_get_clamp(mi_option_retry_on_oom, 0, 2000); // at most 2 seconds\n if (max_retry_msecs == 1) { max_retry_msecs = 100; } // if one sets the option to \"true\"\n for (long tries = 1; tries <= 10; tries++) { // try at most 10 times (=2200ms)\n void* p = win_virtual_alloc_prim_once(addr, size, try_alignment, flags);\n if (p != NULL) {\n // success, return the address\n return p;\n }\n else if (max_retry_msecs > 0 && (try_alignment <= 2*MI_SEGMENT_ALIGN) &&\n (flags&MEM_COMMIT) != 0 && (flags&MEM_LARGE_PAGES) == 0 &&\n win_is_out_of_memory_error(GetLastError())) {\n // if committing regular memory and being out-of-memory,\n // keep trying for a bit in case memory frees up after all. See issue #894\n _mi_warning_message(\"out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\\n\", tries, size, GetLastError(), addr, try_alignment, flags);\n long sleep_msecs = tries*40; // increasing waits\n if (sleep_msecs > max_retry_msecs) { sleep_msecs = max_retry_msecs; }\n max_retry_msecs -= sleep_msecs;\n Sleep(sleep_msecs);\n }\n else {\n // otherwise return with an error\n break;\n }\n }\n return NULL;\n}\n\nstatic void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) {\n mi_assert_internal(!(large_only && !allow_large));\n static _Atomic(size_t) large_page_try_ok; // = 0;\n void* p = NULL;\n // Try to allocate large OS pages (2MiB) if allowed or required.\n if ((large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))\n && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0)\n {\n size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);\n if (!large_only && try_ok > 0) {\n // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive.\n // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times.\n mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1);\n }\n else {\n // large OS pages must always reserve and commit.\n *is_large = true;\n p = win_virtual_alloc_prim(addr, size, try_alignment, flags | MEM_LARGE_PAGES);\n if (large_only) return p;\n // fall back to non-large page allocation on error (`p == NULL`).\n if (p == NULL) {\n mi_atomic_store_release(&large_page_try_ok,10UL); // on error, don't try again for the next N allocations\n }\n }\n }\n // Fall back to regular page allocation\n if (p == NULL) {\n *is_large = ((flags&MEM_LARGE_PAGES) != 0);\n p = win_virtual_alloc_prim(addr, size, try_alignment, flags);\n }\n //if (p == NULL) { _mi_warning_message(\"unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\\n\", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); }\n return p;\n}\n\nint _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {\n mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);\n mi_assert_internal(commit || !allow_large);\n mi_assert_internal(try_alignment > 0);\n *is_zero = true;\n int flags = MEM_RESERVE;\n if (commit) { flags |= MEM_COMMIT; }\n *addr = win_virtual_alloc(hint_addr, size, try_alignment, flags, false, allow_large, is_large);\n return (*addr != NULL ? 0 : (int)GetLastError());\n}\n\n\n//---------------------------------------------\n// Commit/Reset/Protect\n//---------------------------------------------\n#ifdef _MSC_VER\n#pragma warning(disable:6250) // suppress warning calling VirtualFree without MEM_RELEASE (for decommit)\n#endif\n\nint _mi_prim_commit(void* addr, size_t size, bool* is_zero) {\n *is_zero = false;\n /*\n // zero'ing only happens on an initial commit... but checking upfront seems expensive..\n _MEMORY_BASIC_INFORMATION meminfo; _mi_memzero_var(meminfo);\n if (VirtualQuery(addr, &meminfo, size) > 0) {\n if ((meminfo.State & MEM_COMMIT) == 0) {\n *is_zero = true;\n }\n }\n */\n // commit\n void* p = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);\n if (p == NULL) return (int)GetLastError();\n return 0;\n}\n\nint _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {\n BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT);\n *needs_recommit = true; // for safety, assume always decommitted even in the case of an error.\n return (ok ? 0 : (int)GetLastError());\n}\n\nint _mi_prim_reset(void* addr, size_t size) {\n void* p = VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);\n mi_assert_internal(p == addr);\n #if 0\n if (p != NULL) {\n VirtualUnlock(addr,size); // VirtualUnlock after MEM_RESET removes the memory directly from the working set\n }\n #endif\n return (p != NULL ? 0 : (int)GetLastError());\n}\n\nint _mi_prim_reuse(void* addr, size_t size) {\n MI_UNUSED(addr); MI_UNUSED(size);\n return 0;\n}\n\nint _mi_prim_protect(void* addr, size_t size, bool protect) {\n DWORD oldprotect = 0;\n BOOL ok = VirtualProtect(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect);\n return (ok ? 0 : (int)GetLastError());\n}\n\n\n//---------------------------------------------\n// Huge page allocation\n//---------------------------------------------\n\nstatic void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int numa_node)\n{\n const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE;\n\n win_enable_large_os_pages(NULL);\n\n MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} };\n // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages\n static _Atomic(size_t) mi_huge_pages_available = MI_ATOMIC_VAR_INIT(1);\n if (pNtAllocateVirtualMemoryEx != NULL && mi_atomic_load_acquire(&mi_huge_pages_available) != 0) {\n params[0].Type.Type = MiMemExtendedParameterAttributeFlags;\n params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;\n ULONG param_count = 1;\n if (numa_node >= 0) {\n param_count++;\n params[1].Type.Type = MiMemExtendedParameterNumaNode;\n params[1].Arg.ULong = (unsigned)numa_node;\n }\n SIZE_T psize = size;\n void* base = hint_addr;\n LONG err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count);\n if (err == 0 && base != NULL) {\n return base;\n }\n else {\n // fall back to regular large pages\n mi_atomic_store_release(&mi_huge_pages_available,0); // don't try further huge pages\n _mi_warning_message(\"unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\\n\", err);\n }\n }\n // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation\n if (pVirtualAlloc2 != NULL && numa_node >= 0) {\n params[0].Type.Type = MiMemExtendedParameterNumaNode;\n params[0].Arg.ULong = (unsigned)numa_node;\n return (*pVirtualAlloc2)(GetCurrentProcess(), hint_addr, size, flags, PAGE_READWRITE, params, 1);\n }\n\n // otherwise use regular virtual alloc on older windows\n return VirtualAlloc(hint_addr, size, flags, PAGE_READWRITE);\n}\n\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {\n *is_zero = true;\n *addr = _mi_prim_alloc_huge_os_pagesx(hint_addr,size,numa_node);\n return (*addr != NULL ? 0 : (int)GetLastError());\n}\n\n\n//---------------------------------------------\n// Numa nodes\n//---------------------------------------------\n\nsize_t _mi_prim_numa_node(void) {\n USHORT numa_node = 0;\n if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) {\n // Extended API is supported\n MI_PROCESSOR_NUMBER pnum;\n (*pGetCurrentProcessorNumberEx)(&pnum);\n USHORT nnode = 0;\n BOOL ok = (*pGetNumaProcessorNodeEx)(&pnum, &nnode);\n if (ok) { numa_node = nnode; }\n }\n else if (pGetNumaProcessorNode != NULL) {\n // Vista or earlier, use older API that is limited to 64 processors. Issue #277\n DWORD pnum = GetCurrentProcessorNumber();\n UCHAR nnode = 0;\n BOOL ok = pGetNumaProcessorNode((UCHAR)pnum, &nnode);\n if (ok) { numa_node = nnode; }\n }\n return numa_node;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n ULONG numa_max = 0;\n if (pGetNumaHighestNodeNumber!=NULL) {\n (*pGetNumaHighestNodeNumber)(&numa_max);\n }\n // find the highest node number that has actual processors assigned to it. Issue #282\n while (numa_max > 0) {\n if (pGetNumaNodeProcessorMaskEx != NULL) {\n // Extended API is supported\n GROUP_AFFINITY affinity;\n if ((*pGetNumaNodeProcessorMaskEx)((USHORT)numa_max, &affinity)) {\n if (affinity.Mask != 0) break; // found the maximum non-empty node\n }\n }\n else {\n // Vista or earlier, use older API that is limited to 64 processors.\n ULONGLONG mask;\n if (pGetNumaNodeProcessorMask != NULL) {\n if ((*pGetNumaNodeProcessorMask)((UCHAR)numa_max, &mask)) {\n if (mask != 0) break; // found the maximum non-empty node\n }\n };\n }\n // max node was invalid or had no processor assigned, try again\n numa_max--;\n }\n return ((size_t)numa_max + 1);\n}\n\n\n//----------------------------------------------------------------\n// Clock\n//----------------------------------------------------------------\n\nstatic mi_msecs_t mi_to_msecs(LARGE_INTEGER t) {\n static LARGE_INTEGER mfreq; // = 0\n if (mfreq.QuadPart == 0LL) {\n LARGE_INTEGER f;\n QueryPerformanceFrequency(&f);\n mfreq.QuadPart = f.QuadPart/1000LL;\n if (mfreq.QuadPart == 0) mfreq.QuadPart = 1;\n }\n return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart);\n}\n\nmi_msecs_t _mi_prim_clock_now(void) {\n LARGE_INTEGER t;\n QueryPerformanceCounter(&t);\n return mi_to_msecs(t);\n}\n\n\n//----------------------------------------------------------------\n// Process Info\n//----------------------------------------------------------------\n\n#include \n\nstatic mi_msecs_t filetime_msecs(const FILETIME* ftime) {\n ULARGE_INTEGER i;\n i.LowPart = ftime->dwLowDateTime;\n i.HighPart = ftime->dwHighDateTime;\n mi_msecs_t msecs = (i.QuadPart / 10000); // FILETIME is in 100 nano seconds\n return msecs;\n}\n\ntypedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD);\nstatic PGetProcessMemoryInfo pGetProcessMemoryInfo = NULL;\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo)\n{\n FILETIME ct;\n FILETIME ut;\n FILETIME st;\n FILETIME et;\n GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut);\n pinfo->utime = filetime_msecs(&ut);\n pinfo->stime = filetime_msecs(&st);\n\n // load psapi on demand\n mi_atomic_do_once {\n HINSTANCE hDll = LoadLibrary(TEXT(\"psapi.dll\"));\n if (hDll != NULL) {\n pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, \"GetProcessMemoryInfo\");\n // FreeLibrary(hDll); // don't free\n }\n }\n\n // get process info\n PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info);\n if (pGetProcessMemoryInfo != NULL) {\n pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));\n }\n pinfo->current_rss = (size_t)info.WorkingSetSize;\n pinfo->peak_rss = (size_t)info.PeakWorkingSetSize;\n pinfo->current_commit = (size_t)info.PagefileUsage;\n pinfo->peak_commit = (size_t)info.PeakPagefileUsage;\n pinfo->page_faults = (size_t)info.PageFaultCount;\n}\n\n//----------------------------------------------------------------\n// Output\n//----------------------------------------------------------------\n\nvoid _mi_prim_out_stderr( const char* msg )\n{\n // on windows with redirection, the C runtime cannot handle locale dependent output\n // after the main thread closes so we use direct console output.\n if (!_mi_preloading()) {\n // _cputs(msg); // _cputs cannot be used as it aborts when failing to lock the console\n static HANDLE hcon = INVALID_HANDLE_VALUE;\n static bool hconIsConsole = false;\n if (hcon == INVALID_HANDLE_VALUE) {\n hcon = GetStdHandle(STD_ERROR_HANDLE);\n #ifdef MI_HAS_CONSOLE_IO\n CONSOLE_SCREEN_BUFFER_INFO sbi;\n hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi));\n #endif\n }\n const size_t len = _mi_strlen(msg);\n if (len > 0 && len < UINT32_MAX) {\n DWORD written = 0;\n if (hconIsConsole) {\n #ifdef MI_HAS_CONSOLE_IO\n WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL);\n #endif\n }\n else if (hcon != INVALID_HANDLE_VALUE) {\n // use direct write if stderr was redirected\n WriteFile(hcon, msg, (DWORD)len, &written, NULL);\n }\n else {\n // finally fall back to fputs after all\n fputs(msg, stderr);\n }\n }\n }\n}\n\n\n//----------------------------------------------------------------\n// Environment\n//----------------------------------------------------------------\n\n// On Windows use GetEnvironmentVariable instead of getenv to work\n// reliably even when this is invoked before the C runtime is initialized.\n// i.e. when `_mi_preloading() == true`.\n// Note: on windows, environment names are not case sensitive.\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size) {\n result[0] = 0;\n const size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size);\n return (len > 0 && len < result_size);\n}\n\n\n//----------------------------------------------------------------\n// Random\n//----------------------------------------------------------------\n\n#if defined(MI_USE_RTLGENRANDOM) // || defined(__cplusplus)\n// We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using\n// dynamic overriding, we observed it can raise an exception when compiled with C++, and\n// sometimes deadlocks when also running under the VS debugger.\n// In contrast, issue #623 implies that on Windows Server 2019 we need to use BCryptGenRandom.\n// To be continued..\n#pragma comment (lib,\"advapi32.lib\")\n#define RtlGenRandom SystemFunction036\nmi_decl_externc BOOLEAN NTAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength);\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n return (RtlGenRandom(buf, (ULONG)buf_len) != 0);\n}\n\n#else\n\n#ifndef BCRYPT_USE_SYSTEM_PREFERRED_RNG\n#define BCRYPT_USE_SYSTEM_PREFERRED_RNG 0x00000002\n#endif\n\ntypedef LONG (NTAPI *PBCryptGenRandom)(HANDLE, PUCHAR, ULONG, ULONG);\nstatic PBCryptGenRandom pBCryptGenRandom = NULL;\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n mi_assert(buf_len <= ULONG_MAX);\n if (buf_len > ULONG_MAX) return false;\n mi_atomic_do_once {\n HINSTANCE hDll = LoadLibrary(TEXT(\"bcrypt.dll\"));\n if (hDll != NULL) {\n pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, \"BCryptGenRandom\");\n // FreeLibrary(hDll); // don't free\n }\n }\n if (pBCryptGenRandom == NULL) return false;\n return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0);\n}\n\n#endif // MI_USE_RTLGENRANDOM\n\n\n\n//----------------------------------------------------------------\n// Process & Thread Init/Done\n//----------------------------------------------------------------\n\n#if MI_WIN_USE_FIXED_TLS==1\nmi_decl_cache_align size_t _mi_win_tls_offset = 0;\n#endif\n\n//static void mi_debug_out(const char* s) {\n// HANDLE h = GetStdHandle(STD_ERROR_HANDLE);\n// WriteConsole(h, s, (DWORD)_mi_strlen(s), NULL, NULL);\n//}\n\nstatic void mi_win_tls_init(DWORD reason) {\n if (reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) {\n #if MI_WIN_USE_FIXED_TLS==1 // we must allocate a TLS slot dynamically\n if (_mi_win_tls_offset == 0 && reason == DLL_PROCESS_ATTACH) {\n const DWORD tls_slot = TlsAlloc(); // usually returns slot 1\n if (tls_slot == TLS_OUT_OF_INDEXES) {\n _mi_error_message(EFAULT, \"unable to allocate the a TLS slot (rebuild without MI_WIN_USE_FIXED_TLS?)\\n\");\n }\n _mi_win_tls_offset = (size_t)tls_slot * sizeof(void*);\n }\n #endif\n #if MI_HAS_TLS_SLOT >= 2 // we must initialize the TLS slot before any allocation\n if (mi_prim_get_default_heap() == NULL) {\n _mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty);\n #if MI_DEBUG && MI_WIN_USE_FIXED_TLS==1\n void* const p = TlsGetValue((DWORD)(_mi_win_tls_offset / sizeof(void*)));\n mi_assert_internal(p == (void*)&_mi_heap_empty);\n #endif\n }\n #endif\n }\n}\n\nstatic void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {\n MI_UNUSED(reserved);\n MI_UNUSED(module);\n mi_win_tls_init(reason);\n if (reason==DLL_PROCESS_ATTACH) {\n _mi_auto_process_init();\n }\n else if (reason==DLL_PROCESS_DETACH) {\n _mi_auto_process_done();\n }\n else if (reason==DLL_THREAD_DETACH && !_mi_is_redirected()) {\n _mi_thread_done(NULL);\n }\n}\n\n\n/* -----------------------------------------------------------------------\n Auto initialize and finalize mimalloc on process and thread start/end.\n By default we use a combination of CRT init and TLS sections for\n both static and dynamic linkage (`MI_WIN_INIT_USE_CRT_TLS`).\n------------------------------------------------------------------------- */\n#if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS)\n #if !defined(__INTEL_LLVM_COMPILER) && !defined(__INTEL_COMPILER)\n #define MI_WIN_INIT_USE_CRT_TLS 1 \n #else\n #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* default for Intel ICX, see issue #1268 */ \n #endif\n#endif\n\n#if defined(MI_WIN_INIT_USE_CRT_TLS)\n #define MI_PRIM_HAS_PROCESS_ATTACH 1\n // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.\n void _mi_prim_thread_init_auto_done(void) {}\n void _mi_prim_thread_done_auto_done(void) {}\n void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {\n MI_UNUSED(heap);\n }\n\n static bool mi_module_is_dll(PVOID mod) {\n if (mod==NULL) return false;\n PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod;\n PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew);\n return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL);\n }\n\n static bool mi_current_module_is_dll(void) {\n HMODULE mod = NULL;\n const BOOL ok = GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCSTR)mi_current_module_is_dll, &mod);\n return (ok && mi_module_is_dll(mod));\n }\n\n // Hook into CRT initialization and finalization.\n static void mi_cdecl mi_crt_done(void) {\n // mi_debug_out(mi_current_module_is_dll() ? \"crt dll done\\n\" : \"crt exe done\\n\");\n mi_win_main(NULL, DLL_PROCESS_DETACH, 0);\n }\n\n static int mi_cdecl mi_crt_init(void) {\n // mi_debug_out(mi_current_module_is_dll() ? \"crt dll init\\n\" : \"crt exe init\\n\");\n if (mi_current_module_is_dll()) {\n // in a dll, atexit (crt_done) is called after tls process detach\n atexit(&mi_crt_done);\n }\n return 0;\n }\n\n // We also hook into the Windows loader TLS initialization and finalization.\n static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_THREAD_ATTACH) {\n //mi_debug_out(\"tls thread attach\\n\");\n mi_win_main(module, reason, reserved);\n }\n else if (reason == DLL_PROCESS_ATTACH) {\n // mi_debug_out(mi_module_is_dll(module) ? \"tls dll process attach\\n\" : \"tls exe process attach\\n\");\n // tls process attach is always called before crt init\n mi_win_main(module, reason, reserved);\n }\n }\n\n static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_THREAD_DETACH) {\n //mi_debug_out(\"tls thread detach\\n\");\n mi_win_main(module, reason, reserved);\n }\n else if (reason == DLL_PROCESS_DETACH) {\n // mi_debug_out(mi_module_is_dll(module) ? \"tls dll process detach\\n\" : \"tls exe process detach\\n\");\n if (!mi_module_is_dll(module)) {\n // in an exe, tls process detach is called after atexit (mi_crt_done)\n mi_win_main(module, reason, reserved);\n }\n }\n }\n\n // Set up TLS callbacks in a statically linked library by using special data sections.\n // See \n // We use 2 entries to ensure we call attach events before constructors\n // are called, and detach events after destructors are called.\n // We also set up a CRT initialization callback.\n #if defined(__cplusplus)\n extern \"C\" {\n #endif\n\n typedef int (mi_cdecl* mi_crt_callback_t)(void);\n #if defined(_WIN64)\n #pragma comment(linker, \"/INCLUDE:_tls_used\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_post\")\n #pragma const_seg(\".CRT$XLB\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };\n #pragma const_seg()\n #pragma const_seg(\".CRT$XLY\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };\n #pragma const_seg()\n #pragma const_seg(\".CRT$XIB\")\n extern const mi_crt_callback_t _mi_crt_callback_init[];\n const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };\n #pragma const_seg()\n #else\n #pragma comment(linker, \"/INCLUDE:__tls_used\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_post\")\n #pragma data_seg(\".CRT$XLB\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };\n #pragma data_seg()\n #pragma data_seg(\".CRT$XLY\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };\n #pragma data_seg()\n #pragma data_seg(\".CRT$XIB\")\n mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };\n #pragma data_seg()\n #endif\n\n #if defined(__cplusplus)\n }\n #endif\n\n/* ----------------------------------------------------------------\n Here we use a combination of the raw DLL main and TLS sections for\n both static and dynamic linkage. This is perhaps the most robust as\n the raw DLL main ensure we can close mimalloc latest possible.\n However, it can lead to link errors with on `_pRawDllMain` if other\n libraries also try to hook into this (like Boost).\n*/\n#elif defined(MI_WIN_INIT_USE_RAW_DLLMAIN)\n #define MI_PRIM_HAS_PROCESS_ATTACH 1\n // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.\n void _mi_prim_thread_init_auto_done(void) {}\n void _mi_prim_thread_done_auto_done(void) {}\n void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {\n MI_UNUSED(heap);\n }\n\n // If linked into a DLL module, this raw entry is called before the CRT attach and\n // after the CRT detach through the CRT _pRawDllMain pointer.\n static BOOL NTAPI mi_dll_main_raw(PVOID module, DWORD reason, LPVOID reserved) {\n mi_win_main(module, reason, reserved);\n return TRUE;\n }\n\n // Set the value of the CRT _pRawDllMain pointer\n #if defined(__cplusplus)\n extern \"C\"\n #endif\n PVOID _pRawDllMain = (PVOID)&mi_dll_main_raw;\n\n // We also hook into the Windows loader TLS initialization and finalization.\n // If we are linked into an EXE module we rely on these as `mi_dll_main_raw`\n // is not called (and otherwise we ignore the TLS callbacks by checking if we are in a DLL).\n static bool mi_module_is_dll(PVOID mod) {\n if (mod==NULL) return false;\n PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod;\n PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew);\n return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL);\n }\n\n static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {\n if (!mi_module_is_dll(module)) {\n mi_win_main(module, reason, reserved);\n }\n }\n }\n\n static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {\n if (!mi_module_is_dll(module)) {\n mi_win_main(module, reason, reserved);\n }\n }\n }\n\n // Set up TLS callbacks in a statically linked library by using special data sections.\n // See \n // We use 2 entries to ensure we call attach events before constructors\n // are called, and detach events after destructors are called.\n #if defined(__cplusplus)\n extern \"C\" {\n #endif\n\n #if defined(_WIN64)\n #pragma comment(linker, \"/INCLUDE:_tls_used\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_post\")\n #pragma const_seg(\".CRT$XLB\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };\n #pragma const_seg()\n #pragma const_seg(\".CRT$XLY\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };\n #pragma const_seg()\n #else\n #pragma comment(linker, \"/INCLUDE:__tls_used\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_post\")\n #pragma data_seg(\".CRT$XLB\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };\n #pragma data_seg()\n #pragma data_seg(\".CRT$XLY\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };\n #pragma data_seg()\n #endif\n\n #if defined(__cplusplus)\n }\n #endif\n\n/* --------------------------------------------------------------------\n Legacy options: DllMain, TLS, and FLS\n --------------------------------------------------------------------*/\n#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && defined(MI_SHARED_LIB)\n #define MI_PRIM_HAS_PROCESS_ATTACH 1\n\n // Windows DLL: easy to hook into process_init and thread_done\n BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) {\n mi_win_main((PVOID)inst,reason,reserved);\n return TRUE;\n }\n\n // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.\n void _mi_prim_thread_init_auto_done(void) { }\n void _mi_prim_thread_done_auto_done(void) { }\n void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {\n MI_UNUSED(heap);\n }\n\n#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN)\n #define MI_PRIM_HAS_PROCESS_ATTACH 1\n\n static void NTAPI mi_win_main_attach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {\n mi_win_main(module, reason, reserved);\n }\n }\n static void NTAPI mi_win_main_detach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {\n mi_win_main(module, reason, reserved);\n }\n }\n\n // Set up TLS callbacks in a statically linked library by using special data sections.\n // See \n // We use 2 entries to ensure we call attach events before constructors\n // are called, and detach events after destructors are called.\n #if defined(__cplusplus)\n extern \"C\" {\n #endif\n\n #if defined(_WIN64)\n #pragma comment(linker, \"/INCLUDE:_tls_used\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_post\")\n #pragma const_seg(\".CRT$XLB\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };\n #pragma const_seg()\n #pragma const_seg(\".CRT$XLY\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };\n #pragma const_seg()\n #else\n #pragma comment(linker, \"/INCLUDE:__tls_used\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_post\")\n #pragma data_seg(\".CRT$XLB\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };\n #pragma data_seg()\n #pragma data_seg(\".CRT$XIY\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };\n #pragma data_seg()\n #endif\n\n #if defined(__cplusplus)\n }\n #endif\n\n // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.\n void _mi_prim_thread_init_auto_done(void) { }\n void _mi_prim_thread_done_auto_done(void) { }\n void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {\n MI_UNUSED(heap);\n }\n\n#elif defined(MI_WIN_INIT_USE_FLS) // deprecated: statically linked, use fiber api\n\n #if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`)\n // MSVC: use data section magic for static libraries\n // See \n #define MI_PRIM_HAS_PROCESS_ATTACH 1\n\n static int mi_process_attach(void) {\n mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL);\n atexit(&_mi_auto_process_done);\n return 0;\n }\n typedef int(*mi_crt_callback_t)(void);\n #if defined(_WIN64)\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback\")\n #pragma section(\".CRT$XIU\", long, read)\n #else\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback\")\n #endif\n #pragma data_seg(\".CRT$XIU\")\n mi_decl_externc mi_crt_callback_t _mi_tls_callback[] = { &mi_process_attach };\n #pragma data_seg()\n #endif\n\n // use the fiber api for calling `_mi_thread_done`.\n #include \n #if (_WIN32_WINNT < 0x600) // before Windows Vista\n WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback );\n WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex );\n WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData );\n WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex);\n #endif\n\n static DWORD mi_fls_key = (DWORD)(-1);\n\n static void NTAPI mi_fls_done(PVOID value) {\n mi_heap_t* heap = (mi_heap_t*)value;\n if (heap != NULL) {\n _mi_thread_done(heap);\n FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main heap, issue #672\n }\n }\n\n void _mi_prim_thread_init_auto_done(void) {\n mi_fls_key = FlsAlloc(&mi_fls_done);\n }\n\n void _mi_prim_thread_done_auto_done(void) {\n // call thread-done on all threads (except the main thread) to prevent\n // dangling callback pointer if statically linked with a DLL; Issue #208\n FlsFree(mi_fls_key);\n }\n\n void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {\n mi_assert_internal(mi_fls_key != (DWORD)(-1));\n FlsSetValue(mi_fls_key, heap);\n }\n#else\n#error \"define windows process and thread auto initialization\"\n#endif\n\n// ----------------------------------------------------\n// Communicate with the redirection module on Windows\n// ----------------------------------------------------\n#if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT)\n #define MI_PRIM_HAS_ALLOCATOR_INIT 1\n\n static bool mi_redirected = false; // true if malloc redirects to mi_malloc\n\n bool _mi_is_redirected(void) {\n return mi_redirected;\n }\n\n #ifdef __cplusplus\n extern \"C\" {\n #endif\n mi_decl_export void _mi_redirect_entry(DWORD reason) {\n // called on redirection; careful as this may be called before DllMain\n mi_win_tls_init(reason);\n if (reason == DLL_PROCESS_ATTACH) {\n mi_redirected = true;\n }\n else if (reason == DLL_PROCESS_DETACH) {\n mi_redirected = false;\n }\n else if (reason == DLL_THREAD_DETACH) {\n // mi_debug_out(\"redirect thread detach\\n\");\n _mi_thread_done(NULL);\n }\n }\n __declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message);\n __declspec(dllimport) void mi_cdecl mi_allocator_done(void);\n #ifdef __cplusplus\n }\n #endif\n bool _mi_allocator_init(const char** message) {\n return mi_allocator_init(message);\n }\n void _mi_allocator_done(void) {\n mi_allocator_done();\n }\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/random.c", "sha256": "a2c4c4cf5cb2476ba3bf6ef55ea8d10e36d4ffce93020012dcf7c1918b3faf1b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2021, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\" // _mi_prim_random_buf\n#include // memset\n\n/* ----------------------------------------------------------------------------\nWe use our own PRNG to keep predictable performance of random number generation\nand to avoid implementations that use a lock. We only use the OS provided\nrandom source to initialize the initial seeds. Since we do not need ultimate\nperformance but we do rely on the security (for secret cookies in secure mode)\nwe use a cryptographically secure generator (chacha20).\n-----------------------------------------------------------------------------*/\n\n#define MI_CHACHA_ROUNDS (20) // perhaps use 12 for better performance?\n\n\n/* ----------------------------------------------------------------------------\nChacha20 implementation as the original algorithm with a 64-bit nonce\nand counter: https://en.wikipedia.org/wiki/Salsa20\nThe input matrix has sixteen 32-bit values:\nPosition 0 to 3: constant key\nPosition 4 to 11: the key\nPosition 12 to 13: the counter.\nPosition 14 to 15: the nonce.\n\nThe implementation uses regular C code which compiles very well on modern compilers.\n(gcc x64 has no register spills, and clang 6+ uses SSE instructions)\n-----------------------------------------------------------------------------*/\n\nstatic inline uint32_t rotl(uint32_t x, uint32_t shift) {\n return (x << shift) | (x >> (32 - shift));\n}\n\nstatic inline void qround(uint32_t x[16], size_t a, size_t b, size_t c, size_t d) {\n x[a] += x[b]; x[d] = rotl(x[d] ^ x[a], 16);\n x[c] += x[d]; x[b] = rotl(x[b] ^ x[c], 12);\n x[a] += x[b]; x[d] = rotl(x[d] ^ x[a], 8);\n x[c] += x[d]; x[b] = rotl(x[b] ^ x[c], 7);\n}\n\nstatic void chacha_block(mi_random_ctx_t* ctx)\n{\n // scramble into `x`\n uint32_t x[16];\n for (size_t i = 0; i < 16; i++) {\n x[i] = ctx->input[i];\n }\n for (size_t i = 0; i < MI_CHACHA_ROUNDS; i += 2) {\n qround(x, 0, 4, 8, 12);\n qround(x, 1, 5, 9, 13);\n qround(x, 2, 6, 10, 14);\n qround(x, 3, 7, 11, 15);\n qround(x, 0, 5, 10, 15);\n qround(x, 1, 6, 11, 12);\n qround(x, 2, 7, 8, 13);\n qround(x, 3, 4, 9, 14);\n }\n\n // add scrambled data to the initial state\n for (size_t i = 0; i < 16; i++) {\n ctx->output[i] = x[i] + ctx->input[i];\n }\n ctx->output_available = 16;\n\n // increment the counter for the next round\n ctx->input[12] += 1;\n if (ctx->input[12] == 0) {\n ctx->input[13] += 1;\n if (ctx->input[13] == 0) { // and keep increasing into the nonce\n ctx->input[14] += 1;\n }\n }\n}\n\nstatic uint32_t chacha_next32(mi_random_ctx_t* ctx) {\n if (ctx->output_available <= 0) {\n chacha_block(ctx);\n ctx->output_available = 16; // (assign again to suppress static analysis warning)\n }\n const uint32_t x = ctx->output[16 - ctx->output_available];\n ctx->output[16 - ctx->output_available] = 0; // reset once the data is handed out\n ctx->output_available--;\n return x;\n}\n\nstatic inline uint32_t read32(const uint8_t* p, size_t idx32) {\n const size_t i = 4*idx32;\n return ((uint32_t)p[i+0] | (uint32_t)p[i+1] << 8 | (uint32_t)p[i+2] << 16 | (uint32_t)p[i+3] << 24);\n}\n\nstatic void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t nonce)\n{\n // since we only use chacha for randomness (and not encryption) we\n // do not _need_ to read 32-bit values as little endian but we do anyways\n // just for being compatible :-)\n ctx->output_available = 0;\n _mi_memzero(ctx->output,sizeof(ctx->output));\n for (size_t i = 0; i < 4; i++) {\n const uint8_t* sigma = (uint8_t*)\"expand 32-byte k\";\n ctx->input[i] = read32(sigma,i);\n }\n for (size_t i = 0; i < 8; i++) {\n ctx->input[i + 4] = read32(key,i);\n }\n ctx->input[12] = 0;\n ctx->input[13] = 0;\n ctx->input[14] = (uint32_t)nonce;\n ctx->input[15] = (uint32_t)(nonce >> 32);\n}\n\nstatic void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) {\n _mi_memzero(ctx_new, sizeof(*ctx_new));\n ctx_new->weak = ctx->weak;\n _mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input));\n ctx_new->input[12] = 0;\n ctx_new->input[13] = 0;\n ctx_new->input[14] = (uint32_t)nonce;\n ctx_new->input[15] = (uint32_t)(nonce >> 32);\n mi_assert_internal(ctx->input[14] != ctx_new->input[14] || ctx->input[15] != ctx_new->input[15]); // do not reuse nonces!\n chacha_block(ctx_new);\n}\n\n\n/* ----------------------------------------------------------------------------\nRandom interface\n-----------------------------------------------------------------------------*/\n\n#if MI_DEBUG>1\nstatic bool mi_random_is_initialized(mi_random_ctx_t* ctx) {\n return (ctx != NULL && ctx->input[0] != 0);\n}\n#endif\n\nvoid _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) {\n mi_assert_internal(mi_random_is_initialized(ctx));\n mi_assert_internal(ctx != ctx_new);\n chacha_split(ctx, (uintptr_t)ctx_new /*nonce*/, ctx_new);\n}\n\nuintptr_t _mi_random_next(mi_random_ctx_t* ctx) {\n mi_assert_internal(mi_random_is_initialized(ctx));\n uintptr_t r;\n do {\n #if MI_INTPTR_SIZE <= 4\n r = chacha_next32(ctx);\n #elif MI_INTPTR_SIZE == 8\n r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx));\n #else\n # error \"define mi_random_next for this platform\"\n #endif\n } while (r==0);\n return r;\n}\n\n\n/* ----------------------------------------------------------------------------\nTo initialize a fresh random context.\nIf we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR.\n-----------------------------------------------------------------------------*/\n\nuintptr_t _mi_os_random_weak(uintptr_t extra_seed) {\n uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random\n x ^= _mi_prim_clock_now(); \n // and do a few randomization steps\n uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1;\n for (uintptr_t i = 0; i < max || x==0; i++, x++) {\n x = _mi_random_shuffle(x);\n }\n mi_assert_internal(x != 0);\n return x;\n}\n\nstatic void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) {\n uint8_t key[32];\n if (use_weak || !_mi_prim_random_buf(key, sizeof(key))) {\n // if we fail to get random data from the OS, we fall back to a\n // weak random source based on the current time\n #if !defined(__wasi__)\n if (!use_weak) { _mi_warning_message(\"unable to use secure randomness\\n\"); }\n #endif\n uintptr_t x = _mi_os_random_weak(0);\n for (size_t i = 0; i < 32; i+=4, x++) { \n x = _mi_random_shuffle(x);\n key[i] = (uint8_t)(x);\n key[i+1] = (uint8_t)(x>>8);\n key[i+2] = (uint8_t)(x>>16);\n key[i+3] = (uint8_t)(x>>24); \n }\n ctx->weak = true;\n }\n else {\n ctx->weak = false;\n }\n chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ );\n}\n\nvoid _mi_random_init(mi_random_ctx_t* ctx) {\n mi_random_init_ex(ctx, false);\n}\n\nvoid _mi_random_init_weak(mi_random_ctx_t * ctx) {\n mi_random_init_ex(ctx, true);\n}\n\nvoid _mi_random_reinit_if_weak(mi_random_ctx_t * ctx) {\n if (ctx->weak) {\n _mi_random_init(ctx);\n }\n}\n\n/* --------------------------------------------------------\ntest vectors from \n----------------------------------------------------------- */\n/*\nstatic bool array_equals(uint32_t* x, uint32_t* y, size_t n) {\n for (size_t i = 0; i < n; i++) {\n if (x[i] != y[i]) return false;\n }\n return true;\n}\nstatic void chacha_test(void)\n{\n uint32_t x[4] = { 0x11111111, 0x01020304, 0x9b8d6f43, 0x01234567 };\n uint32_t x_out[4] = { 0xea2a92f4, 0xcb1cf8ce, 0x4581472e, 0x5881c4bb };\n qround(x, 0, 1, 2, 3);\n mi_assert_internal(array_equals(x, x_out, 4));\n\n uint32_t y[16] = {\n 0x879531e0, 0xc5ecf37d, 0x516461b1, 0xc9a62f8a,\n 0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0x2a5f714c,\n 0x53372767, 0xb00a5631, 0x974c541a, 0x359e9963,\n 0x5c971061, 0x3d631689, 0x2098d9d6, 0x91dbd320 };\n uint32_t y_out[16] = {\n 0x879531e0, 0xc5ecf37d, 0xbdb886dc, 0xc9a62f8a,\n 0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0xcfacafd2,\n 0xe46bea80, 0xb00a5631, 0x974c541a, 0x359e9963,\n 0x5c971061, 0xccc07c79, 0x2098d9d6, 0x91dbd320 };\n qround(y, 2, 7, 8, 13);\n mi_assert_internal(array_equals(y, y_out, 16));\n\n mi_random_ctx_t r = {\n { 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574,\n 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c,\n 0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c,\n 0x00000001, 0x09000000, 0x4a000000, 0x00000000 },\n {0},\n 0\n };\n uint32_t r_out[16] = {\n 0xe4e7f110, 0x15593bd1, 0x1fdd0f50, 0xc47120a3,\n 0xc7f4d1c7, 0x0368c033, 0x9aaa2204, 0x4e6cd4c3,\n 0x466482d2, 0x09aa9f07, 0x05d7c214, 0xa2028bd9,\n 0xd19c12b5, 0xb94e16de, 0xe883d0cb, 0x4e3c50a2 };\n chacha_block(&r);\n mi_assert_internal(array_equals(r.output, r_out, 16));\n}\n*/\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/segment-map.c", "sha256": "a4a8e2838cd471920eac30eedc8cb47cd28f820b06b4ee344e2492a9c98a8fb6", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* -----------------------------------------------------------\n The following functions are to reliably find the segment or\n block that encompasses any pointer p (or NULL if it is not\n in any of our segments).\n We maintain a bitmap of all memory with 1 bit per MI_SEGMENT_SIZE (64MiB)\n set to 1 if it contains the segment meta data.\n----------------------------------------------------------- */\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n\n// Reduce total address space to reduce .bss (due to the `mi_segment_map`)\n#if (MI_INTPTR_SIZE > 4) && MI_TRACK_ASAN\n#define MI_SEGMENT_MAP_MAX_ADDRESS (128*1024ULL*MI_GiB) // 128 TiB (see issue #881)\n#elif (MI_INTPTR_SIZE > 4)\n#define MI_SEGMENT_MAP_MAX_ADDRESS (48*1024ULL*MI_GiB) // 48 TiB\n#else\n#define MI_SEGMENT_MAP_MAX_ADDRESS (UINT32_MAX)\n#endif\n\n#define MI_SEGMENT_MAP_PART_SIZE (MI_INTPTR_SIZE*MI_KiB - 128) // 128 > sizeof(mi_memid_t) ! \n#define MI_SEGMENT_MAP_PART_BITS (8*MI_SEGMENT_MAP_PART_SIZE)\n#define MI_SEGMENT_MAP_PART_ENTRIES (MI_SEGMENT_MAP_PART_SIZE / MI_INTPTR_SIZE)\n#define MI_SEGMENT_MAP_PART_BIT_SPAN (MI_SEGMENT_ALIGN) // memory area covered by 1 bit\n\n#if (MI_SEGMENT_MAP_PART_BITS < (MI_SEGMENT_MAP_MAX_ADDRESS / MI_SEGMENT_MAP_PART_BIT_SPAN)) // prevent overflow on 32-bit (issue #1017)\n#define MI_SEGMENT_MAP_PART_SPAN (MI_SEGMENT_MAP_PART_BITS * MI_SEGMENT_MAP_PART_BIT_SPAN)\n#else\n#define MI_SEGMENT_MAP_PART_SPAN MI_SEGMENT_MAP_MAX_ADDRESS\n#endif\n\n#define MI_SEGMENT_MAP_MAX_PARTS ((MI_SEGMENT_MAP_MAX_ADDRESS / MI_SEGMENT_MAP_PART_SPAN) + 1)\n\n// A part of the segment map.\ntypedef struct mi_segmap_part_s {\n mi_memid_t memid;\n _Atomic(uintptr_t) map[MI_SEGMENT_MAP_PART_ENTRIES];\n} mi_segmap_part_t;\n\n// Allocate parts on-demand to reduce .bss footprint\nstatic _Atomic(mi_segmap_part_t*) mi_segment_map[MI_SEGMENT_MAP_MAX_PARTS]; // = { NULL, .. }\n\nstatic mi_segmap_part_t* mi_segment_map_index_of(const mi_segment_t* segment, bool create_on_demand, size_t* idx, size_t* bitidx) {\n // note: segment can be invalid or NULL.\n mi_assert_internal(_mi_ptr_segment(segment + 1) == segment); // is it aligned on MI_SEGMENT_SIZE?\n *idx = 0;\n *bitidx = 0; \n if ((uintptr_t)segment >= MI_SEGMENT_MAP_MAX_ADDRESS) return NULL;\n const uintptr_t segindex = ((uintptr_t)segment) / MI_SEGMENT_MAP_PART_SPAN;\n if (segindex >= MI_SEGMENT_MAP_MAX_PARTS) return NULL;\n mi_segmap_part_t* part = mi_atomic_load_ptr_relaxed(mi_segmap_part_t, &mi_segment_map[segindex]);\n\n // allocate on demand to reduce .bss footprint\n if mi_unlikely(part == NULL) {\n if (!create_on_demand) return NULL;\n mi_memid_t memid;\n part = (mi_segmap_part_t*)_mi_os_zalloc(sizeof(mi_segmap_part_t), &memid);\n if (part == NULL) return NULL;\n part->memid = memid;\n mi_segmap_part_t* expected = NULL;\n if (!mi_atomic_cas_ptr_strong_release(mi_segmap_part_t, &mi_segment_map[segindex], &expected, part)) {\n _mi_os_free(part, sizeof(mi_segmap_part_t), memid);\n part = expected;\n if (part == NULL) return NULL;\n }\n }\n mi_assert(part != NULL);\n const uintptr_t offset = ((uintptr_t)segment) % MI_SEGMENT_MAP_PART_SPAN;\n const uintptr_t bitofs = offset / MI_SEGMENT_MAP_PART_BIT_SPAN;\n *idx = bitofs / MI_INTPTR_BITS;\n *bitidx = bitofs % MI_INTPTR_BITS;\n return part;\n}\n\nvoid _mi_segment_map_allocated_at(const mi_segment_t* segment) {\n if (segment->memid.memkind == MI_MEM_ARENA) return; // we lookup segments first in the arena's and don't need the segment map\n size_t index;\n size_t bitidx;\n mi_segmap_part_t* part = mi_segment_map_index_of(segment, true /* alloc map if needed */, &index, &bitidx);\n if (part == NULL) return; // outside our address range..\n uintptr_t mask = mi_atomic_load_relaxed(&part->map[index]);\n uintptr_t newmask;\n do {\n newmask = (mask | ((uintptr_t)1 << bitidx));\n } while (!mi_atomic_cas_weak_release(&part->map[index], &mask, newmask));\n}\n\nvoid _mi_segment_map_freed_at(const mi_segment_t* segment) {\n if (segment->memid.memkind == MI_MEM_ARENA) return;\n size_t index;\n size_t bitidx;\n mi_segmap_part_t* part = mi_segment_map_index_of(segment, false /* don't alloc if not present */, &index, &bitidx);\n if (part == NULL) return; // outside our address range..\n uintptr_t mask = mi_atomic_load_relaxed(&part->map[index]);\n uintptr_t newmask;\n do {\n newmask = (mask & ~((uintptr_t)1 << bitidx));\n } while (!mi_atomic_cas_weak_release(&part->map[index], &mask, newmask));\n}\n\n// Determine the segment belonging to a pointer or NULL if it is not in a valid segment.\nstatic mi_segment_t* _mi_segment_of(const void* p) {\n if (p == NULL) return NULL;\n mi_segment_t* segment = _mi_ptr_segment(p); // segment can be NULL \n size_t index;\n size_t bitidx;\n mi_segmap_part_t* part = mi_segment_map_index_of(segment, false /* dont alloc if not present */, &index, &bitidx);\n if (part == NULL) return NULL; \n const uintptr_t mask = mi_atomic_load_relaxed(&part->map[index]); \n if mi_likely((mask & ((uintptr_t)1 << bitidx)) != 0) {\n // yes, allocated by us\n const bool cookie_ok = (_mi_ptr_cookie(segment) == segment->cookie);\n if mi_likely(cookie_ok) {\n return segment; // yes, allocated by us and valid\n }\n else {\n _mi_error_message(EFAULT, \"segment map found an invalid segment, possibly corrupted meta-data (address=%p, segment=%p)\\n\", p, segment);\n }\n }\n return NULL;\n}\n\n// Is this a valid pointer in our heap?\nstatic bool mi_is_valid_pointer(const void* p) {\n // first check if it is in an arena, then check if it is OS allocated\n return (_mi_arena_contains(p) || _mi_segment_of(p) != NULL);\n}\n\nmi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {\n return mi_is_valid_pointer(p);\n}\n\nvoid _mi_segment_map_unsafe_destroy(void) {\n for (size_t i = 0; i < MI_SEGMENT_MAP_MAX_PARTS; i++) {\n mi_segmap_part_t* part = mi_atomic_exchange_ptr_relaxed(mi_segmap_part_t, &mi_segment_map[i], NULL);\n if (part != NULL) {\n _mi_os_free(part, sizeof(mi_segmap_part_t), part->memid);\n }\n }\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/segment.c", "sha256": "5dd7955307d8dd388ef4a395b9b85b6865774857fcb0d86cf705c2c76f7a2816", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n\n#include // memset\n#include \n\n// -------------------------------------------------------------------\n// Segments\n// mimalloc pages reside in segments. See `mi_segment_valid` for invariants.\n// -------------------------------------------------------------------\n\n\nstatic void mi_segment_try_purge(mi_segment_t* segment, bool force);\n\n\n// -------------------------------------------------------------------\n// commit mask\n// -------------------------------------------------------------------\n\nstatic bool mi_commit_mask_all_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) {\n for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {\n if ((commit->mask[i] & cm->mask[i]) != cm->mask[i]) return false;\n }\n return true;\n}\n\nstatic bool mi_commit_mask_any_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) {\n for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {\n if ((commit->mask[i] & cm->mask[i]) != 0) return true;\n }\n return false;\n}\n\nstatic void mi_commit_mask_create_intersect(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm, mi_commit_mask_t* res) {\n for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {\n res->mask[i] = (commit->mask[i] & cm->mask[i]);\n }\n}\n\nstatic void mi_commit_mask_clear(mi_commit_mask_t* res, const mi_commit_mask_t* cm) {\n for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {\n res->mask[i] &= ~(cm->mask[i]);\n }\n}\n\nstatic void mi_commit_mask_set(mi_commit_mask_t* res, const mi_commit_mask_t* cm) {\n for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {\n res->mask[i] |= cm->mask[i];\n }\n}\n\nstatic void mi_commit_mask_create(size_t bitidx, size_t bitcount, mi_commit_mask_t* cm) {\n mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS);\n mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS);\n if (bitcount == MI_COMMIT_MASK_BITS) {\n mi_assert_internal(bitidx==0);\n mi_commit_mask_create_full(cm);\n }\n else if (bitcount == 0) {\n mi_commit_mask_create_empty(cm);\n }\n else {\n mi_commit_mask_create_empty(cm);\n size_t i = bitidx / MI_COMMIT_MASK_FIELD_BITS;\n size_t ofs = bitidx % MI_COMMIT_MASK_FIELD_BITS;\n while (bitcount > 0) {\n mi_assert_internal(i < MI_COMMIT_MASK_FIELD_COUNT);\n size_t avail = MI_COMMIT_MASK_FIELD_BITS - ofs;\n size_t count = (bitcount > avail ? avail : bitcount);\n size_t mask = (count >= MI_COMMIT_MASK_FIELD_BITS ? ~((size_t)0) : (((size_t)1 << count) - 1) << ofs);\n cm->mask[i] = mask;\n bitcount -= count;\n ofs = 0;\n i++;\n }\n }\n}\n\nsize_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) {\n mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0);\n size_t count = 0;\n for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {\n size_t mask = cm->mask[i];\n if (~mask == 0) {\n count += MI_COMMIT_MASK_FIELD_BITS;\n }\n else {\n for (; mask != 0; mask >>= 1) { // todo: use popcount\n if ((mask&1)!=0) count++;\n }\n }\n }\n // we use total since for huge segments each commit bit may represent a larger size\n return ((total / MI_COMMIT_MASK_BITS) * count);\n}\n\n\nsize_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx) {\n size_t i = (*idx) / MI_COMMIT_MASK_FIELD_BITS;\n size_t ofs = (*idx) % MI_COMMIT_MASK_FIELD_BITS;\n size_t mask = 0;\n // find first ones\n while (i < MI_COMMIT_MASK_FIELD_COUNT) {\n mask = cm->mask[i];\n mask >>= ofs;\n if (mask != 0) {\n while ((mask&1) == 0) {\n mask >>= 1;\n ofs++;\n }\n break;\n }\n i++;\n ofs = 0;\n }\n if (i >= MI_COMMIT_MASK_FIELD_COUNT) {\n // not found\n *idx = MI_COMMIT_MASK_BITS;\n return 0;\n }\n else {\n // found, count ones\n size_t count = 0;\n *idx = (i*MI_COMMIT_MASK_FIELD_BITS) + ofs;\n do {\n mi_assert_internal(ofs < MI_COMMIT_MASK_FIELD_BITS && (mask&1) == 1);\n do {\n count++;\n mask >>= 1;\n } while ((mask&1) == 1);\n if ((((*idx + count) % MI_COMMIT_MASK_FIELD_BITS) == 0)) {\n i++;\n if (i >= MI_COMMIT_MASK_FIELD_COUNT) break;\n mask = cm->mask[i];\n ofs = 0;\n }\n } while ((mask&1) == 1);\n mi_assert_internal(count > 0);\n return count;\n }\n}\n\n\n/* --------------------------------------------------------------------------------\n Segment allocation\n We allocate pages inside bigger \"segments\" (32 MiB on 64-bit). This is to avoid\n splitting VMA's on Linux and reduce fragmentation on other OS's.\n Each thread owns its own segments.\n\n Currently we have:\n - small pages (64KiB)\n - medium pages (512KiB)\n - large pages (4MiB),\n - huge segments have 1 page in one segment that can be larger than `MI_SEGMENT_SIZE`.\n it is used for blocks `> MI_LARGE_OBJ_SIZE_MAX` or with alignment `> MI_BLOCK_ALIGNMENT_MAX`.\n\n The memory for a segment is usually committed on demand.\n (i.e. we are careful to not touch the memory until we actually allocate a block there)\n\n If a thread ends, it \"abandons\" pages that still contain live blocks.\n Such segments are abandoned and these can be reclaimed by still running threads,\n (much like work-stealing).\n-------------------------------------------------------------------------------- */\n\n\n/* -----------------------------------------------------------\n Slices\n----------------------------------------------------------- */\n\n\nstatic const mi_slice_t* mi_segment_slices_end(const mi_segment_t* segment) {\n return &segment->slices[segment->slice_entries];\n}\n\nstatic uint8_t* mi_slice_start(const mi_slice_t* slice) {\n mi_segment_t* segment = _mi_ptr_segment(slice);\n mi_assert_internal(slice >= segment->slices && slice < mi_segment_slices_end(segment));\n return ((uint8_t*)segment + ((slice - segment->slices)*MI_SEGMENT_SLICE_SIZE));\n}\n\n\n/* -----------------------------------------------------------\n Bins\n----------------------------------------------------------- */\n// Use bit scan forward to quickly find the first zero bit if it is available\n\nstatic inline size_t mi_slice_bin8(size_t slice_count) {\n if (slice_count<=1) return slice_count;\n mi_assert_internal(slice_count <= MI_SLICES_PER_SEGMENT);\n slice_count--;\n size_t s = mi_bsr(slice_count); // slice_count > 1\n if (s <= 2) return slice_count + 1;\n size_t bin = ((s << 2) | ((slice_count >> (s - 2))&0x03)) - 4;\n return bin;\n}\n\nstatic inline size_t mi_slice_bin(size_t slice_count) {\n mi_assert_internal(slice_count*MI_SEGMENT_SLICE_SIZE <= MI_SEGMENT_SIZE);\n mi_assert_internal(mi_slice_bin8(MI_SLICES_PER_SEGMENT) <= MI_SEGMENT_BIN_MAX);\n size_t bin = mi_slice_bin8(slice_count);\n mi_assert_internal(bin <= MI_SEGMENT_BIN_MAX);\n return bin;\n}\n\nstatic inline size_t mi_slice_index(const mi_slice_t* slice) {\n mi_segment_t* segment = _mi_ptr_segment(slice);\n ptrdiff_t index = slice - segment->slices;\n mi_assert_internal(index >= 0 && index < (ptrdiff_t)segment->slice_entries);\n return index;\n}\n\n\n/* -----------------------------------------------------------\n Slice span queues\n----------------------------------------------------------- */\n\nstatic void mi_span_queue_push(mi_span_queue_t* sq, mi_slice_t* slice) {\n // todo: or push to the end?\n mi_assert_internal(slice->prev == NULL && slice->next==NULL);\n slice->prev = NULL; // paranoia\n slice->next = sq->first;\n sq->first = slice;\n if (slice->next != NULL) slice->next->prev = slice;\n else sq->last = slice;\n slice->block_size = 0; // free\n}\n\nstatic mi_span_queue_t* mi_span_queue_for(size_t slice_count, mi_segments_tld_t* tld) {\n size_t bin = mi_slice_bin(slice_count);\n mi_span_queue_t* sq = &tld->spans[bin];\n mi_assert_internal(sq->slice_count >= slice_count);\n return sq;\n}\n\nstatic void mi_span_queue_delete(mi_span_queue_t* sq, mi_slice_t* slice) {\n mi_assert_internal(slice->block_size==0 && slice->slice_count>0 && slice->slice_offset==0);\n // should work too if the queue does not contain slice (which can happen during reclaim)\n if (slice->prev != NULL) slice->prev->next = slice->next;\n if (slice == sq->first) sq->first = slice->next;\n if (slice->next != NULL) slice->next->prev = slice->prev;\n if (slice == sq->last) sq->last = slice->prev;\n slice->prev = NULL;\n slice->next = NULL;\n slice->block_size = 1; // no more free\n}\n\n\n/* -----------------------------------------------------------\n Invariant checking\n----------------------------------------------------------- */\n\nstatic bool mi_slice_is_used(const mi_slice_t* slice) {\n return (slice->block_size > 0);\n}\n\n\n#if (MI_DEBUG>=3)\nstatic bool mi_span_queue_contains(mi_span_queue_t* sq, mi_slice_t* slice) {\n for (mi_slice_t* s = sq->first; s != NULL; s = s->next) {\n if (s==slice) return true;\n }\n return false;\n}\n\nstatic bool mi_segment_is_valid(mi_segment_t* segment, mi_segments_tld_t* tld) {\n mi_assert_internal(segment != NULL);\n mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie);\n mi_assert_internal(segment->abandoned <= segment->used);\n mi_assert_internal(segment->thread_id == 0 || segment->thread_id == _mi_thread_id());\n mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); // can only decommit committed blocks\n\n // [specbot S-NEW-1] segment must have at least one slice (L1, O(1))\n mi_assert_internal(segment->segment_slices > 0);\n // [specbot S-NEW-2] info slices must leave room for at least one data slice (L1, O(1))\n mi_assert_internal(segment->segment_info_slices < segment->segment_slices);\n // [specbot S-NEW-3] slice_entries must not exceed array bounds (L1, O(1))\n mi_assert_internal(segment->slice_entries <= MI_SLICES_PER_SEGMENT);\n\n //mi_assert_internal(segment->segment_info_size % MI_SEGMENT_SLICE_SIZE == 0);\n mi_slice_t* slice = &segment->slices[0];\n const mi_slice_t* end = mi_segment_slices_end(segment);\n size_t used_count = 0;\n size_t total_slice_count = 0;\n mi_span_queue_t* sq;\n while(slice < end) {\n mi_assert_internal(slice->slice_count > 0);\n total_slice_count += slice->slice_count;\n mi_assert_internal(slice->slice_offset == 0);\n size_t index = mi_slice_index(slice);\n size_t maxindex = (index + slice->slice_count >= segment->slice_entries ? segment->slice_entries : index + slice->slice_count) - 1;\n if (mi_slice_is_used(slice)) { // a page in use, we need at least MAX_SLICE_OFFSET_COUNT valid back offsets\n used_count++;\n mi_assert_internal(slice->is_huge == (segment->kind == MI_SEGMENT_HUGE));\n for (size_t i = 0; i <= MI_MAX_SLICE_OFFSET_COUNT && index + i <= maxindex; i++) {\n mi_assert_internal(segment->slices[index + i].slice_offset == i*sizeof(mi_slice_t));\n mi_assert_internal(i==0 || segment->slices[index + i].slice_count == 0);\n mi_assert_internal(i==0 || segment->slices[index + i].block_size == 1);\n }\n // and the last entry as well (for coalescing)\n const mi_slice_t* last = slice + slice->slice_count - 1;\n if (last > slice && last < mi_segment_slices_end(segment)) {\n mi_assert_internal(last->slice_offset == (slice->slice_count-1)*sizeof(mi_slice_t));\n mi_assert_internal(last->slice_count == 0);\n mi_assert_internal(last->block_size == 1);\n }\n }\n else { // free range of slices; only last slice needs a valid back offset\n mi_slice_t* last = &segment->slices[maxindex];\n if (segment->kind != MI_SEGMENT_HUGE || slice->slice_count <= (segment->slice_entries - segment->segment_info_slices)) {\n mi_assert_internal((uint8_t*)slice == (uint8_t*)last - last->slice_offset);\n }\n mi_assert_internal(slice == last || last->slice_count == 0 );\n mi_assert_internal(last->block_size == 0 || (segment->kind==MI_SEGMENT_HUGE && last->block_size==1));\n if (segment->kind != MI_SEGMENT_HUGE && segment->thread_id != 0) { // segment is not huge or abandoned\n sq = mi_span_queue_for(slice->slice_count,tld);\n mi_assert_internal(mi_span_queue_contains(sq,slice));\n }\n }\n slice = &segment->slices[maxindex+1];\n }\n mi_assert_internal(slice == end);\n // [specbot S-NEW-5] Total slice counts must sum to segment_slices (L2, O(n))\n mi_assert_internal(total_slice_count == segment->segment_slices);\n mi_assert_internal(used_count == segment->used + 1);\n return true;\n}\n#endif\n\n/* -----------------------------------------------------------\n Segment size calculations\n----------------------------------------------------------- */\n\nstatic size_t mi_segment_info_size(mi_segment_t* segment) {\n return segment->segment_info_slices * MI_SEGMENT_SLICE_SIZE;\n}\n\nstatic uint8_t* _mi_segment_page_start_from_slice(const mi_segment_t* segment, const mi_slice_t* slice, size_t block_size, size_t* page_size)\n{\n const ptrdiff_t idx = slice - segment->slices;\n const size_t psize = (size_t)slice->slice_count * MI_SEGMENT_SLICE_SIZE;\n uint8_t* const pstart = (uint8_t*)segment + (idx*MI_SEGMENT_SLICE_SIZE);\n // make the start not OS page aligned for smaller blocks to avoid page/cache effects\n // note: the offset must always be a block_size multiple since we assume small allocations\n // are aligned (see `mi_heap_malloc_aligned`).\n size_t start_offset = 0;\n if (block_size > 0 && block_size <= MI_MAX_ALIGN_GUARANTEE) {\n // for small objects, ensure the page start is aligned with the block size (PR#66 by kickunderscore)\n const size_t adjust = block_size - ((uintptr_t)pstart % block_size);\n if (adjust < block_size && psize >= block_size + adjust) {\n start_offset += adjust;\n }\n }\n if (block_size >= MI_INTPTR_SIZE) {\n if (block_size <= 64) { start_offset += 3*block_size; }\n else if (block_size <= 512) { start_offset += block_size; }\n }\n start_offset = _mi_align_up(start_offset, MI_MAX_ALIGN_SIZE);\n mi_assert_internal(_mi_is_aligned(pstart + start_offset, MI_MAX_ALIGN_SIZE));\n mi_assert_internal(block_size == 0 || block_size > MI_MAX_ALIGN_GUARANTEE || _mi_is_aligned(pstart + start_offset,block_size));\n if (page_size != NULL) { *page_size = psize - start_offset; }\n return (pstart + start_offset);\n}\n\n// Start of the page available memory; can be used on uninitialized pages\nuint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size)\n{\n const mi_slice_t* slice = mi_page_to_slice((mi_page_t*)page);\n uint8_t* p = _mi_segment_page_start_from_slice(segment, slice, mi_page_block_size(page), page_size);\n mi_assert_internal(mi_page_block_size(page) > 0 || _mi_ptr_page(p) == page);\n mi_assert_internal(_mi_ptr_segment(p) == segment);\n return p;\n}\n\n\nstatic size_t mi_segment_calculate_slices(size_t required, size_t* info_slices) {\n size_t page_size = _mi_os_page_size();\n size_t isize = _mi_align_up(sizeof(mi_segment_t), page_size);\n size_t guardsize = 0;\n\n if (MI_SECURE>0) {\n // in secure mode, we set up a protected page in between the segment info\n // and the page data (and one at the end of the segment)\n guardsize = page_size;\n if (required > 0) {\n required = _mi_align_up(required, MI_SEGMENT_SLICE_SIZE) + page_size;\n }\n }\n\n isize = _mi_align_up(isize + guardsize, MI_SEGMENT_SLICE_SIZE);\n if (info_slices != NULL) *info_slices = isize / MI_SEGMENT_SLICE_SIZE;\n size_t segment_size = (required==0 ? MI_SEGMENT_SIZE : _mi_align_up( required + isize + guardsize, MI_SEGMENT_SLICE_SIZE) );\n mi_assert_internal(segment_size % MI_SEGMENT_SLICE_SIZE == 0);\n return (segment_size / MI_SEGMENT_SLICE_SIZE);\n}\n\n\n/* ----------------------------------------------------------------------------\nSegment caches\nWe keep a small segment cache per thread to increase local\nreuse and avoid setting/clearing guard pages in secure mode.\n------------------------------------------------------------------------------- */\n\nstatic void mi_segments_track_size(long segment_size, mi_segments_tld_t* tld) {\n if (segment_size>=0) _mi_stat_increase(&tld->stats->segments,1);\n else _mi_stat_decrease(&tld->stats->segments,1);\n tld->count += (segment_size >= 0 ? 1 : -1);\n if (tld->count > tld->peak_count) tld->peak_count = tld->count;\n tld->current_size += segment_size;\n if (tld->current_size > tld->peak_size) tld->peak_size = tld->current_size;\n}\n\nstatic void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) {\n segment->thread_id = 0;\n _mi_segment_map_freed_at(segment);\n mi_segments_track_size(-((long)mi_segment_size(segment)),tld);\n if (segment->was_reclaimed) {\n tld->reclaim_count--;\n segment->was_reclaimed = false;\n }\n if (MI_SECURE>0) {\n // _mi_os_unprotect(segment, mi_segment_size(segment)); // ensure no more guard pages are set\n // unprotect the guard pages; we cannot just unprotect the whole segment size as part may be decommitted\n size_t os_pagesize = _mi_os_page_size();\n _mi_os_unprotect((uint8_t*)segment + mi_segment_info_size(segment) - os_pagesize, os_pagesize);\n uint8_t* end = (uint8_t*)segment + mi_segment_size(segment) - os_pagesize;\n _mi_os_unprotect(end, os_pagesize);\n }\n\n // purge delayed decommits now? (no, leave it to the arena)\n // mi_segment_try_purge(segment,true,tld->stats);\n\n const size_t size = mi_segment_size(segment);\n const size_t csize = _mi_commit_mask_committed_size(&segment->commit_mask, size);\n\n _mi_arena_free(segment, mi_segment_size(segment), csize, segment->memid);\n}\n\n/* -----------------------------------------------------------\n Commit/Decommit ranges\n----------------------------------------------------------- */\n\nstatic void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uint8_t* p, size_t size, uint8_t** start_p, size_t* full_size, mi_commit_mask_t* cm) {\n mi_assert_internal(_mi_ptr_segment(p + 1) == segment);\n mi_assert_internal(segment->kind != MI_SEGMENT_HUGE);\n mi_commit_mask_create_empty(cm);\n if (size == 0 || size > MI_SEGMENT_SIZE || segment->kind == MI_SEGMENT_HUGE) return;\n const size_t segstart = mi_segment_info_size(segment);\n const size_t segsize = mi_segment_size(segment);\n if (p >= (uint8_t*)segment + segsize) return;\n\n size_t pstart = (p - (uint8_t*)segment);\n mi_assert_internal(pstart + size <= segsize);\n\n size_t start;\n size_t end;\n if (conservative) {\n // decommit conservative\n start = _mi_align_up(pstart, MI_COMMIT_SIZE);\n end = _mi_align_down(pstart + size, MI_COMMIT_SIZE);\n mi_assert_internal(start >= segstart);\n mi_assert_internal(end <= segsize);\n }\n else {\n // commit liberal\n start = _mi_align_down(pstart, MI_MINIMAL_COMMIT_SIZE);\n end = _mi_align_up(pstart + size, MI_MINIMAL_COMMIT_SIZE);\n }\n if (pstart >= segstart && start < segstart) { // note: the mask is also calculated for an initial commit of the info area\n start = segstart;\n }\n if (end > segsize) {\n end = segsize;\n }\n\n mi_assert_internal(start <= pstart && (pstart + size) <= end);\n mi_assert_internal(start % MI_COMMIT_SIZE==0 && end % MI_COMMIT_SIZE == 0);\n *start_p = (uint8_t*)segment + start;\n *full_size = (end > start ? end - start : 0);\n if (*full_size == 0) return;\n\n size_t bitidx = start / MI_COMMIT_SIZE;\n mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS);\n\n size_t bitcount = *full_size / MI_COMMIT_SIZE; // can be 0\n if (bitidx + bitcount > MI_COMMIT_MASK_BITS) {\n _mi_warning_message(\"commit mask overflow: idx=%zu count=%zu start=%zx end=%zx p=0x%p size=%zu fullsize=%zu\\n\", bitidx, bitcount, start, end, p, size, *full_size);\n }\n mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS);\n mi_commit_mask_create(bitidx, bitcount, cm);\n}\n\nstatic bool mi_segment_commit(mi_segment_t* segment, uint8_t* p, size_t size) {\n mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask));\n\n // commit liberal\n uint8_t* start = NULL;\n size_t full_size = 0;\n mi_commit_mask_t mask;\n mi_segment_commit_mask(segment, false /* conservative? */, p, size, &start, &full_size, &mask);\n if (mi_commit_mask_is_empty(&mask) || full_size == 0) return true;\n\n if (!mi_commit_mask_all_set(&segment->commit_mask, &mask)) {\n // committing\n bool is_zero = false;\n mi_commit_mask_t cmask;\n mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);\n _mi_stat_decrease(&_mi_stats_main.committed, _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap\n if (!_mi_os_commit(start, full_size, &is_zero)) return false;\n mi_commit_mask_set(&segment->commit_mask, &mask);\n }\n\n // increase purge expiration when using part of delayed purges -- we assume more allocations are coming soon.\n if (mi_commit_mask_any_set(&segment->purge_mask, &mask)) {\n segment->purge_expire = _mi_clock_now() + mi_option_get(mi_option_purge_delay);\n }\n\n // always clear any delayed purges in our range (as they are either committed now)\n mi_commit_mask_clear(&segment->purge_mask, &mask);\n return true;\n}\n\nstatic bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_t size) {\n mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask));\n // note: assumes commit_mask is always full for huge segments as otherwise the commit mask bits can overflow\n if (mi_commit_mask_is_full(&segment->commit_mask) && mi_commit_mask_is_empty(&segment->purge_mask)) return true; // fully committed\n mi_assert_internal(segment->kind != MI_SEGMENT_HUGE);\n return mi_segment_commit(segment, p, size);\n}\n\nstatic bool mi_segment_purge(mi_segment_t* segment, uint8_t* p, size_t size) {\n mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask));\n if (!segment->allow_purge) return true;\n\n // purge conservative\n uint8_t* start = NULL;\n size_t full_size = 0;\n mi_commit_mask_t mask;\n mi_segment_commit_mask(segment, true /* conservative? */, p, size, &start, &full_size, &mask);\n if (mi_commit_mask_is_empty(&mask) || full_size==0) return true;\n\n if (mi_commit_mask_any_set(&segment->commit_mask, &mask)) {\n // purging\n mi_assert_internal((void*)start != (void*)segment);\n mi_assert_internal(segment->allow_decommit);\n const bool decommitted = _mi_os_purge(start, full_size); // reset or decommit\n if (decommitted) {\n mi_commit_mask_t cmask;\n mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);\n _mi_stat_increase(&_mi_stats_main.committed, full_size - _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for double counting\n mi_commit_mask_clear(&segment->commit_mask, &mask);\n }\n }\n\n // always clear any scheduled purges in our range\n mi_commit_mask_clear(&segment->purge_mask, &mask);\n return true;\n}\n\nstatic void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t size) {\n if (!segment->allow_purge) return;\n\n if (mi_option_get(mi_option_purge_delay) == 0) {\n mi_segment_purge(segment, p, size);\n }\n else {\n // register for future purge in the purge mask\n uint8_t* start = NULL;\n size_t full_size = 0;\n mi_commit_mask_t mask;\n mi_segment_commit_mask(segment, true /*conservative*/, p, size, &start, &full_size, &mask);\n if (mi_commit_mask_is_empty(&mask) || full_size==0) return;\n\n // update delayed commit\n mi_assert_internal(segment->purge_expire > 0 || mi_commit_mask_is_empty(&segment->purge_mask));\n mi_commit_mask_t cmask;\n mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); // only purge what is committed; span_free may try to decommit more\n mi_commit_mask_set(&segment->purge_mask, &cmask);\n mi_msecs_t now = _mi_clock_now();\n if (segment->purge_expire == 0) {\n // no previous purgess, initialize now\n segment->purge_expire = now + mi_option_get(mi_option_purge_delay);\n }\n else if (segment->purge_expire <= now) {\n // previous purge mask already expired\n if (segment->purge_expire + mi_option_get(mi_option_purge_extend_delay) <= now) {\n mi_segment_try_purge(segment, true);\n }\n else {\n segment->purge_expire = now + mi_option_get(mi_option_purge_extend_delay); // (mi_option_get(mi_option_purge_delay) / 8); // wait a tiny bit longer in case there is a series of free's\n }\n }\n else {\n // previous purge mask is not yet expired, increase the expiration by a bit.\n segment->purge_expire += mi_option_get(mi_option_purge_extend_delay);\n }\n }\n}\n\nstatic void mi_segment_try_purge(mi_segment_t* segment, bool force) {\n if (!segment->allow_purge || segment->purge_expire == 0 || mi_commit_mask_is_empty(&segment->purge_mask)) return;\n mi_msecs_t now = _mi_clock_now();\n if (!force && now < segment->purge_expire) return;\n\n mi_commit_mask_t mask = segment->purge_mask;\n segment->purge_expire = 0;\n mi_commit_mask_create_empty(&segment->purge_mask);\n\n size_t idx;\n size_t count;\n mi_commit_mask_foreach(&mask, idx, count) {\n // if found, decommit that sequence\n if (count > 0) {\n uint8_t* p = (uint8_t*)segment + (idx*MI_COMMIT_SIZE);\n size_t size = count * MI_COMMIT_SIZE;\n mi_segment_purge(segment, p, size);\n }\n }\n mi_commit_mask_foreach_end()\n mi_assert_internal(mi_commit_mask_is_empty(&segment->purge_mask));\n}\n\n// called from `mi_heap_collect_ex`\n// this can be called per-page so it is important that try_purge has fast exit path\nvoid _mi_segment_collect(mi_segment_t* segment, bool force) {\n mi_segment_try_purge(segment, force);\n}\n\n/* -----------------------------------------------------------\n Span free\n----------------------------------------------------------- */\n\nstatic bool mi_segment_is_abandoned(mi_segment_t* segment) {\n return (mi_atomic_load_relaxed(&segment->thread_id) == 0);\n}\n\n// note: can be called on abandoned segments\nstatic void mi_segment_span_free(mi_segment_t* segment, size_t slice_index, size_t slice_count, bool allow_purge, mi_segments_tld_t* tld) {\n mi_assert_internal(slice_index < segment->slice_entries);\n mi_span_queue_t* sq = (segment->kind == MI_SEGMENT_HUGE || mi_segment_is_abandoned(segment)\n ? NULL : mi_span_queue_for(slice_count,tld));\n if (slice_count==0) slice_count = 1;\n mi_assert_internal(slice_index + slice_count - 1 < segment->slice_entries);\n\n // set first and last slice (the intermediates can be undetermined)\n mi_slice_t* slice = &segment->slices[slice_index];\n slice->slice_count = (uint32_t)slice_count;\n mi_assert_internal(slice->slice_count == slice_count); // no overflow?\n slice->slice_offset = 0;\n if (slice_count > 1) {\n mi_slice_t* last = slice + slice_count - 1;\n mi_slice_t* end = (mi_slice_t*)mi_segment_slices_end(segment);\n if (last > end) { last = end; }\n last->slice_count = 0;\n last->slice_offset = (uint32_t)(sizeof(mi_page_t)*(slice_count - 1));\n last->block_size = 0;\n }\n\n // perhaps decommit\n if (allow_purge) {\n mi_segment_schedule_purge(segment, mi_slice_start(slice), slice_count * MI_SEGMENT_SLICE_SIZE);\n }\n\n // and push it on the free page queue (if it was not a huge page)\n if (sq != NULL) mi_span_queue_push( sq, slice );\n else slice->block_size = 0; // mark huge page as free anyways\n}\n\n/*\n// called from reclaim to add existing free spans\nstatic void mi_segment_span_add_free(mi_slice_t* slice, mi_segments_tld_t* tld) {\n mi_segment_t* segment = _mi_ptr_segment(slice);\n mi_assert_internal(slice->xblock_size==0 && slice->slice_count>0 && slice->slice_offset==0);\n size_t slice_index = mi_slice_index(slice);\n mi_segment_span_free(segment,slice_index,slice->slice_count,tld);\n}\n*/\n\nstatic void mi_segment_span_remove_from_queue(mi_slice_t* slice, mi_segments_tld_t* tld) {\n mi_assert_internal(slice->slice_count > 0 && slice->slice_offset==0 && slice->block_size==0);\n mi_assert_internal(_mi_ptr_segment(slice)->kind != MI_SEGMENT_HUGE);\n mi_span_queue_t* sq = mi_span_queue_for(slice->slice_count, tld);\n mi_span_queue_delete(sq, slice);\n}\n\n// note: can be called on abandoned segments\nstatic mi_slice_t* mi_segment_span_free_coalesce(mi_slice_t* slice, mi_segments_tld_t* tld) {\n mi_assert_internal(slice != NULL && slice->slice_count > 0 && slice->slice_offset == 0);\n mi_segment_t* const segment = _mi_ptr_segment(slice);\n\n // for huge pages, just mark as free but don't add to the queues\n if (segment->kind == MI_SEGMENT_HUGE) {\n // issue #691: segment->used can be 0 if the huge page block was freed while abandoned (reclaim will get here in that case)\n mi_assert_internal((segment->used==0 && slice->block_size==0) || segment->used == 1); // decreased right after this call in `mi_segment_page_clear`\n slice->block_size = 0; // mark as free anyways\n // we should mark the last slice `xblock_size=0` now to maintain invariants but we skip it to\n // avoid a possible cache miss (and the segment is about to be freed)\n return slice;\n }\n\n // otherwise coalesce the span and add to the free span queues\n const bool is_abandoned = mi_segment_is_abandoned(segment);\n size_t slice_count = slice->slice_count;\n mi_slice_t* next = slice + slice->slice_count;\n mi_assert_internal(next <= mi_segment_slices_end(segment));\n if (next < mi_segment_slices_end(segment) && next->block_size==0) {\n // free next block -- remove it from free and merge\n mi_assert_internal(next->slice_count > 0 && next->slice_offset==0);\n slice_count += next->slice_count; // extend\n if (!is_abandoned) { mi_segment_span_remove_from_queue(next, tld); }\n }\n if (slice > segment->slices) {\n mi_slice_t* prev = mi_slice_first(slice - 1);\n mi_assert_internal(prev >= segment->slices);\n if (prev->block_size==0) {\n // free previous slice -- remove it from free and merge\n mi_assert_internal(prev->slice_count > 0 && prev->slice_offset==0);\n slice_count += prev->slice_count;\n slice->slice_count = 0;\n slice->slice_offset = (uint32_t)((uint8_t*)slice - (uint8_t*)prev); // set the slice offset for `segment_force_abandon` (in case the previous free block is very large).\n if (!is_abandoned) { mi_segment_span_remove_from_queue(prev, tld); }\n slice = prev;\n }\n }\n\n // and add the new free page\n mi_segment_span_free(segment, mi_slice_index(slice), slice_count, true, tld);\n return slice;\n}\n\n\n\n/* -----------------------------------------------------------\n Page allocation\n----------------------------------------------------------- */\n\n// Note: may still return NULL if committing the memory failed\nstatic mi_page_t* mi_segment_span_allocate(mi_segment_t* segment, size_t slice_index, size_t slice_count) {\n mi_assert_internal(slice_index < segment->slice_entries);\n mi_slice_t* const slice = &segment->slices[slice_index];\n mi_assert_internal(slice->block_size==0 || slice->block_size==1);\n\n // commit before changing the slice data\n if (!mi_segment_ensure_committed(segment, _mi_segment_page_start_from_slice(segment, slice, 0, NULL), slice_count * MI_SEGMENT_SLICE_SIZE)) {\n return NULL; // commit failed!\n }\n\n // convert the slices to a page\n slice->slice_offset = 0;\n slice->slice_count = (uint32_t)slice_count;\n mi_assert_internal(slice->slice_count == slice_count);\n const size_t bsize = slice_count * MI_SEGMENT_SLICE_SIZE;\n slice->block_size = bsize;\n mi_page_t* page = mi_slice_to_page(slice);\n mi_assert_internal(mi_page_block_size(page) == bsize);\n\n // set slice back pointers for the first MI_MAX_SLICE_OFFSET_COUNT entries\n size_t extra = slice_count-1;\n if (extra > MI_MAX_SLICE_OFFSET_COUNT) extra = MI_MAX_SLICE_OFFSET_COUNT;\n if (slice_index + extra >= segment->slice_entries) extra = segment->slice_entries - slice_index - 1; // huge objects may have more slices than avaiable entries in the segment->slices\n\n mi_slice_t* slice_next = slice + 1;\n for (size_t i = 1; i <= extra; i++, slice_next++) {\n slice_next->slice_offset = (uint32_t)(sizeof(mi_slice_t)*i);\n slice_next->slice_count = 0;\n slice_next->block_size = 1;\n }\n\n // and also for the last one (if not set already) (the last one is needed for coalescing and for large alignments)\n // note: the cast is needed for ubsan since the index can be larger than MI_SLICES_PER_SEGMENT for huge allocations (see #543)\n mi_slice_t* last = slice + slice_count - 1;\n mi_slice_t* end = (mi_slice_t*)mi_segment_slices_end(segment);\n if (last > end) last = end;\n if (last > slice) {\n last->slice_offset = (uint32_t)(sizeof(mi_slice_t) * (last - slice));\n last->slice_count = 0;\n last->block_size = 1;\n }\n\n // and initialize the page\n page->is_committed = true;\n page->is_zero_init = segment->free_is_zero;\n page->is_huge = (segment->kind == MI_SEGMENT_HUGE);\n segment->used++;\n return page;\n}\n\nstatic void mi_segment_slice_split(mi_segment_t* segment, mi_slice_t* slice, size_t slice_count, mi_segments_tld_t* tld) {\n mi_assert_internal(_mi_ptr_segment(slice) == segment);\n mi_assert_internal(slice->slice_count >= slice_count);\n mi_assert_internal(slice->block_size > 0); // no more in free queue\n if (slice->slice_count <= slice_count) return;\n mi_assert_internal(segment->kind != MI_SEGMENT_HUGE);\n size_t next_index = mi_slice_index(slice) + slice_count;\n size_t next_count = slice->slice_count - slice_count;\n mi_segment_span_free(segment, next_index, next_count, false /* don't purge left-over part */, tld);\n slice->slice_count = (uint32_t)slice_count;\n}\n\nstatic mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld) {\n mi_assert_internal(slice_count*MI_SEGMENT_SLICE_SIZE <= MI_LARGE_OBJ_SIZE_MAX);\n // search from best fit up\n mi_span_queue_t* sq = mi_span_queue_for(slice_count, tld);\n if (slice_count == 0) slice_count = 1;\n while (sq <= &tld->spans[MI_SEGMENT_BIN_MAX]) {\n for (mi_slice_t* slice = sq->first; slice != NULL; slice = slice->next) {\n if (slice->slice_count >= slice_count) {\n // found one\n mi_segment_t* segment = _mi_ptr_segment(slice);\n if (_mi_arena_memid_is_suitable(segment->memid, req_arena_id)) {\n // found a suitable page span\n mi_span_queue_delete(sq, slice);\n\n if (slice->slice_count > slice_count) {\n mi_segment_slice_split(segment, slice, slice_count, tld);\n }\n mi_assert_internal(slice != NULL && slice->slice_count == slice_count && slice->block_size > 0);\n mi_page_t* page = mi_segment_span_allocate(segment, mi_slice_index(slice), slice->slice_count);\n if (page == NULL) {\n // commit failed; return NULL but first restore the slice\n mi_segment_span_free_coalesce(slice, tld);\n return NULL;\n }\n return page;\n }\n }\n }\n sq++;\n }\n // could not find a page..\n return NULL;\n}\n\n\n/* -----------------------------------------------------------\n Segment allocation\n----------------------------------------------------------- */\n\nstatic mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment, bool eager_delayed, mi_arena_id_t req_arena_id,\n size_t* psegment_slices, size_t* pinfo_slices,\n bool commit, mi_segments_tld_t* tld)\n\n{\n mi_memid_t memid;\n bool allow_large = (!eager_delayed && (MI_SECURE == 0)); // only allow large OS pages once we are no longer lazy\n size_t align_offset = 0;\n size_t alignment = MI_SEGMENT_ALIGN;\n\n if (page_alignment > 0) {\n // mi_assert_internal(huge_page != NULL);\n mi_assert_internal(page_alignment >= MI_SEGMENT_ALIGN);\n alignment = page_alignment;\n const size_t info_size = (*pinfo_slices) * MI_SEGMENT_SLICE_SIZE;\n align_offset = _mi_align_up( info_size, MI_SEGMENT_ALIGN );\n const size_t extra = align_offset - info_size;\n // recalculate due to potential guard pages\n *psegment_slices = mi_segment_calculate_slices(required + extra, pinfo_slices);\n mi_assert_internal(*psegment_slices > 0 && *psegment_slices <= UINT32_MAX);\n }\n\n const size_t segment_size = (*psegment_slices) * MI_SEGMENT_SLICE_SIZE;\n mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid);\n if (segment == NULL) {\n return NULL; // failed to allocate\n }\n\n // ensure metadata part of the segment is committed\n mi_commit_mask_t commit_mask;\n if (memid.initially_committed) {\n mi_commit_mask_create_full(&commit_mask);\n }\n else {\n // at least commit the info slices\n const size_t commit_needed = _mi_divide_up((*pinfo_slices)*MI_SEGMENT_SLICE_SIZE, MI_COMMIT_SIZE);\n mi_assert_internal(commit_needed>0);\n mi_commit_mask_create(0, commit_needed, &commit_mask);\n mi_assert_internal(commit_needed*MI_COMMIT_SIZE >= (*pinfo_slices)*MI_SEGMENT_SLICE_SIZE);\n if (!_mi_os_commit(segment, commit_needed*MI_COMMIT_SIZE, NULL)) {\n _mi_arena_free(segment,segment_size,0,memid);\n return NULL;\n }\n }\n mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0);\n\n segment->memid = memid;\n segment->allow_decommit = !memid.is_pinned;\n segment->allow_purge = segment->allow_decommit && (mi_option_get(mi_option_purge_delay) >= 0);\n segment->segment_size = segment_size;\n segment->subproc = tld->subproc;\n segment->commit_mask = commit_mask;\n segment->purge_expire = 0;\n segment->free_is_zero = memid.initially_zero;\n mi_commit_mask_create_empty(&segment->purge_mask);\n\n mi_segments_track_size((long)(segment_size), tld);\n _mi_segment_map_allocated_at(segment);\n return segment;\n}\n\n\n// Allocate a segment from the OS aligned to `MI_SEGMENT_SIZE` .\nstatic mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_page_t** huge_page)\n{\n mi_assert_internal((required==0 && huge_page==NULL) || (required>0 && huge_page != NULL));\n\n // calculate needed sizes first\n size_t info_slices;\n size_t segment_slices = mi_segment_calculate_slices(required, &info_slices);\n mi_assert_internal(segment_slices > 0 && segment_slices <= UINT32_MAX);\n\n // Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little)\n const bool eager_delay = (// !_mi_os_has_overcommit() && // never delay on overcommit systems\n _mi_current_thread_count() > 1 && // do not delay for the first N threads\n tld->peak_count < (size_t)mi_option_get(mi_option_eager_commit_delay));\n const bool eager = !eager_delay && mi_option_is_enabled(mi_option_eager_commit);\n bool commit = eager || (required > 0);\n\n // Allocate the segment from the OS\n mi_segment_t* segment = mi_segment_os_alloc(required, page_alignment, eager_delay, req_arena_id,\n &segment_slices, &info_slices, commit, tld);\n if (segment == NULL) return NULL;\n\n // zero the segment info? -- not always needed as it may be zero initialized from the OS\n if (!segment->memid.initially_zero) {\n ptrdiff_t ofs = offsetof(mi_segment_t, next);\n size_t prefix = offsetof(mi_segment_t, slices) - ofs;\n size_t zsize = prefix + (sizeof(mi_slice_t) * (segment_slices + 1)); // one more\n _mi_memzero((uint8_t*)segment + ofs, zsize);\n }\n\n // initialize the rest of the segment info\n const size_t slice_entries = (segment_slices > MI_SLICES_PER_SEGMENT ? MI_SLICES_PER_SEGMENT : segment_slices);\n segment->segment_slices = segment_slices;\n segment->segment_info_slices = info_slices;\n segment->thread_id = _mi_thread_id();\n segment->cookie = _mi_ptr_cookie(segment);\n segment->slice_entries = slice_entries;\n segment->kind = (required == 0 ? MI_SEGMENT_NORMAL : MI_SEGMENT_HUGE);\n\n // _mi_memzero(segment->slices, sizeof(mi_slice_t)*(info_slices+1));\n _mi_stat_increase(&tld->stats->page_committed, mi_segment_info_size(segment));\n\n // set up guard pages\n size_t guard_slices = 0;\n if (MI_SECURE>0) {\n // in secure mode, we set up a protected page in between the segment info\n // and the page data, and at the end of the segment.\n size_t os_pagesize = _mi_os_page_size();\n _mi_os_protect((uint8_t*)segment + mi_segment_info_size(segment) - os_pagesize, os_pagesize);\n uint8_t* end = (uint8_t*)segment + mi_segment_size(segment) - os_pagesize;\n mi_segment_ensure_committed(segment, end, os_pagesize);\n _mi_os_protect(end, os_pagesize);\n if (slice_entries == segment_slices) segment->slice_entries--; // don't use the last slice :-(\n guard_slices = 1;\n }\n\n // reserve first slices for segment info\n mi_page_t* page0 = mi_segment_span_allocate(segment, 0, info_slices);\n mi_assert_internal(page0!=NULL); if (page0==NULL) return NULL; // cannot fail as we always commit in advance\n mi_assert_internal(segment->used == 1);\n segment->used = 0; // don't count our internal slices towards usage\n\n // initialize initial free pages\n if (segment->kind == MI_SEGMENT_NORMAL) { // not a huge page\n mi_assert_internal(huge_page==NULL);\n mi_segment_span_free(segment, info_slices, segment->slice_entries - info_slices, false /* don't purge */, tld);\n }\n else {\n mi_assert_internal(huge_page!=NULL);\n mi_assert_internal(mi_commit_mask_is_empty(&segment->purge_mask));\n mi_assert_internal(mi_commit_mask_is_full(&segment->commit_mask));\n *huge_page = mi_segment_span_allocate(segment, info_slices, segment_slices - info_slices - guard_slices);\n mi_assert_internal(*huge_page != NULL); // cannot fail as we commit in advance\n }\n\n mi_assert_expensive(mi_segment_is_valid(segment,tld));\n return segment;\n}\n\n\nstatic void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) {\n MI_UNUSED(force);\n mi_assert_internal(segment != NULL);\n mi_assert_internal(segment->next == NULL);\n mi_assert_internal(segment->used == 0);\n\n // in `mi_segment_force_abandon` we set this to true to ensure the segment's memory stays valid\n if (segment->dont_free) return;\n\n // Remove the free pages\n mi_slice_t* slice = &segment->slices[0];\n const mi_slice_t* end = mi_segment_slices_end(segment);\n #if MI_DEBUG>1\n size_t page_count = 0;\n #endif\n while (slice < end) {\n mi_assert_internal(slice->slice_count > 0);\n mi_assert_internal(slice->slice_offset == 0);\n mi_assert_internal(mi_slice_index(slice)==0 || slice->block_size == 0); // no more used pages ..\n if (slice->block_size == 0 && segment->kind != MI_SEGMENT_HUGE) {\n mi_segment_span_remove_from_queue(slice, tld);\n }\n #if MI_DEBUG>1\n page_count++;\n #endif\n slice = slice + slice->slice_count;\n }\n mi_assert_internal(page_count == 2); // first page is allocated by the segment itself\n\n // stats\n // _mi_stat_decrease(&tld->stats->page_committed, mi_segment_info_size(segment));\n\n // return it to the OS\n mi_segment_os_free(segment, tld);\n}\n\n\n/* -----------------------------------------------------------\n Page Free\n----------------------------------------------------------- */\n\nstatic void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld);\n\n// note: can be called on abandoned pages\nstatic mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld) {\n mi_assert_internal(page->block_size > 0);\n mi_assert_internal(mi_page_all_free(page));\n mi_segment_t* segment = _mi_ptr_segment(page);\n mi_assert_internal(segment->used > 0);\n\n size_t inuse = page->capacity * mi_page_block_size(page);\n _mi_stat_decrease(&tld->stats->page_committed, inuse);\n _mi_stat_decrease(&tld->stats->pages, 1);\n _mi_stat_decrease(&tld->stats->page_bins[_mi_page_stats_bin(page)], 1);\n \n // reset the page memory to reduce memory pressure?\n if (segment->allow_decommit && mi_option_is_enabled(mi_option_deprecated_page_reset)) {\n size_t psize;\n uint8_t* start = _mi_segment_page_start(segment, page, &psize);\n _mi_os_reset(start, psize);\n }\n\n // zero the page data, but not the segment fields and heap tag\n page->is_zero_init = false;\n uint8_t heap_tag = page->heap_tag;\n ptrdiff_t ofs = offsetof(mi_page_t, capacity);\n _mi_memzero((uint8_t*)page + ofs, sizeof(*page) - ofs);\n page->block_size = 1;\n page->heap_tag = heap_tag;\n\n // and free it\n mi_slice_t* slice = mi_segment_span_free_coalesce(mi_page_to_slice(page), tld);\n segment->used--;\n segment->free_is_zero = false;\n\n // cannot assert segment valid as it is called during reclaim\n // mi_assert_expensive(mi_segment_is_valid(segment, tld));\n return slice;\n}\n\nvoid _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld)\n{\n mi_assert(page != NULL);\n mi_segment_t* segment = _mi_page_segment(page);\n mi_assert_expensive(mi_segment_is_valid(segment,tld));\n\n // mark it as free now\n mi_segment_page_clear(page, tld);\n mi_assert_expensive(mi_segment_is_valid(segment, tld));\n\n if (segment->used == 0) {\n // no more used pages; remove from the free list and free the segment\n mi_segment_free(segment, force, tld);\n }\n else if (segment->used == segment->abandoned) {\n // only abandoned pages; remove from free list and abandon\n mi_segment_abandon(segment,tld);\n }\n else {\n // perform delayed purges\n mi_segment_try_purge(segment, false /* force? */);\n }\n}\n\n\n/* -----------------------------------------------------------\nAbandonment\n\nWhen threads terminate, they can leave segments with\nlive blocks (reachable through other threads). Such segments\nare \"abandoned\" and will be reclaimed by other threads to\nreuse their pages and/or free them eventually. The\n`thread_id` of such segments is 0.\n\nWhen a block is freed in an abandoned segment, the segment\nis reclaimed into that thread.\n\nMoreover, if threads are looking for a fresh segment, they\nwill first consider abandoned segments -- these can be found\nby scanning the arena memory\n(segments outside arena memoryare only reclaimed by a free).\n----------------------------------------------------------- */\n\n/* -----------------------------------------------------------\n Abandon segment/page\n----------------------------------------------------------- */\n\nstatic void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) {\n mi_assert_internal(segment->used == segment->abandoned);\n mi_assert_internal(segment->used > 0);\n mi_assert_internal(segment->abandoned_visits == 0);\n mi_assert_expensive(mi_segment_is_valid(segment,tld));\n\n // remove the free pages from the free page queues\n mi_slice_t* slice = &segment->slices[0];\n const mi_slice_t* end = mi_segment_slices_end(segment);\n while (slice < end) {\n mi_assert_internal(slice->slice_count > 0);\n mi_assert_internal(slice->slice_offset == 0);\n if (slice->block_size == 0) { // a free page\n mi_segment_span_remove_from_queue(slice,tld);\n slice->block_size = 0; // but keep it free\n }\n slice = slice + slice->slice_count;\n }\n\n // perform delayed decommits (forcing is much slower on mstress)\n // Only abandoned segments in arena memory can be reclaimed without a free\n // so if a segment is not from an arena we force purge here to be conservative.\n const bool force_purge = (segment->memid.memkind != MI_MEM_ARENA) || mi_option_is_enabled(mi_option_abandoned_page_purge);\n mi_segment_try_purge(segment, force_purge);\n\n // all pages in the segment are abandoned; add it to the abandoned list\n _mi_stat_increase(&tld->stats->segments_abandoned, 1);\n mi_segments_track_size(-((long)mi_segment_size(segment)), tld);\n segment->thread_id = 0;\n segment->abandoned_visits = 1; // from 0 to 1 to signify it is abandoned\n if (segment->was_reclaimed) {\n tld->reclaim_count--;\n segment->was_reclaimed = false;\n }\n _mi_arena_segment_mark_abandoned(segment);\n}\n\nvoid _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld) {\n mi_assert(page != NULL);\n mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE);\n mi_assert_internal(mi_page_heap(page) == NULL);\n mi_segment_t* segment = _mi_page_segment(page);\n\n mi_assert_expensive(mi_segment_is_valid(segment,tld));\n segment->abandoned++;\n\n _mi_stat_increase(&tld->stats->pages_abandoned, 1);\n mi_assert_internal(segment->abandoned <= segment->used);\n if (segment->used == segment->abandoned) {\n // all pages are abandoned, abandon the entire segment\n mi_segment_abandon(segment, tld);\n }\n}\n\n/* -----------------------------------------------------------\n Reclaim abandoned pages\n----------------------------------------------------------- */\n\nstatic mi_slice_t* mi_slices_start_iterate(mi_segment_t* segment, const mi_slice_t** end) {\n mi_slice_t* slice = &segment->slices[0];\n *end = mi_segment_slices_end(segment);\n mi_assert_internal(slice->slice_count>0 && slice->block_size>0); // segment allocated page\n slice = slice + slice->slice_count; // skip the first segment allocated page\n return slice;\n}\n\n// Possibly free pages and check if free space is available\nstatic bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, size_t block_size, mi_segments_tld_t* tld)\n{\n mi_assert_internal(mi_segment_is_abandoned(segment));\n bool has_page = false;\n\n // for all slices\n const mi_slice_t* end;\n mi_slice_t* slice = mi_slices_start_iterate(segment, &end);\n while (slice < end) {\n mi_assert_internal(slice->slice_count > 0);\n mi_assert_internal(slice->slice_offset == 0);\n if (mi_slice_is_used(slice)) { // used page\n // ensure used count is up to date and collect potential concurrent frees\n mi_page_t* const page = mi_slice_to_page(slice);\n _mi_page_free_collect(page, false);\n if (mi_page_all_free(page)) {\n // if this page is all free now, free it without adding to any queues (yet)\n mi_assert_internal(page->next == NULL && page->prev==NULL);\n _mi_stat_decrease(&tld->stats->pages_abandoned, 1);\n segment->abandoned--;\n slice = mi_segment_page_clear(page, tld); // re-assign slice due to coalesce!\n mi_assert_internal(!mi_slice_is_used(slice));\n if (slice->slice_count >= slices_needed) {\n has_page = true;\n }\n }\n else if (mi_page_block_size(page) == block_size && mi_page_has_any_available(page)) {\n // a page has available free blocks of the right size\n has_page = true;\n }\n }\n else {\n // empty span\n if (slice->slice_count >= slices_needed) {\n has_page = true;\n }\n }\n slice = slice + slice->slice_count;\n }\n return has_page;\n}\n\n// Reclaim an abandoned segment; returns NULL if the segment was freed\n// set `right_page_reclaimed` to `true` if it reclaimed a page of the right `block_size` that was not full.\nstatic mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, size_t requested_block_size, bool* right_page_reclaimed, mi_segments_tld_t* tld) {\n if (right_page_reclaimed != NULL) { *right_page_reclaimed = false; }\n // can be 0 still with abandoned_next, or already a thread id for segments outside an arena that are reclaimed on a free.\n mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == 0 || mi_atomic_load_relaxed(&segment->thread_id) == _mi_thread_id());\n mi_assert_internal(segment->subproc == heap->tld->segments.subproc); // only reclaim within the same subprocess\n mi_atomic_store_release(&segment->thread_id, _mi_thread_id());\n segment->abandoned_visits = 0;\n segment->was_reclaimed = true;\n tld->reclaim_count++;\n mi_segments_track_size((long)mi_segment_size(segment), tld);\n mi_assert_internal(segment->next == NULL);\n _mi_stat_decrease(&tld->stats->segments_abandoned, 1);\n\n // for all slices\n const mi_slice_t* end;\n mi_slice_t* slice = mi_slices_start_iterate(segment, &end);\n while (slice < end) {\n mi_assert_internal(slice->slice_count > 0);\n mi_assert_internal(slice->slice_offset == 0);\n if (mi_slice_is_used(slice)) {\n // in use: reclaim the page in our heap\n mi_page_t* page = mi_slice_to_page(slice);\n mi_assert_internal(page->is_committed);\n mi_assert_internal(mi_page_thread_free_flag(page)==MI_NEVER_DELAYED_FREE);\n mi_assert_internal(mi_page_heap(page) == NULL);\n mi_assert_internal(page->next == NULL && page->prev==NULL);\n _mi_stat_decrease(&tld->stats->pages_abandoned, 1);\n segment->abandoned--;\n // get the target heap for this thread which has a matching heap tag (so we reclaim into a matching heap)\n mi_heap_t* target_heap = _mi_heap_by_tag(heap, page->heap_tag); // allow custom heaps to separate objects\n if (target_heap == NULL) {\n target_heap = heap;\n _mi_error_message(EFAULT, \"page with tag %u cannot be reclaimed by a heap with the same tag (using heap tag %u instead)\\n\", page->heap_tag, heap->tag );\n }\n // associate the heap with this page, and allow heap thread delayed free again.\n mi_page_set_heap(page, target_heap);\n _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after heap is set)\n _mi_page_free_collect(page, false); // ensure used count is up to date\n if (mi_page_all_free(page)) {\n // if everything free by now, free the page\n slice = mi_segment_page_clear(page, tld); // set slice again due to coalesceing\n }\n else {\n // otherwise reclaim it into the heap\n _mi_page_reclaim(target_heap, page);\n if (requested_block_size == mi_page_block_size(page) && mi_page_has_any_available(page) && heap == target_heap) {\n if (right_page_reclaimed != NULL) { *right_page_reclaimed = true; }\n }\n }\n }\n else {\n // the span is free, add it to our page queues\n slice = mi_segment_span_free_coalesce(slice, tld); // set slice again due to coalesceing\n }\n mi_assert_internal(slice->slice_count>0 && slice->slice_offset==0);\n slice = slice + slice->slice_count;\n }\n\n mi_assert(segment->abandoned == 0);\n mi_assert_expensive(mi_segment_is_valid(segment, tld));\n if (segment->used == 0) { // due to page_clear\n mi_assert_internal(right_page_reclaimed == NULL || !(*right_page_reclaimed));\n mi_segment_free(segment, false, tld);\n return NULL;\n }\n else {\n return segment;\n }\n}\n\n\n// attempt to reclaim a particular segment (called from multi threaded free `alloc.c:mi_free_block_mt`)\nbool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment) {\n if (mi_atomic_load_relaxed(&segment->thread_id) != 0) return false; // it is not abandoned\n if (segment->subproc != heap->tld->segments.subproc) return false; // only reclaim within the same subprocess\n if (!_mi_heap_memid_is_suitable(heap,segment->memid)) return false; // don't reclaim between exclusive and non-exclusive arena's\n const long target = _mi_option_get_fast(mi_option_target_segments_per_thread);\n if (target > 0 && (size_t)target <= heap->tld->segments.count) return false; // don't reclaim if going above the target count\n\n // don't reclaim more from a `free` call than half the current segments\n // this is to prevent a pure free-ing thread to start owning too many segments\n // (but not for out-of-arena segments as that is the main way to be reclaimed for those)\n if (segment->memid.memkind == MI_MEM_ARENA && heap->tld->segments.reclaim_count * 2 > heap->tld->segments.count) {\n return false;\n }\n if (_mi_arena_segment_clear_abandoned(segment)) { // atomically unabandon\n mi_segment_t* res = mi_segment_reclaim(segment, heap, 0, NULL, &heap->tld->segments);\n mi_assert_internal(res == segment);\n return (res != NULL);\n }\n return false;\n}\n\nvoid _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) {\n mi_segment_t* segment;\n mi_arena_field_cursor_t current;\n _mi_arena_field_cursor_init(heap, tld->subproc, true /* visit all, blocking */, ¤t);\n while ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL) {\n mi_segment_reclaim(segment, heap, 0, NULL, tld);\n }\n _mi_arena_field_cursor_done(¤t);\n}\n\n\nstatic bool segment_count_is_within_target(mi_segments_tld_t* tld, size_t* ptarget) {\n const size_t target = (size_t)mi_option_get_clamp(mi_option_target_segments_per_thread, 0, 1024);\n if (ptarget != NULL) { *ptarget = target; }\n return (target == 0 || tld->count < target);\n}\n\nstatic long mi_segment_get_reclaim_tries(mi_segments_tld_t* tld) {\n // limit the tries to 10% (default) of the abandoned segments with at least 8 and at most 1024 tries.\n const size_t perc = (size_t)mi_option_get_clamp(mi_option_max_segment_reclaim, 0, 100);\n if (perc <= 0) return 0;\n const size_t total_count = mi_atomic_load_relaxed(&tld->subproc->abandoned_count);\n if (total_count == 0) return 0;\n const size_t relative_count = (total_count > 10000 ? (total_count / 100) * perc : (total_count * perc) / 100); // avoid overflow\n long max_tries = (long)(relative_count <= 1 ? 1 : (relative_count > 1024 ? 1024 : relative_count));\n if (max_tries < 8 && total_count > 8) { max_tries = 8; }\n return max_tries;\n}\n\nstatic mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slices, size_t block_size, bool* reclaimed, mi_segments_tld_t* tld)\n{\n *reclaimed = false;\n long max_tries = mi_segment_get_reclaim_tries(tld);\n if (max_tries <= 0) return NULL;\n\n mi_segment_t* result = NULL;\n mi_segment_t* segment = NULL;\n mi_arena_field_cursor_t current;\n _mi_arena_field_cursor_init(heap, tld->subproc, false /* non-blocking */, ¤t);\n while (segment_count_is_within_target(tld,NULL) && (max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL))\n {\n mi_assert(segment->subproc == heap->tld->segments.subproc); // cursor only visits segments in our sub-process\n segment->abandoned_visits++;\n // todo: should we respect numa affinity for abandoned reclaim? perhaps only for the first visit?\n // todo: an arena exclusive heap will potentially visit many abandoned unsuitable segments and use many tries\n // Perhaps we can skip non-suitable ones in a better way?\n bool is_suitable = _mi_heap_memid_is_suitable(heap, segment->memid);\n bool has_page = mi_segment_check_free(segment,needed_slices,block_size,tld); // try to free up pages (due to concurrent frees)\n if (segment->used == 0) {\n // free the segment (by forced reclaim) to make it available to other threads.\n // note1: we prefer to free a segment as that might lead to reclaiming another\n // segment that is still partially used.\n // note2: we could in principle optimize this by skipping reclaim and directly\n // freeing but that would violate some invariants temporarily)\n mi_segment_reclaim(segment, heap, 0, NULL, tld);\n }\n else if (has_page && is_suitable) {\n // found a large enough free span, or a page of the right block_size with free space\n // we return the result of reclaim (which is usually `segment`) as it might free\n // the segment due to concurrent frees (in which case `NULL` is returned).\n result = mi_segment_reclaim(segment, heap, block_size, reclaimed, tld);\n break;\n }\n else if (segment->abandoned_visits > 3 && is_suitable) {\n // always reclaim on 3rd visit to limit the abandoned segment count.\n mi_segment_reclaim(segment, heap, 0, NULL, tld);\n }\n else {\n // otherwise, push on the visited list so it gets not looked at too quickly again\n max_tries++; // don't count this as a try since it was not suitable\n mi_segment_try_purge(segment, false /* true force? */); // force purge if needed as we may not visit soon again\n _mi_arena_segment_mark_abandoned(segment);\n }\n }\n _mi_arena_field_cursor_done(¤t);\n return result;\n}\n\n// collect abandoned segments\nvoid _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld)\n{\n mi_segment_t* segment;\n mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap, tld->subproc, force /* blocking? */, ¤t);\n long max_tries = (force ? (long)mi_atomic_load_relaxed(&tld->subproc->abandoned_count) : 1024); // limit latency\n while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL)) {\n mi_segment_check_free(segment,0,0,tld); // try to free up pages (due to concurrent frees)\n if (segment->used == 0) {\n // free the segment (by forced reclaim) to make it available to other threads.\n // note: we could in principle optimize this by skipping reclaim and directly\n // freeing but that would violate some invariants temporarily)\n mi_segment_reclaim(segment, heap, 0, NULL, tld);\n }\n else {\n // otherwise, purge if needed and push on the visited list\n // note: forced purge can be expensive if many threads are destroyed/created as in mstress.\n mi_segment_try_purge(segment, force);\n _mi_arena_segment_mark_abandoned(segment);\n }\n }\n _mi_arena_field_cursor_done(¤t);\n}\n\n/* -----------------------------------------------------------\n Force abandon a segment that is in use by our thread\n----------------------------------------------------------- */\n\n// force abandon a segment\nstatic void mi_segment_force_abandon(mi_segment_t* segment, mi_segments_tld_t* tld)\n{\n mi_assert_internal(!mi_segment_is_abandoned(segment));\n mi_assert_internal(!segment->dont_free);\n\n // ensure the segment does not get free'd underneath us (so we can check if a page has been freed in `mi_page_force_abandon`)\n segment->dont_free = true;\n\n // for all slices\n const mi_slice_t* end;\n mi_slice_t* slice = mi_slices_start_iterate(segment, &end);\n while (slice < end) {\n mi_assert_internal(slice->slice_count > 0);\n mi_assert_internal(slice->slice_offset == 0);\n if (mi_slice_is_used(slice)) {\n // ensure used count is up to date and collect potential concurrent frees\n mi_page_t* const page = mi_slice_to_page(slice);\n _mi_page_free_collect(page, false);\n {\n // abandon the page if it is still in-use (this will free it if possible as well)\n mi_assert_internal(segment->used > 0);\n if (segment->used == segment->abandoned+1) {\n // the last page.. abandon and return as the segment will be abandoned after this\n // and we should no longer access it.\n segment->dont_free = false;\n _mi_page_force_abandon(page);\n return;\n }\n else {\n // abandon and continue\n _mi_page_force_abandon(page);\n // it might be freed, reset the slice (note: relies on coalesce setting the slice_offset)\n slice = mi_slice_first(slice);\n }\n }\n }\n slice = slice + slice->slice_count;\n }\n segment->dont_free = false;\n mi_assert(segment->used == segment->abandoned);\n mi_assert(segment->used == 0);\n if (segment->used == 0) { // paranoia\n // all free now\n mi_segment_free(segment, false, tld);\n }\n else {\n // perform delayed purges\n mi_segment_try_purge(segment, false /* force? */);\n }\n}\n\n\n// try abandon segments.\n// this should be called from `reclaim_or_alloc` so we know all segments are (about) fully in use.\nstatic void mi_segments_try_abandon_to_target(mi_heap_t* heap, size_t target, mi_segments_tld_t* tld) {\n if (target <= 1) return;\n const size_t min_target = (target > 4 ? (target*3)/4 : target); // 75%\n // todo: we should maintain a list of segments per thread; for now, only consider segments from the heap full pages\n for (int i = 0; i < 64 && tld->count >= min_target; i++) {\n mi_page_t* page = heap->pages[MI_BIN_FULL].first;\n while (page != NULL && mi_page_block_size(page) > MI_LARGE_OBJ_SIZE_MAX) {\n page = page->next;\n }\n if (page==NULL) {\n break;\n }\n mi_segment_t* segment = _mi_page_segment(page);\n mi_segment_force_abandon(segment, tld);\n mi_assert_internal(page != heap->pages[MI_BIN_FULL].first); // as it is just abandoned\n }\n}\n\n// try abandon segments.\n// this should be called from `reclaim_or_alloc` so we know all segments are (about) fully in use.\nstatic void mi_segments_try_abandon(mi_heap_t* heap, mi_segments_tld_t* tld) {\n // we call this when we are about to add a fresh segment so we should be under our target segment count.\n size_t target = 0;\n if (segment_count_is_within_target(tld, &target)) return;\n mi_segments_try_abandon_to_target(heap, target, tld);\n}\n\nvoid mi_collect_reduce(size_t target_size) mi_attr_noexcept {\n mi_collect(true);\n mi_heap_t* heap = mi_heap_get_default();\n mi_segments_tld_t* tld = &heap->tld->segments;\n size_t target = target_size / MI_SEGMENT_SIZE;\n if (target == 0) {\n target = (size_t)mi_option_get_clamp(mi_option_target_segments_per_thread, 1, 1024);\n }\n mi_segments_try_abandon_to_target(heap, target, tld);\n}\n\n/* -----------------------------------------------------------\n Reclaim or allocate\n----------------------------------------------------------- */\n\nstatic mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_slices, size_t block_size, mi_segments_tld_t* tld)\n{\n mi_assert_internal(block_size <= MI_LARGE_OBJ_SIZE_MAX);\n\n // try to abandon some segments to increase reuse between threads\n mi_segments_try_abandon(heap,tld);\n\n // 1. try to reclaim an abandoned segment\n bool reclaimed;\n mi_segment_t* segment = mi_segment_try_reclaim(heap, needed_slices, block_size, &reclaimed, tld);\n if (reclaimed) {\n // reclaimed the right page right into the heap\n mi_assert_internal(segment != NULL);\n return NULL; // pretend out-of-memory as the page will be in the page queue of the heap with available blocks\n }\n else if (segment != NULL) {\n // reclaimed a segment with a large enough empty span in it\n return segment;\n }\n // 2. otherwise allocate a fresh segment\n return mi_segment_alloc(0, 0, heap->arena_id, tld, NULL);\n}\n\n\n/* -----------------------------------------------------------\n Page allocation\n----------------------------------------------------------- */\n\nstatic mi_page_t* mi_segments_page_alloc(mi_heap_t* heap, mi_page_kind_t page_kind, size_t required, size_t block_size, mi_segments_tld_t* tld)\n{\n mi_assert_internal(required <= MI_LARGE_OBJ_SIZE_MAX && page_kind <= MI_PAGE_LARGE);\n\n // find a free page\n size_t page_size = _mi_align_up(required, (required > MI_MEDIUM_PAGE_SIZE ? MI_MEDIUM_PAGE_SIZE : MI_SEGMENT_SLICE_SIZE));\n size_t slices_needed = page_size / MI_SEGMENT_SLICE_SIZE;\n mi_assert_internal(slices_needed * MI_SEGMENT_SLICE_SIZE == page_size);\n mi_page_t* page = mi_segments_page_find_and_allocate(slices_needed, heap->arena_id, tld); //(required <= MI_SMALL_SIZE_MAX ? 0 : slices_needed), tld);\n if (page==NULL) {\n // no free page, allocate a new segment and try again\n if (mi_segment_reclaim_or_alloc(heap, slices_needed, block_size, tld) == NULL) {\n // OOM or reclaimed a good page in the heap\n return NULL;\n }\n else {\n // otherwise try again\n return mi_segments_page_alloc(heap, page_kind, required, block_size, tld);\n }\n }\n mi_assert_internal(page != NULL && page->slice_count*MI_SEGMENT_SLICE_SIZE == page_size);\n mi_assert_internal(_mi_ptr_segment(page)->thread_id == _mi_thread_id());\n mi_segment_try_purge(_mi_ptr_segment(page), false);\n return page;\n}\n\n\n\n/* -----------------------------------------------------------\n Huge page allocation\n----------------------------------------------------------- */\n\nstatic mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld)\n{\n mi_page_t* page = NULL;\n mi_segment_t* segment = mi_segment_alloc(size,page_alignment,req_arena_id,tld,&page);\n if (segment == NULL || page==NULL) return NULL;\n mi_assert_internal(segment->used==1);\n mi_assert_internal(mi_page_block_size(page) >= size);\n #if MI_HUGE_PAGE_ABANDON\n segment->thread_id = 0; // huge segments are immediately abandoned\n #endif\n\n // for huge pages we initialize the block_size as we may\n // overallocate to accommodate large alignments.\n size_t psize;\n uint8_t* start = _mi_segment_page_start(segment, page, &psize);\n page->block_size = psize;\n mi_assert_internal(page->is_huge);\n\n // decommit the part of the prefix of a page that will not be used; this can be quite large (close to MI_SEGMENT_SIZE)\n if (page_alignment > 0 && segment->allow_decommit) {\n uint8_t* aligned_p = (uint8_t*)_mi_align_up((uintptr_t)start, page_alignment);\n mi_assert_internal(_mi_is_aligned(aligned_p, page_alignment));\n mi_assert_internal(psize - (aligned_p - start) >= size);\n uint8_t* decommit_start = start + sizeof(mi_block_t); // for the free list\n ptrdiff_t decommit_size = aligned_p - decommit_start;\n _mi_os_reset(decommit_start, decommit_size); // note: cannot use segment_decommit on huge segments\n }\n\n return page;\n}\n\n#if MI_HUGE_PAGE_ABANDON\n// free huge block from another thread\nvoid _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block) {\n // huge page segments are always abandoned and can be freed immediately by any thread\n mi_assert_internal(segment->kind==MI_SEGMENT_HUGE);\n mi_assert_internal(segment == _mi_page_segment(page));\n mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id)==0);\n\n // claim it and free\n mi_heap_t* heap = mi_heap_get_default(); // issue #221; don't use the internal get_default_heap as we need to ensure the thread is initialized.\n // paranoia: if this it the last reference, the cas should always succeed\n size_t expected_tid = 0;\n if (mi_atomic_cas_strong_acq_rel(&segment->thread_id, &expected_tid, heap->thread_id)) {\n mi_block_set_next(page, block, page->free);\n page->free = block;\n page->used--;\n page->is_zero_init = false;\n mi_assert(page->used == 0);\n mi_tld_t* tld = heap->tld;\n _mi_segment_page_free(page, true, &tld->segments);\n }\n#if (MI_DEBUG!=0)\n else {\n mi_assert_internal(false);\n }\n#endif\n}\n\n#else\n// reset memory of a huge block from another thread\nvoid _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block) {\n MI_UNUSED(page);\n mi_assert_internal(segment->kind == MI_SEGMENT_HUGE);\n mi_assert_internal(segment == _mi_page_segment(page));\n mi_assert_internal(page->used == 1); // this is called just before the free\n mi_assert_internal(page->free == NULL);\n if (segment->allow_decommit) {\n size_t csize = mi_usable_size(block);\n if (csize > sizeof(mi_block_t)) {\n csize = csize - sizeof(mi_block_t);\n uint8_t* p = (uint8_t*)block + sizeof(mi_block_t);\n _mi_os_reset(p, csize); // note: cannot use segment_decommit on huge segments\n }\n }\n}\n#endif\n\n/* -----------------------------------------------------------\n Page allocation and free\n----------------------------------------------------------- */\nmi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld) {\n mi_page_t* page;\n if mi_unlikely(page_alignment > MI_BLOCK_ALIGNMENT_MAX) {\n mi_assert_internal(_mi_is_power_of_two(page_alignment));\n mi_assert_internal(page_alignment >= MI_SEGMENT_SIZE);\n if (page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; }\n page = mi_segment_huge_page_alloc(block_size,page_alignment,heap->arena_id,tld);\n }\n else if (block_size <= MI_SMALL_OBJ_SIZE_MAX) {\n page = mi_segments_page_alloc(heap,MI_PAGE_SMALL,block_size,block_size,tld);\n }\n else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX) {\n page = mi_segments_page_alloc(heap,MI_PAGE_MEDIUM,MI_MEDIUM_PAGE_SIZE,block_size,tld);\n }\n else if (block_size <= MI_LARGE_OBJ_SIZE_MAX) {\n page = mi_segments_page_alloc(heap,MI_PAGE_LARGE,block_size,block_size,tld);\n }\n else {\n page = mi_segment_huge_page_alloc(block_size,page_alignment,heap->arena_id,tld);\n }\n mi_assert_internal(page == NULL || _mi_heap_memid_is_suitable(heap, _mi_page_segment(page)->memid));\n mi_assert_expensive(page == NULL || mi_segment_is_valid(_mi_page_segment(page),tld));\n mi_assert_internal(page == NULL || _mi_page_segment(page)->subproc == tld->subproc);\n return page;\n}\n\n\n/* -----------------------------------------------------------\n Visit blocks in a segment (only used for abandoned segments)\n----------------------------------------------------------- */\n\nstatic bool mi_segment_visit_page(mi_page_t* page, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {\n mi_heap_area_t area;\n _mi_heap_area_init(&area, page);\n if (!visitor(NULL, &area, NULL, area.block_size, arg)) return false;\n if (visit_blocks) {\n _mi_page_free_collect(page,true); // collect so the used count is accurate\n return _mi_heap_area_visit_blocks(&area, page, visitor, arg);\n }\n else {\n return true;\n }\n}\n\nbool _mi_segment_visit_blocks(mi_segment_t* segment, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {\n const mi_slice_t* end;\n mi_slice_t* slice = mi_slices_start_iterate(segment, &end);\n while (slice < end) {\n if (mi_slice_is_used(slice)) {\n mi_page_t* const page = mi_slice_to_page(slice);\n if (heap_tag < 0 || (int)page->heap_tag == heap_tag) {\n if (!mi_segment_visit_page(page, visit_blocks, visitor, arg)) return false;\n }\n }\n slice = slice + slice->slice_count;\n }\n return true;\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/static.c", "sha256": "81e27487e494d2b32cb16a2605d61ee64fe3a71c409d5be3a673950ff769bd73", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2020, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#ifndef _DEFAULT_SOURCE\n#define _DEFAULT_SOURCE\n#endif\n#if defined(__sun)\n// same remarks as os.c for the static's context.\n#undef _XOPEN_SOURCE\n#undef _POSIX_C_SOURCE\n#endif\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n\n// For a static override we create a single object file\n// containing the whole library. If it is linked first\n// it will override all the standard library allocation\n// functions (on Unix's).\n#include \"alloc.c\" // includes alloc-override.c\n#include \"alloc-aligned.c\"\n#include \"alloc-posix.c\"\n#include \"arena.c\"\n#include \"bitmap.c\"\n#include \"heap.c\"\n#include \"init.c\"\n#include \"libc.c\"\n#include \"options.c\"\n#include \"os.c\"\n#include \"page.c\" // includes page-queue.c\n#include \"random.c\"\n#include \"segment.c\"\n#include \"segment-map.c\"\n#include \"stats.c\"\n#include \"prim/prim.c\"\n#if MI_OSX_ZONE\n#include \"prim/osx/alloc-override-zone.c\"\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v2/src/stats.c", "sha256": "7767e241bb0fe702d37083f213b1a194457d4e9ef7fef77f3324a6067b440239", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2021, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\"\n\n#include // memset\n\n#if defined(_MSC_VER) && (_MSC_VER < 1920)\n#pragma warning(disable:4204) // non-constant aggregate initializer\n#endif\n\n/* -----------------------------------------------------------\n Statistics operations\n----------------------------------------------------------- */\n\nstatic bool mi_is_in_main(void* stat) {\n return ((uint8_t*)stat >= (uint8_t*)&_mi_stats_main\n && (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t)));\n}\n\nstatic void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {\n if (amount == 0) return;\n if mi_unlikely(mi_is_in_main(stat))\n {\n // add atomically (for abandoned pages)\n int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount);\n // if (stat == &_mi_stats_main.committed) { mi_assert_internal(current + amount >= 0); };\n mi_atomic_maxi64_relaxed(&stat->peak, current + amount);\n if (amount > 0) {\n mi_atomic_addi64_relaxed(&stat->total,amount);\n }\n }\n else {\n // add thread local\n stat->current += amount;\n if (stat->current > stat->peak) { stat->peak = stat->current; }\n if (amount > 0) { stat->total += amount; }\n }\n}\n\nvoid _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {\n if (mi_is_in_main(stat)) {\n mi_atomic_addi64_relaxed( &stat->total, (int64_t)amount );\n }\n else {\n stat->total += amount;\n }\n}\n\nvoid _mi_stat_increase(mi_stat_count_t* stat, size_t amount) {\n mi_stat_update(stat, (int64_t)amount);\n}\n\nvoid _mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {\n mi_stat_update(stat, -((int64_t)amount));\n}\n\n\nstatic void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) {\n if (amount == 0) return;\n if mi_unlikely(mi_is_in_main(stat))\n {\n // adjust atomically\n mi_atomic_addi64_relaxed(&stat->current, amount);\n mi_atomic_addi64_relaxed(&stat->total,amount);\n }\n else {\n // adjust local\n stat->current += amount;\n stat->total += amount;\n }\n}\n\nvoid _mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) {\n mi_stat_adjust(stat, -((int64_t)amount));\n}\n\n\n// must be thread safe as it is called from stats_merge\nstatic void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) {\n if (stat==src) return;\n mi_atomic_void_addi64_relaxed(&stat->total, &src->total);\n const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src->current);\n\n // Global current plus thread peak approximates new global peak\n // note: peak scores do really not work across threads.\n // we used to just add them together but that often overestimates in practice.\n // similarly, max does not seem to work well. The current approach\n // by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112\n // for a longer description.\n mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src->peak);\n}\n\nstatic void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) {\n if (stat==src) return;\n mi_atomic_void_addi64_relaxed(&stat->total, &src->total);\n}\n\n#define MI_STAT_COUNT(stat) mi_stat_count_add_mt(&stats->stat, &src->stat);\n#define MI_STAT_COUNTER(stat) mi_stat_counter_add_mt(&stats->stat, &src->stat);\n\n// must be thread safe as it is called from stats_merge\nstatic void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {\n if (stats==src) return;\n\n // copy all fields\n MI_STAT_FIELDS()\n\n #if MI_STAT>1\n for (size_t i = 0; i <= MI_BIN_HUGE; i++) {\n mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]);\n }\n #endif\n for (size_t i = 0; i <= MI_BIN_HUGE; i++) {\n mi_stat_count_add_mt(&stats->page_bins[i], &src->page_bins[i]);\n }\n}\n\n#undef MI_STAT_COUNT\n#undef MI_STAT_COUNTER\n\n/* -----------------------------------------------------------\n Display statistics\n----------------------------------------------------------- */\n\n// unit > 0 : size in binary bytes\n// unit == 0: count as decimal\n// unit < 0 : count in binary\nstatic void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) {\n char buf[32]; buf[0] = 0;\n int len = 32;\n const char* suffix = (unit <= 0 ? \" \" : \"B\");\n const int64_t base = (unit == 0 ? 1000 : 1024);\n if (unit>0) n *= unit;\n\n const int64_t pos = (n < 0 ? -n : n);\n if (pos < base) {\n if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column\n _mi_snprintf(buf, len, \"%lld %-3s\", (long long)n, (n==0 ? \"\" : suffix));\n }\n }\n else {\n int64_t divider = base;\n const char* magnitude = \"K\";\n if (pos >= divider*base) { divider *= base; magnitude = \"M\"; }\n if (pos >= divider*base) { divider *= base; magnitude = \"G\"; }\n const int64_t tens = (n / (divider/10));\n const long whole = (long)(tens/10);\n const long frac1 = (long)(tens%10);\n char unitdesc[8];\n _mi_snprintf(unitdesc, 8, \"%s%s%s\", magnitude, (base==1024 ? \"i\" : \"\"), suffix);\n _mi_snprintf(buf, len, \"%ld.%ld %-3s\", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc);\n }\n _mi_fprintf(out, arg, (fmt==NULL ? \"%12s\" : fmt), buf);\n}\n\n\nstatic void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {\n mi_printf_amount(n,unit,out,arg,NULL);\n}\n\nstatic void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {\n if (unit==1) _mi_fprintf(out, arg, \"%12s\",\" \");\n else mi_print_amount(n,0,out,arg);\n}\n\nstatic void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) {\n _mi_fprintf(out, arg,\"%10s:\", msg);\n if (unit != 0) {\n if (unit > 0) {\n mi_print_amount(stat->peak, unit, out, arg);\n mi_print_amount(stat->total, unit, out, arg);\n // mi_print_amount(stat->freed, unit, out, arg);\n mi_print_amount(stat->current, unit, out, arg);\n mi_print_amount(unit, 1, out, arg);\n mi_print_count(stat->total, unit, out, arg);\n }\n else {\n mi_print_amount(stat->peak, -1, out, arg);\n mi_print_amount(stat->total, -1, out, arg);\n // mi_print_amount(stat->freed, -1, out, arg);\n mi_print_amount(stat->current, -1, out, arg);\n if (unit == -1) {\n _mi_fprintf(out, arg, \"%24s\", \"\");\n }\n else {\n mi_print_amount(-unit, 1, out, arg);\n mi_print_count((stat->total / -unit), 0, out, arg);\n }\n }\n if (stat->current != 0) {\n _mi_fprintf(out, arg, \" \");\n _mi_fprintf(out, arg, (notok == NULL ? \"not all freed\" : notok));\n _mi_fprintf(out, arg, \"\\n\");\n }\n else {\n _mi_fprintf(out, arg, \" ok\\n\");\n }\n }\n else {\n mi_print_amount(stat->peak, 0, out, arg);\n mi_print_amount(stat->total, 0, out, arg);\n mi_print_amount(stat->current, 0, out, arg);\n _mi_fprintf(out, arg, \"\\n\");\n }\n}\n\nstatic void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {\n mi_stat_print_ex(stat, msg, unit, out, arg, NULL);\n}\n\n#if MI_STAT>1\nstatic void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {\n _mi_fprintf(out, arg, \"%10s:\", msg);\n _mi_fprintf(out, arg, \"%12s\", \" \"); // no peak\n mi_print_amount(stat->total, unit, out, arg);\n _mi_fprintf(out, arg, \"\\n\");\n}\n#endif\n\nstatic void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {\n _mi_fprintf(out, arg, \"%10s:\", msg);\n mi_print_amount(stat->total, 0, out, arg);\n _mi_fprintf(out, arg, \"\\n\");\n}\n\nstatic void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {\n _mi_fprintf(out, arg, \"%10s:\", msg);\n mi_print_amount(stat->total, 1, out, arg);\n _mi_fprintf(out, arg, \"\\n\");\n}\n\nstatic void mi_stat_average_print(int64_t count, int64_t total, const char* msg, mi_output_fun* out, void* arg) {\n const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count));\n const int64_t avg_whole = avg_tens/10;\n const int64_t avg_frac1 = avg_tens%10;\n _mi_fprintf(out, arg, \"%10s: %5lld.%lld avg\\n\", msg, avg_whole, avg_frac1);\n}\n\n\nstatic void mi_print_header(mi_output_fun* out, void* arg ) {\n _mi_fprintf(out, arg, \"%10s: %11s %11s %11s %11s %11s\\n\", \"heap stats\", \"peak \", \"total \", \"current \", \"block \", \"total# \");\n}\n\n#if MI_STAT>1\nstatic void mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out, void* arg) {\n bool found = false;\n char buf[64];\n for (size_t i = 0; i <= max; i++) {\n if (bins[i].total > 0) {\n found = true;\n int64_t unit = _mi_bin_size((uint8_t)i);\n _mi_snprintf(buf, 64, \"%s %3lu\", fmt, (long)i);\n mi_stat_print(&bins[i], buf, unit, out, arg);\n }\n }\n if (found) {\n _mi_fprintf(out, arg, \"\\n\");\n mi_print_header(out, arg);\n }\n}\n#endif\n\n\n\n//------------------------------------------------------------\n// Use an output wrapper for line-buffered output\n// (which is nice when using loggers etc.)\n//------------------------------------------------------------\ntypedef struct buffered_s {\n mi_output_fun* out; // original output function\n void* arg; // and state\n char* buf; // local buffer of at least size `count+1`\n size_t used; // currently used chars `used <= count`\n size_t count; // total chars available for output\n} buffered_t;\n\nstatic void mi_buffered_flush(buffered_t* buf) {\n buf->buf[buf->used] = 0;\n _mi_fputs(buf->out, buf->arg, NULL, buf->buf);\n buf->used = 0;\n}\n\nstatic void mi_cdecl mi_buffered_out(const char* msg, void* arg) {\n buffered_t* buf = (buffered_t*)arg;\n if (msg==NULL || buf==NULL) return;\n for (const char* src = msg; *src != 0; src++) {\n char c = *src;\n if (buf->used >= buf->count) mi_buffered_flush(buf);\n mi_assert_internal(buf->used < buf->count);\n buf->buf[buf->used++] = c;\n if (c == '\\n') mi_buffered_flush(buf);\n }\n}\n\n//------------------------------------------------------------\n// Print statistics\n//------------------------------------------------------------\n\nstatic void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept {\n // wrap the output function to be line buffered\n char buf[256];\n buffered_t buffer = { out0, arg0, NULL, 0, 255 };\n buffer.buf = buf;\n mi_output_fun* out = &mi_buffered_out;\n void* arg = &buffer;\n\n // and print using that\n mi_print_header(out,arg);\n #if MI_STAT>1\n mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, \"bin\",out,arg);\n #endif\n #if MI_STAT\n mi_stat_print(&stats->malloc_normal, \"binned\", (stats->malloc_normal_count.total == 0 ? -1 : 1), out, arg);\n mi_stat_print(&stats->malloc_huge, \"huge\", (stats->malloc_huge_count.total == 0 ? -1 : 1), out, arg);\n mi_stat_count_t total = { 0,0,0 };\n mi_stat_count_add_mt(&total, &stats->malloc_normal);\n // mi_stat_count_add(&total, &stats->malloc_large);\n mi_stat_count_add_mt(&total, &stats->malloc_huge);\n mi_stat_print_ex(&total, \"total\", 1, out, arg, \"\");\n #endif\n #if MI_STAT>1\n mi_stat_total_print(&stats->malloc_requested, \"malloc req\", 1, out, arg);\n _mi_fprintf(out, arg, \"\\n\");\n #endif\n mi_stat_print_ex(&stats->reserved, \"reserved\", 1, out, arg, \"\");\n mi_stat_print_ex(&stats->committed, \"committed\", 1, out, arg, \"\");\n mi_stat_counter_print_size(&stats->reset, \"reset\", out, arg );\n mi_stat_counter_print_size(&stats->purged, \"purged\", out, arg );\n mi_stat_print_ex(&stats->page_committed, \"touched\", 1, out, arg, \"\");\n mi_stat_print(&stats->segments, \"segments\", 0, out, arg);\n mi_stat_print(&stats->segments_abandoned, \"-abandoned\", 0, out, arg);\n mi_stat_print(&stats->segments_cache, \"-cached\", 0, out, arg);\n mi_stat_print(&stats->pages, \"pages\", 0, out, arg);\n mi_stat_print(&stats->pages_abandoned, \"-abandoned\", 0, out, arg);\n mi_stat_counter_print(&stats->pages_extended, \"-extended\", out, arg);\n mi_stat_counter_print(&stats->pages_retire, \"-retire\", out, arg);\n mi_stat_counter_print(&stats->arena_count, \"arenas\", out, arg);\n // mi_stat_counter_print(&stats->arena_crossover_count, \"-crossover\", out, arg);\n mi_stat_counter_print(&stats->arena_rollback_count, \"-rollback\", out, arg);\n mi_stat_counter_print(&stats->mmap_calls, \"mmaps\", out, arg);\n mi_stat_counter_print(&stats->commit_calls, \"commits\", out, arg);\n mi_stat_counter_print(&stats->reset_calls, \"resets\", out, arg);\n mi_stat_counter_print(&stats->purge_calls, \"purges\", out, arg);\n mi_stat_counter_print(&stats->malloc_guarded_count, \"guarded\", out, arg);\n mi_stat_print(&stats->threads, \"threads\", 0, out, arg);\n mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, \"searches\", out, arg);\n _mi_fprintf(out, arg, \"%10s: %5i\\n\", \"numa nodes\", _mi_os_numa_node_count());\n\n size_t elapsed;\n size_t user_time;\n size_t sys_time;\n size_t current_rss;\n size_t peak_rss;\n size_t current_commit;\n size_t peak_commit;\n size_t page_faults;\n mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults);\n _mi_fprintf(out, arg, \"%10s: %5zu.%03zu s\\n\", \"elapsed\", elapsed/1000, elapsed%1000);\n _mi_fprintf(out, arg, \"%10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: \", \"process\",\n user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults );\n mi_printf_amount((int64_t)peak_rss, 1, out, arg, \"%s\");\n if (peak_commit > 0) {\n _mi_fprintf(out, arg, \", peak commit: \");\n mi_printf_amount((int64_t)peak_commit, 1, out, arg, \"%s\");\n }\n _mi_fprintf(out, arg, \"\\n\");\n}\n\nstatic mi_msecs_t mi_process_start; // = 0\n\nstatic mi_stats_t* mi_stats_get_default(void) {\n mi_heap_t* heap = mi_heap_get_default();\n return &heap->tld->stats;\n}\n\nstatic void mi_stats_merge_from(mi_stats_t* stats) {\n if (stats != &_mi_stats_main) {\n mi_stats_add(&_mi_stats_main, stats);\n memset(stats, 0, sizeof(mi_stats_t));\n }\n}\n\nvoid mi_stats_reset(void) mi_attr_noexcept {\n mi_stats_t* stats = mi_stats_get_default();\n if (stats != &_mi_stats_main) { memset(stats, 0, sizeof(mi_stats_t)); }\n memset(&_mi_stats_main, 0, sizeof(mi_stats_t));\n if (mi_process_start == 0) { mi_process_start = _mi_clock_start(); };\n}\n\nvoid mi_stats_merge(void) mi_attr_noexcept {\n mi_stats_merge_from( mi_stats_get_default() );\n}\n\nvoid _mi_stats_merge_thread(mi_tld_t* tld) {\n mi_stats_merge_from( &tld->stats );\n}\n\nvoid _mi_stats_done(mi_stats_t* stats) { // called from `mi_thread_done`\n mi_stats_merge_from(stats);\n}\n\nvoid mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {\n mi_stats_merge_from(mi_stats_get_default());\n _mi_stats_print(&_mi_stats_main, out, arg);\n}\n\nvoid mi_stats_print(void* out) mi_attr_noexcept {\n // for compatibility there is an `out` parameter (which can be `stdout` or `stderr`)\n mi_stats_print_out((mi_output_fun*)out, NULL);\n}\n\nvoid mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {\n _mi_stats_print(mi_stats_get_default(), out, arg);\n}\n\n\n// ----------------------------------------------------------------\n// Basic timer for convenience; use milli-seconds to avoid doubles\n// ----------------------------------------------------------------\n\nstatic mi_msecs_t mi_clock_diff;\n\nmi_msecs_t _mi_clock_now(void) {\n return _mi_prim_clock_now();\n}\n\nmi_msecs_t _mi_clock_start(void) {\n if (mi_clock_diff == 0.0) {\n mi_msecs_t t0 = _mi_clock_now();\n mi_clock_diff = _mi_clock_now() - t0;\n }\n return _mi_clock_now();\n}\n\nmi_msecs_t _mi_clock_end(mi_msecs_t start) {\n mi_msecs_t end = _mi_clock_now();\n return (end - start - mi_clock_diff);\n}\n\n\n// --------------------------------------------------------\n// Basic process statistics\n// --------------------------------------------------------\n\nmi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept\n{\n mi_process_info_t pinfo;\n _mi_memzero_var(pinfo);\n pinfo.elapsed = _mi_clock_end(mi_process_start);\n pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current));\n pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak));\n pinfo.current_rss = pinfo.current_commit;\n pinfo.peak_rss = pinfo.peak_commit;\n pinfo.utime = 0;\n pinfo.stime = 0;\n pinfo.page_faults = 0;\n\n _mi_prim_process_info(&pinfo);\n\n if (elapsed_msecs!=NULL) *elapsed_msecs = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX));\n if (user_msecs!=NULL) *user_msecs = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX));\n if (system_msecs!=NULL) *system_msecs = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX));\n if (current_rss!=NULL) *current_rss = pinfo.current_rss;\n if (peak_rss!=NULL) *peak_rss = pinfo.peak_rss;\n if (current_commit!=NULL) *current_commit = pinfo.current_commit;\n if (peak_commit!=NULL) *peak_commit = pinfo.peak_commit;\n if (page_faults!=NULL) *page_faults = pinfo.page_faults;\n}\n\n\n// --------------------------------------------------------\n// Return statistics\n// --------------------------------------------------------\n\nbool mi_stats_get(mi_stats_t* stats) mi_attr_noexcept {\n if (stats == NULL || stats->size != sizeof(mi_stats_t) || stats->version != MI_STAT_VERSION) return false;\n _mi_memzero(stats,stats->size);\n _mi_memcpy(stats, &_mi_stats_main, sizeof(mi_stats_t));\n return true;\n}\n\n\n// --------------------------------------------------------\n// Statics in json format\n// --------------------------------------------------------\n\ntypedef struct mi_heap_buf_s {\n char* buf;\n size_t size;\n size_t used;\n bool can_realloc;\n} mi_heap_buf_t;\n\nstatic bool mi_heap_buf_expand(mi_heap_buf_t* hbuf) {\n if (hbuf==NULL) return false;\n if (hbuf->buf != NULL && hbuf->size>0) {\n hbuf->buf[hbuf->size-1] = 0;\n }\n if (hbuf->size > SIZE_MAX/2 || !hbuf->can_realloc) return false;\n const size_t newsize = (hbuf->size == 0 ? mi_good_size(12*MI_KiB) : 2*hbuf->size);\n char* const newbuf = (char*)mi_rezalloc(hbuf->buf, newsize);\n if (newbuf == NULL) return false;\n hbuf->buf = newbuf;\n hbuf->size = newsize;\n return true;\n}\n\nstatic void mi_heap_buf_print(mi_heap_buf_t* hbuf, const char* msg) {\n if (msg==NULL || hbuf==NULL) return;\n if (hbuf->used + 1 >= hbuf->size && !hbuf->can_realloc) return;\n for (const char* src = msg; *src != 0; src++) {\n char c = *src;\n if (hbuf->used + 1 >= hbuf->size) {\n if (!mi_heap_buf_expand(hbuf)) return;\n }\n mi_assert_internal(hbuf->used < hbuf->size);\n hbuf->buf[hbuf->used++] = c;\n }\n mi_assert_internal(hbuf->used < hbuf->size);\n hbuf->buf[hbuf->used] = 0;\n}\n\nstatic void mi_heap_buf_print_count_bin(mi_heap_buf_t* hbuf, const char* prefix, mi_stat_count_t* stat, size_t bin, bool add_comma) {\n const size_t binsize = _mi_bin_size(bin);\n const size_t pagesize = (binsize <= MI_SMALL_OBJ_SIZE_MAX ? MI_SMALL_PAGE_SIZE :\n (binsize <= MI_MEDIUM_OBJ_SIZE_MAX ? MI_MEDIUM_PAGE_SIZE :\n #if MI_LARGE_PAGE_SIZE\n (binsize <= MI_LARGE_OBJ_SIZE_MAX ? MI_LARGE_PAGE_SIZE : 0)\n #else\n 0\n #endif\n ));\n char buf[128];\n _mi_snprintf(buf, 128, \"%s{ \\\"total\\\": %lld, \\\"peak\\\": %lld, \\\"current\\\": %lld, \\\"block_size\\\": %zu, \\\"page_size\\\": %zu }%s\\n\", prefix, stat->total, stat->peak, stat->current, binsize, pagesize, (add_comma ? \",\" : \"\"));\n buf[127] = 0;\n mi_heap_buf_print(hbuf, buf);\n}\n\nstatic void mi_heap_buf_print_count(mi_heap_buf_t* hbuf, const char* prefix, mi_stat_count_t* stat, bool add_comma) {\n char buf[128];\n _mi_snprintf(buf, 128, \"%s{ \\\"total\\\": %lld, \\\"peak\\\": %lld, \\\"current\\\": %lld }%s\\n\", prefix, stat->total, stat->peak, stat->current, (add_comma ? \",\" : \"\"));\n buf[127] = 0;\n mi_heap_buf_print(hbuf, buf);\n}\n\nstatic void mi_heap_buf_print_count_value(mi_heap_buf_t* hbuf, const char* name, mi_stat_count_t* stat) {\n char buf[128];\n _mi_snprintf(buf, 128, \" \\\"%s\\\": \", name);\n buf[127] = 0;\n mi_heap_buf_print(hbuf, buf);\n mi_heap_buf_print_count(hbuf, \"\", stat, true);\n}\n\nstatic void mi_heap_buf_print_value(mi_heap_buf_t* hbuf, const char* name, int64_t val) {\n char buf[128];\n _mi_snprintf(buf, 128, \" \\\"%s\\\": %lld,\\n\", name, val);\n buf[127] = 0;\n mi_heap_buf_print(hbuf, buf);\n}\n\nstatic void mi_heap_buf_print_size(mi_heap_buf_t* hbuf, const char* name, size_t val, bool add_comma) {\n char buf[128];\n _mi_snprintf(buf, 128, \" \\\"%s\\\": %zu%s\\n\", name, val, (add_comma ? \",\" : \"\"));\n buf[127] = 0;\n mi_heap_buf_print(hbuf, buf);\n}\n\nstatic void mi_heap_buf_print_counter_value(mi_heap_buf_t* hbuf, const char* name, mi_stat_counter_t* stat) {\n mi_heap_buf_print_value(hbuf, name, stat->total);\n}\n\n#define MI_STAT_COUNT(stat) mi_heap_buf_print_count_value(&hbuf, #stat, &stats->stat);\n#define MI_STAT_COUNTER(stat) mi_heap_buf_print_counter_value(&hbuf, #stat, &stats->stat);\n\nchar* mi_stats_get_json(size_t output_size, char* output_buf) mi_attr_noexcept {\n mi_heap_buf_t hbuf = { NULL, 0, 0, true };\n if (output_size > 0 && output_buf != NULL) {\n _mi_memzero(output_buf, output_size);\n hbuf.buf = output_buf;\n hbuf.size = output_size;\n hbuf.can_realloc = false;\n }\n else {\n if (!mi_heap_buf_expand(&hbuf)) return NULL;\n }\n mi_heap_buf_print(&hbuf, \"{\\n\");\n mi_heap_buf_print_value(&hbuf, \"stat_version\", MI_STAT_VERSION);\n mi_heap_buf_print_value(&hbuf, \"mimalloc_version\", MI_MALLOC_VERSION);\n\n // process info\n mi_heap_buf_print(&hbuf, \" \\\"process\\\": {\\n\");\n size_t elapsed;\n size_t user_time;\n size_t sys_time;\n size_t current_rss;\n size_t peak_rss;\n size_t current_commit;\n size_t peak_commit;\n size_t page_faults;\n mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults);\n mi_heap_buf_print_size(&hbuf, \"elapsed_msecs\", elapsed, true);\n mi_heap_buf_print_size(&hbuf, \"user_msecs\", user_time, true);\n mi_heap_buf_print_size(&hbuf, \"system_msecs\", sys_time, true);\n mi_heap_buf_print_size(&hbuf, \"page_faults\", page_faults, true);\n mi_heap_buf_print_size(&hbuf, \"rss_current\", current_rss, true);\n mi_heap_buf_print_size(&hbuf, \"rss_peak\", peak_rss, true);\n mi_heap_buf_print_size(&hbuf, \"commit_current\", current_commit, true);\n mi_heap_buf_print_size(&hbuf, \"commit_peak\", peak_commit, false);\n mi_heap_buf_print(&hbuf, \" },\\n\");\n\n // statistics\n mi_stats_t* stats = &_mi_stats_main;\n MI_STAT_FIELDS()\n\n // size bins\n mi_heap_buf_print(&hbuf, \" \\\"malloc_bins\\\": [\\n\");\n for (size_t i = 0; i <= MI_BIN_HUGE; i++) {\n mi_heap_buf_print_count_bin(&hbuf, \" \", &stats->malloc_bins[i], i, i!=MI_BIN_HUGE);\n }\n mi_heap_buf_print(&hbuf, \" ],\\n\");\n mi_heap_buf_print(&hbuf, \" \\\"page_bins\\\": [\\n\");\n for (size_t i = 0; i <= MI_BIN_HUGE; i++) {\n mi_heap_buf_print_count_bin(&hbuf, \" \", &stats->page_bins[i], i, i!=MI_BIN_HUGE);\n }\n mi_heap_buf_print(&hbuf, \" ]\\n\");\n mi_heap_buf_print(&hbuf, \"}\\n\");\n if (hbuf.used >= hbuf.size) {\n // failed\n if (hbuf.can_realloc) { mi_free(hbuf.buf); }\n return NULL;\n }\n else {\n return hbuf.buf;\n }\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/include/mimalloc-new-delete.h", "sha256": "dd4f25cae53209d45d73f8e6a2b9c219e8fc7434d97f20eba9af1a8b850030fd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2020 Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_NEW_DELETE_H\n#define MIMALLOC_NEW_DELETE_H\n\n// ----------------------------------------------------------------------------\n// This header provides convenient overrides for the new and\n// delete operations in C++.\n//\n// This header should be included in only one source file!\n//\n// On Windows, or when linking dynamically with mimalloc, these\n// can be more performant than the standard new-delete operations.\n// See \n// ---------------------------------------------------------------------------\n#if defined(__cplusplus)\n #include \n #include \n\n #if defined(_MSC_VER) && defined(_Ret_notnull_) && defined(_Post_writable_byte_size_)\n // stay consistent with VCRT definitions\n #define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict _Ret_notnull_ _Post_writable_byte_size_(n)\n #define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict _Ret_maybenull_ _Success_(return != NULL) _Post_writable_byte_size_(n)\n #else\n #define mi_decl_new(n) mi_decl_nodiscard mi_decl_restrict\n #define mi_decl_new_nothrow(n) mi_decl_nodiscard mi_decl_restrict\n #endif\n\n void operator delete(void* p) noexcept { mi_free(p); };\n void operator delete[](void* p) noexcept { mi_free(p); };\n\n void operator delete (void* p, const std::nothrow_t&) noexcept { mi_free(p); }\n void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); }\n\n mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }\n mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }\n\n mi_decl_new_nothrow(n) void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }\n mi_decl_new_nothrow(n) void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }\n\n #if (__cplusplus >= 201402L || _MSC_VER >= 1916)\n void operator delete (void* p, std::size_t n) noexcept { mi_free_size(p,n); };\n void operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); };\n #endif\n\n #if (__cplusplus > 201402L || defined(__cpp_aligned_new))\n void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); };\n void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); };\n void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); }\n\n void* operator new (std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); }\n void* operator new[](std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); }\n void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); }\n void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); }\n #endif\n#endif\n\n#endif // MIMALLOC_NEW_DELETE_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/include/mimalloc-override.h", "sha256": "21fcf61c4443341ac6bf6ea528af31dc7267e8e3456fc64bfd07704503032175", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2020 Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_OVERRIDE_H\n#define MIMALLOC_OVERRIDE_H\n\n/* ----------------------------------------------------------------------------\nThis header can be used to statically redirect malloc/free and new/delete\nto the mimalloc variants. This can be useful if one can include this file on\neach source file in a project (but be careful when using external code to\nnot accidentally mix pointers from different allocators).\n-----------------------------------------------------------------------------*/\n\n#include \n\n// Standard C allocation\n#define malloc(n) mi_malloc(n)\n#define calloc(c,n) mi_calloc(c,n)\n#define realloc(p,n) mi_realloc(p,n)\n#define free(p) mi_free(p)\n\n#define strdup(s) mi_strdup(s)\n#define strndup(s,n) mi_strndup(s,n)\n#define realpath(f,n) mi_realpath(f,n)\n\n// Microsoft extensions\n#define _expand(p,n) mi_expand(p,n)\n#define _msize(p) mi_usable_size(p)\n#define _recalloc(p,c,n) mi_recalloc(p,c,n)\n\n#define _strdup(s) mi_strdup(s)\n#define _strndup(s,n) mi_strndup(s,n)\n#define _wcsdup(s) mi_wcsdup(s)\n#define _mbsdup(s) mi_mbsdup(s)\n#define _dupenv_s(buf,n,nm) mi_dupenv_s(buf,n,nm)\n#define _wdupenv_s(buf,n,nm) mi_wdupenv_s(buf,n,nm)\n\n// Various Posix and Unix variants\n#define reallocf(p,n) mi_reallocf(p,n)\n#define malloc_size(p) mi_usable_size(p)\n#define malloc_usable_size(p) mi_usable_size(p)\n#define malloc_good_size(n) mi_malloc_good_size(n)\n#define cfree(p) mi_free(p)\n\n#define valloc(n) mi_valloc(n)\n#define pvalloc(n) mi_pvalloc(n)\n#define reallocarray(p,c,n) mi_reallocarray(p,c,n)\n#define reallocarr(ptrp,c,n) mi_reallocarr(ptrp,c,n)\n#define memalign(a,n) mi_memalign(a,n)\n#define aligned_alloc(a,n) mi_aligned_alloc(a,n)\n#define posix_memalign(p,a,n) mi_posix_memalign(p,a,n)\n#define _posix_memalign(p,a,n) mi_posix_memalign(p,a,n)\n\n// Microsoft aligned variants\n#define _aligned_malloc(n,a) mi_malloc_aligned(n,a)\n#define _aligned_realloc(p,n,a) mi_realloc_aligned(p,n,a)\n#define _aligned_recalloc(p,c,n,a) mi_aligned_recalloc(p,c,n,a)\n#define _aligned_msize(p,a,o) mi_usable_size(p)\n#define _aligned_free(p) mi_free(p)\n#define _aligned_offset_malloc(n,a,o) mi_malloc_aligned_at(n,a,o)\n#define _aligned_offset_realloc(p,n,a,o) mi_realloc_aligned_at(p,n,a,o)\n#define _aligned_offset_recalloc(p,c,n,a,o) mi_recalloc_aligned_at(p,c,n,a,o)\n\n#endif // MIMALLOC_OVERRIDE_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/include/mimalloc-stats.h", "sha256": "5260df227ea1612b8678a06b063cc8f48e739b551554541cdeb4d430a7e257b3", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2024-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_STATS_H\n#define MIMALLOC_STATS_H\n\n#include \n#include // memset\n#include // int64_t\n\n#define MI_STAT_VERSION 5 // increased on every backward incompatible change\n\n// alignment for atomic fields\n#if defined(_MSC_VER)\n#define mi_decl_align(a) __declspec(align(a))\n#elif defined(__GNUC__)\n#define mi_decl_align(a) __attribute__((aligned(a)))\n#elif __cplusplus >= 201103L\n#define mi_decl_align(a) alignas(a)\n#else\n#define mi_decl_align(a)\n#endif\n\n\n// count allocation over time\ntypedef struct mi_stat_count_s {\n int64_t total; // total allocated\n int64_t peak; // peak allocation\n int64_t current; // current allocation\n} mi_stat_count_t;\n\n// counters only increase\ntypedef struct mi_stat_counter_s {\n int64_t total; // total count\n} mi_stat_counter_t;\n\n#define MI_STAT_FIELDS() \\\n MI_STAT_COUNT(pages) /* count of mimalloc pages */ \\\n MI_STAT_COUNT(reserved) /* reserved memory bytes */ \\\n MI_STAT_COUNT(committed) /* committed bytes */ \\\n MI_STAT_COUNTER(reset) /* reset bytes */ \\\n MI_STAT_COUNTER(purged) /* purged bytes */ \\\n MI_STAT_COUNT(page_committed) /* committed memory inside pages */ \\\n MI_STAT_COUNT(pages_abandoned) /* abandoned pages count */ \\\n MI_STAT_COUNT(threads) /* number of threads */ \\\n MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \\\n MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \\\n MI_STAT_COUNT(malloc_requested) /* malloc requested bytes */ \\\n \\\n MI_STAT_COUNTER(mmap_calls) \\\n MI_STAT_COUNTER(commit_calls) \\\n MI_STAT_COUNTER(reset_calls) \\\n MI_STAT_COUNTER(purge_calls) \\\n MI_STAT_COUNTER(arena_count) /* number of memory arena's */ \\\n MI_STAT_COUNTER(malloc_normal_count) /* number of blocks <= MI_LARGE_OBJ_SIZE_MAX */ \\\n MI_STAT_COUNTER(malloc_huge_count) /* number of huge bloks */ \\\n MI_STAT_COUNTER(malloc_guarded_count) /* number of allocations with guard pages */ \\\n \\\n /* internal statistics */ \\\n MI_STAT_COUNTER(arena_rollback_count) \\\n MI_STAT_COUNTER(arena_purges) \\\n MI_STAT_COUNTER(pages_extended) /* number of page extensions */ \\\n MI_STAT_COUNTER(pages_retire) /* number of pages that are retired */ \\\n MI_STAT_COUNTER(page_searches) /* total pages searched for a fresh page */ \\\n MI_STAT_COUNTER(page_searches_count) /* searched count for a fresh page */ \\\n /* only on v1 and v2 */ \\\n MI_STAT_COUNT(segments) \\\n MI_STAT_COUNT(segments_abandoned) \\\n MI_STAT_COUNT(segments_cache) \\\n MI_STAT_COUNT(_segments_reserved) \\\n /* only on v3 */ \\\n MI_STAT_COUNT(heaps) \\\n MI_STAT_COUNT(theaps) \\\n MI_STAT_COUNTER(pages_reclaim_on_alloc) \\\n MI_STAT_COUNTER(pages_reclaim_on_free) \\\n MI_STAT_COUNTER(pages_reabandon_full) \\\n MI_STAT_COUNTER(pages_unabandon_busy_wait) \\\n MI_STAT_COUNTER(heaps_delete_wait)\n\n// Size bins for chunks\ntypedef enum mi_chunkbin_e {\n MI_CBIN_SMALL, // slice_count == 1\n MI_CBIN_OTHER, // slice_count: any other from the other bins, and 1 <= slice_count <= MI_BCHUNK_BITS\n MI_CBIN_MEDIUM, // slice_count == 8\n MI_CBIN_LARGE, // slice_count == MI_SIZE_BITS (only used if MI_ENABLE_LARGE_PAGES is 1)\n MI_CBIN_HUGE, // slice_count > MI_BCHUNK_BITS\n MI_CBIN_NONE, // no bin assigned yet (the chunk is completely free)\n MI_CBIN_COUNT\n} mi_chunkbin_t;\n\n\n// Define the statistics structure\n#define MI_BIN_HUGE (73U) // see types.h\n#define MI_STAT_COUNT(stat) mi_stat_count_t stat;\n#define MI_STAT_COUNTER(stat) mi_stat_counter_t stat;\n\ntypedef struct mi_stats_s\n{\n size_t size; // size of the mi_stats_t structure \n size_t version; \n\n mi_decl_align(8) MI_STAT_FIELDS()\n\n // future extension\n mi_stat_count_t _stat_reserved[4];\n mi_stat_counter_t _stat_counter_reserved[4];\n\n // size segregated statistics\n mi_stat_count_t malloc_bins[MI_BIN_HUGE+1]; // allocation per size bin\n mi_stat_count_t page_bins[MI_BIN_HUGE+1]; // pages allocated per size bin\n mi_stat_count_t chunk_bins[MI_CBIN_COUNT]; // chunks per page sizes\n} mi_stats_t;\n\n#undef MI_STAT_COUNT\n#undef MI_STAT_COUNTER\n\n// Initialization\nstatic inline void mi_stats_header_init(mi_stats_t* stats) {\n stats->size = sizeof(*stats);\n stats->version = MI_STAT_VERSION;\n}\nstatic inline void mi_stats_init(mi_stats_t* stats) {\n memset(stats,0,sizeof(*stats));\n mi_stats_header_init(stats);\n}\n\n#define mi_stats_t_decl(name) mi_stats_t name; mi_stats_init(&name);\n\n// Exported definitions\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// stats from a heap\nmi_decl_export bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept;\nmi_decl_export char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL\nmi_decl_export void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept;\n\n// stats from a subprocess and its heaps aggregated\nmi_decl_export bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept;\nmi_decl_export char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL\nmi_decl_export void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept;\n// print subprocess and all its heap stats segregated\nmi_decl_export void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept;\n\n// stats aggregated for the current subprocess and all its heaps.\nmi_decl_export bool mi_stats_get(mi_stats_t* stats) mi_attr_noexcept;\nmi_decl_export char* mi_stats_get_json(size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL\nmi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;\n\n// add the stats of the heap to the subprocess and clear the heap stats\nmi_decl_export void mi_heap_stats_merge_to_subproc(mi_heap_t* heap);\n\n// stats from the subprocess without aggregating its heaps\nmi_decl_export bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept;\n\nmi_decl_export char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL\nmi_decl_export size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept;\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif // MIMALLOC_STATS_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/include/mimalloc.h", "sha256": "869161433ab2d807a75752d0762accf5b68b6b24ce67e417af46bf181b9b6c72", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2026, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_H\n#define MIMALLOC_H\n\n#define MI_MALLOC_VERSION 30302 // major + 2 digits minor + 2 digits patch\n\n// ------------------------------------------------------\n// Compiler specific attributes\n// ------------------------------------------------------\n\n#ifdef __cplusplus\n #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11\n #define mi_attr_noexcept noexcept\n #else\n #define mi_attr_noexcept throw()\n #endif\n#else\n #define mi_attr_noexcept\n#endif\n\n#if defined(__cplusplus) && (__cplusplus >= 201703)\n #define mi_decl_nodiscard [[nodiscard]]\n#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl\n #define mi_decl_nodiscard __attribute__((warn_unused_result))\n#elif defined(_HAS_NODISCARD)\n #define mi_decl_nodiscard _NODISCARD\n#elif (_MSC_VER >= 1700)\n #define mi_decl_nodiscard _Check_return_\n#else\n #define mi_decl_nodiscard\n#endif\n\n#if defined(_MSC_VER) || defined(__MINGW32__)\n #if !defined(MI_SHARED_LIB)\n #define mi_decl_export\n #elif defined(MI_SHARED_LIB_EXPORT)\n #define mi_decl_export __declspec(dllexport)\n #else\n #define mi_decl_export __declspec(dllimport)\n #endif\n #if defined(__MINGW32__)\n #define mi_decl_restrict\n #define mi_attr_malloc __attribute__((malloc))\n #else\n #if (_MSC_VER >= 1900) && !defined(__EDG__)\n #define mi_decl_restrict __declspec(allocator) __declspec(restrict)\n #else\n #define mi_decl_restrict __declspec(restrict)\n #endif\n #define mi_attr_malloc\n #endif\n #define mi_cdecl __cdecl\n #define mi_attr_alloc_size(s)\n #define mi_attr_alloc_size2(s1,s2)\n #define mi_attr_alloc_align(p)\n#elif defined(__GNUC__) // includes clang and icc\n #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT)\n #define mi_decl_export __attribute__((visibility(\"default\")))\n #else\n #define mi_decl_export\n #endif\n #define mi_cdecl // leads to warnings... __attribute__((cdecl))\n #define mi_decl_restrict\n #define mi_attr_malloc __attribute__((malloc))\n #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5)\n #define mi_attr_alloc_size(s)\n #define mi_attr_alloc_size2(s1,s2)\n #define mi_attr_alloc_align(p)\n #elif defined(__INTEL_COMPILER)\n #define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))\n #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))\n #define mi_attr_alloc_align(p)\n #else\n #define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))\n #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))\n #define mi_attr_alloc_align(p) __attribute__((alloc_align(p)))\n #endif\n#else\n #define mi_cdecl\n #define mi_decl_export\n #define mi_decl_restrict\n #define mi_attr_malloc\n #define mi_attr_alloc_size(s)\n #define mi_attr_alloc_size2(s1,s2)\n #define mi_attr_alloc_align(p)\n#endif\n\n// ------------------------------------------------------\n// Includes\n// ------------------------------------------------------\n\n#include // size_t\n#include // bool\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// ------------------------------------------------------\n// Standard malloc interface\n// ------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);\nmi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);\nmi_decl_export void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);\n\nmi_decl_export void mi_free(void* p) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept;\n\n// ------------------------------------------------------\n// Extended allocation functions\n// ------------------------------------------------------\n#define MI_SMALL_WSIZE_MAX (128)\n#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*))\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);\nmi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);\nmi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);\n\nmi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;\n\n// `mi_free_small` is for special applications like language runtimes.\n// it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free`\nmi_decl_export void mi_free_small(void* p) mi_attr_noexcept; \n\n// -------------------------------------------------------------------------------------\n// Aligned allocation\n// Note that `alignment` always follows `size` for consistency with unaligned\n// allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`.\n// -------------------------------------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2) mi_attr_alloc_align(3);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2);\nmi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);\nmi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);\n\n\n// ------------------------------------------------------\n// Typed allocation, the type is always the first parameter\n// ------------------------------------------------------\n\n#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp)))\n#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp)))\n#define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp)))\n#define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp)))\n#define mi_reallocn_tp(tp,p,n) ((tp*)mi_reallocn(p,n,sizeof(tp)))\n#define mi_recalloc_tp(tp,p,n) ((tp*)mi_recalloc(p,n,sizeof(tp)))\n\n#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))\n#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))\n#define mi_heap_calloc_tp(tp,hp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp)))\n#define mi_heap_mallocn_tp(tp,hp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp)))\n#define mi_heap_reallocn_tp(tp,hp,p,n) ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp)))\n#define mi_heap_recalloc_tp(tp,hp,p,n) ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp)))\n\n\n// ------------------------------------------------------\n// Internals\n// See also `mimalloc-stats.h` for statistics\n// ------------------------------------------------------\n\ntypedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);\nmi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept;\n\ntypedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg);\nmi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept;\n\ntypedef void (mi_cdecl mi_error_fun)(int err, void* arg);\nmi_decl_export void mi_register_error(mi_error_fun* fun, void* arg);\n\nmi_decl_export void mi_collect(bool force) mi_attr_noexcept;\nmi_decl_export int mi_version(void) mi_attr_noexcept;\nmi_decl_export void mi_options_print(void) mi_attr_noexcept;\nmi_decl_export void mi_process_info_print(void) mi_attr_noexcept;\nmi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;\nmi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;\nmi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs,\n size_t* current_rss, size_t* peak_rss,\n size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept;\n\n\n\n// Generally do not use the following as these are usually called automatically\nmi_decl_export void mi_process_init(void) mi_attr_noexcept;\nmi_decl_export void mi_cdecl mi_process_done(void) mi_attr_noexcept;\nmi_decl_export void mi_thread_init(void) mi_attr_noexcept;\nmi_decl_export void mi_thread_done(void) mi_attr_noexcept;\nmi_decl_export void mi_thread_set_in_threadpool(void) mi_attr_noexcept; // communicate that a thread is in a threadpool\n\n\n// -----------------------------------------------------------------\n// Return allocated block size (if the return value is not NULL)\n// -----------------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);\nmi_decl_nodiscard mi_decl_export void* mi_urealloc(void* p, size_t newsize, size_t* block_size_pre, size_t* block_size_post) mi_attr_noexcept mi_attr_alloc_size(2);\nmi_decl_export void mi_ufree(void* p, size_t* block_size) mi_attr_noexcept;\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\n\n\n// -------------------------------------------------------------------------------------\n// Heaps: first-class. Can allocate from any thread (and be free'd from any thread)\n// Heaps keep allocations in separate pages from each other (but share the arena's and free'd pages)\n// -------------------------------------------------------------------------------------\n\nstruct mi_heap_s;\ntypedef struct mi_heap_s mi_heap_t;\n\nmi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void);\nmi_decl_export void mi_heap_delete(mi_heap_t* heap); // move live blocks to the main heap\nmi_decl_export void mi_heap_destroy(mi_heap_t* heap); // free all live blocks\nmi_decl_export void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node);\nmi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force);\n\nmi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_main(void);\nmi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_of(const void* p);\nmi_decl_nodiscard mi_decl_export bool mi_heap_contains(const mi_heap_t* heap, const void* p);\nmi_decl_nodiscard mi_decl_export bool mi_any_heap_contains(const void* p);\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\n\nmi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);\nmi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3, 4);\nmi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept;\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);\nmi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);\nmi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);\n\n\n// --------------------------------------------------------------------------------\n// Zero initialized re-allocation.\n// Only valid on memory that was originally allocated with zero initialization too.\n// e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc.\n// see \n// --------------------------------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);\n\nmi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);\nmi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4);\nmi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3);\n\nmi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);\nmi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3, 4);\n\nmi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);\nmi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);\nmi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3, 4) mi_attr_alloc_align(5);\nmi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3, 4);\n\n\n\n// ------------------------------------------------------\n// Visiting pages and individual blocks in a heap.\n// ------------------------------------------------------\n\n// An area of heap space contains blocks of a single size.\ntypedef struct mi_heap_area_s {\n void* blocks; // start of the area containing theap blocks\n size_t reserved; // bytes reserved for this area (virtual)\n size_t committed; // current available bytes for this area\n size_t used; // number of allocated blocks\n size_t block_size; // size in bytes of each block\n size_t full_block_size; // size in bytes of a full block including padding and metadata.\n void* reserved1; // internal\n} mi_heap_area_t;\n\ntypedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);\n\nmi_decl_export bool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);\nmi_decl_export bool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);\n\n\n// ------------------------------------------------------\n// Arena memory management\n// Arena's are larger memory area's provided by the OS or user\n// ------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept;\n\nmi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept;\nmi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept;\n\nmi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;\nmi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_pinned /* cannot decommit/reset? */, bool is_zero, int numa_node) mi_attr_noexcept;\n\nmi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept;\nmi_decl_export void mi_arenas_print(void) mi_attr_noexcept;\nmi_decl_export size_t mi_arena_min_alignment(void);\nmi_decl_export size_t mi_arena_min_size(void);\n\ntypedef void* mi_arena_id_t;\nmi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);\nmi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;\nmi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;\nmi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;\nmi_decl_export bool mi_arena_contains(mi_arena_id_t arena_id, const void* p);\n\n// Create a heap that only allocates in the specified arena\nmi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);\n\n\n// ------------------------------------------------------\n// Subprocesses\n// Advanced: allow sub-processes whose memory arena's stay fully separated (and no reclamation between them).\n// Used for example for separate interpreters in one process.\n// ------------------------------------------------------\n\ntypedef struct { void* _mi_subproc_id; } mi_subproc_id_t; // abstract type\nmi_decl_export mi_subproc_id_t mi_subproc_main(void);\nmi_decl_export mi_subproc_id_t mi_subproc_current(void);\nmi_decl_export mi_subproc_id_t mi_subproc_new(void);\nmi_decl_export void mi_subproc_destroy(mi_subproc_id_t subproc);\nmi_decl_export void mi_subproc_add_current_thread(mi_subproc_id_t subproc); // this should be called right after a thread is created (and no allocation has taken place yet)\n\ntypedef bool (mi_cdecl mi_heap_visit_fun)(mi_heap_t* heap, void* arg);\nmi_decl_export bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg);\n\n\n// -------------------------------------------------------------------------------------\n// A \"theap\" is a thread-local heap. This API is only provided for special circumstances like runtimes\n// that already have a thread-local context and can store the theap there for (slightly) faster allocations.\n// This also allows to set a default theap for the current thread so that `malloc` etc. allocate from\n// that theap (instead of the main (t)heap).\n// Theaps are first-class, but can only allocate from the same thread that created it.\n// Allocation through a `theap` may be a tiny bit faster than using plain malloc\n// (as we don't need to lookup the thread local variable).\n// -------------------------------------------------------------------------------------\n\nstruct mi_theap_s;\ntypedef struct mi_theap_s mi_theap_t;\n\nmi_decl_export mi_theap_t* mi_heap_theap(mi_heap_t* heap);\nmi_decl_export mi_theap_t* mi_theap_set_default(mi_theap_t* theap);\nmi_decl_export mi_theap_t* mi_theap_get_default(void);\nmi_decl_export void mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept;\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);\nmi_decl_nodiscard mi_decl_export void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);\n\n\n// ------------------------------------------------------\n// Experimental\n// ------------------------------------------------------\n\n// Experimental: objects followed by a guard page.\n// Setting the sample rate on a specific theap can be used to test parts of the program more\n// specifically (in combination with `mi_theap_set_default`).\n// A sample rate of 0 disables guarded objects, while 1 uses a guard page for every object.\n// A seed of 0 uses a random start point. Only objects within the size bound are eligable for guard pages.\nmi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed);\nmi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max);\n\n// very experimental\ntypedef bool (mi_cdecl mi_commit_fun_t)(bool commit, void* start, size_t size, bool* is_zero, void* user_arg);\nmi_decl_export bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive,\n mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept;\n\n//mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* size);\n//mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id);\n//mi_decl_export bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena);\n//mi_decl_export void mi_theap_unload(mi_theap_t* theap);\n\n// unsafe: assumes the page belonging to `p` is only accessed by the calling thread.\nmi_decl_export bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept;\n\n// ------------------------------------------------------\n// Deprecated\n// ------------------------------------------------------\n\nmi_decl_export bool mi_check_owned(const void* p);\n\nmi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;\nmi_decl_export bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);\n\nmi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;\nmi_decl_export void mi_collect_reduce(size_t target_thread_owned) mi_attr_noexcept;\n\nmi_decl_export void mi_stats_reset(void) mi_attr_noexcept;\nmi_decl_export void mi_stats_merge(void) mi_attr_noexcept;\nmi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL\n\nmi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; // not deprecated but declared in `mimalloc-stats.h` now.\n\n\n// ------------------------------------------------------\n// Options\n// ------------------------------------------------------\n\ntypedef enum mi_option_e {\n // stable options\n mi_option_show_errors, // print error messages\n mi_option_show_stats, // print statistics on termination\n mi_option_verbose, // print verbose messages\n // advanced options\n mi_option_deprecated_eager_commit, \n mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2)\n mi_option_purge_decommits, // should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)\n mi_option_allow_large_os_pages, // allow use of large (2 or 4 MiB) OS pages, implies eager commit.\n mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB pages) at startup\n mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node\n mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`)\n mi_option_deprecated_segment_cache,\n mi_option_deprecated_page_reset,\n mi_option_deprecated_abandoned_page_purge,\n mi_option_deprecated_segment_reset,\n mi_option_deprecated_eager_commit_delay, \n mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10)\n mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes.\n mi_option_disallow_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas)\n mi_option_os_tag, // tag used for OS logging (macOS only for now) (=100)\n mi_option_max_errors, // issue at most N error messages\n mi_option_max_warnings, // issue at most N warning messages\n mi_option_deprecated_max_segment_reclaim, // max. percentage of the abandoned segments can be reclaimed per try (=10%)\n mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe\n mi_option_arena_reserve, // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`)\n mi_option_arena_purge_mult, // multiplier for `purge_delay` for the purging delay for arenas (=10)\n mi_option_deprecated_purge_extend_delay,\n mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's)\n mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)\n mi_option_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0)\n mi_option_guarded_min, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)\n mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)\n mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)\n mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000)\n mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)\n mi_option_generic_collect, // collect theaps every N (=10000) generic allocation calls\n mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from the current theap, 1 allow always\n mi_option_page_full_retain, // retain N full (small) pages per size class (=2)\n mi_option_page_max_candidates, // max candidate pages to consider for allocation (=4)\n mi_option_max_vabits, // max user space virtual address bits to consider (=48)\n mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0)\n mi_option_page_commit_on_demand, // commit page memory on-demand\n mi_option_page_max_reclaim, // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited))\n mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16)\n mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.\n mi_option_minimal_purge_size, // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled.\n mi_option_arena_max_object_size, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). \n mi_option_arena_is_numa_local, // experimental\n _mi_option_last,\n // legacy option names\n mi_option_large_os_pages = mi_option_allow_large_os_pages,\n mi_option_eager_region_commit = mi_option_arena_eager_commit,\n mi_option_reset_decommits = mi_option_purge_decommits,\n mi_option_reset_delay = mi_option_purge_delay,\n mi_option_limit_os_alloc = mi_option_disallow_os_alloc\n} mi_option_t;\n\n\nmi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option);\nmi_decl_export void mi_option_enable(mi_option_t option);\nmi_decl_export void mi_option_disable(mi_option_t option);\nmi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);\nmi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);\n\nmi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);\nmi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max);\nmi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option);\nmi_decl_export void mi_option_set(mi_option_t option, long value);\nmi_decl_export void mi_option_set_default(mi_option_t option, long value);\n\n\n// -------------------------------------------------------------------------------------------------------\n// \"mi\" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions.\n// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.)\n// note: we use `mi_cfree` as \"checked free\" and it checks if the pointer is in our theap before free-ing.\n// -------------------------------------------------------------------------------------------------------\n\nmi_decl_export void mi_cfree(void* p) mi_attr_noexcept;\nmi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;\n\nmi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);\n\nmi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);\nmi_decl_nodiscard mi_decl_export int mi_reallocarr(void* ptrp, size_t count, size_t size) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;\n\nmi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept;\nmi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;\nmi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept;\nmi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept;\n\n// wide characters\n#include // wchar_t\nmi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc;\nmi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc;\n\n// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.\n// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2);\nmi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);\n\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2);\nmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);\n\n#ifdef __cplusplus\n}\n#endif\n\n// ---------------------------------------------------------------------------------------------\n// Implement the C++ std::allocator interface for use in STL containers.\n// (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally)\n// ---------------------------------------------------------------------------------------------\n#ifdef __cplusplus\n\n#include // std::size_t\n#include // PTRDIFF_MAX\n#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11\n#include // std::true_type\n#include // std::forward\n#endif\n\ntemplate struct _mi_stl_allocator_common {\n typedef T value_type;\n typedef std::size_t size_type;\n typedef std::ptrdiff_t difference_type;\n typedef value_type& reference;\n typedef value_type const& const_reference;\n typedef value_type* pointer;\n typedef value_type const* const_pointer;\n\n #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11\n using propagate_on_container_copy_assignment = std::true_type;\n using propagate_on_container_move_assignment = std::true_type;\n using propagate_on_container_swap = std::true_type;\n template void construct(U* p, Args&& ...args) { ::new(p) U(std::forward(args)...); }\n template void destroy(U* p) mi_attr_noexcept { p->~U(); }\n #else\n void construct(pointer p, value_type const& val) { ::new(p) value_type(val); }\n void destroy(pointer p) { p->~value_type(); }\n #endif\n\n size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); }\n pointer address(reference x) const { return &x; }\n const_pointer address(const_reference x) const { return &x; }\n};\n\ntemplate struct mi_stl_allocator : public _mi_stl_allocator_common {\n using typename _mi_stl_allocator_common::size_type;\n using typename _mi_stl_allocator_common::value_type;\n using typename _mi_stl_allocator_common::pointer;\n template struct rebind { typedef mi_stl_allocator other; };\n\n mi_stl_allocator() mi_attr_noexcept = default;\n mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept = default;\n template mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept { }\n mi_stl_allocator select_on_container_copy_construction() const { return *this; }\n void deallocate(T* p, size_type) { mi_free(p); }\n\n #if (__cplusplus >= 201703L) // C++17\n mi_decl_nodiscard T* allocate(size_type count) { return static_cast(mi_new_n(count, sizeof(T))); }\n mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }\n #else\n mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast(mi_new_n(count, sizeof(value_type))); }\n #endif\n\n #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11\n using is_always_equal = std::true_type;\n #endif\n};\n\ntemplate bool operator==(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return true; }\ntemplate bool operator!=(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return false; }\n\n\n#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11\n#define MI_HAS_HEAP_STL_ALLOCATOR 1\n\n#include // std::shared_ptr\n\n// Common base class for STL allocators in a specific theap\ntemplate struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common {\n using typename _mi_stl_allocator_common::size_type;\n using typename _mi_stl_allocator_common::value_type;\n using typename _mi_stl_allocator_common::pointer;\n\n _mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp, [](mi_heap_t*) {}) {} /* will not delete nor destroy the passed in heap */\n\n #if (__cplusplus >= 201703L) // C++17\n mi_decl_nodiscard T* allocate(size_type count) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); }\n mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); }\n #else\n mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); }\n #endif\n\n #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11\n using is_always_equal = std::false_type;\n #endif\n\n void collect(bool force) { mi_heap_collect(this->heap.get(), force); }\n template bool is_equal(const _mi_heap_stl_allocator_common& x) const { return (this->heap == x.heap); }\n\nprotected:\n std::shared_ptr heap;\n template friend struct _mi_heap_stl_allocator_common;\n\n _mi_heap_stl_allocator_common() {\n mi_heap_t* hp = mi_heap_new();\n this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */\n }\n _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { }\n template _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { }\n\nprivate:\n static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } }\n static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } }\n};\n\n// STL allocator allocation in a specific heap\ntemplate struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common {\n using typename _mi_heap_stl_allocator_common::size_type;\n mi_heap_stl_allocator() : _mi_heap_stl_allocator_common() { } // creates fresh heap that is deleted when the destructor is called\n mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap\n template mi_heap_stl_allocator(const mi_heap_stl_allocator& x) mi_attr_noexcept : _mi_heap_stl_allocator_common(x) { }\n\n mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; }\n void deallocate(T* p, size_type) { mi_free(p); }\n template struct rebind { typedef mi_heap_stl_allocator other; };\n};\n\ntemplate bool operator==(const mi_heap_stl_allocator& x, const mi_heap_stl_allocator& y) mi_attr_noexcept { return (x.is_equal(y)); }\ntemplate bool operator!=(const mi_heap_stl_allocator& x, const mi_heap_stl_allocator& y) mi_attr_noexcept { return (!x.is_equal(y)); }\n\n\n// STL allocator allocation in a specific heap, where `free` does nothing and\n// the heap is destroyed in one go on destruction -- use with care!\ntemplate struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common {\n using typename _mi_heap_stl_allocator_common::size_type;\n mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common() { } // creates fresh heap that is destroyed when the destructor is called\n mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap\n template mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator& x) mi_attr_noexcept : _mi_heap_stl_allocator_common(x) { }\n\n mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; }\n void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ }\n template struct rebind { typedef mi_heap_destroy_stl_allocator other; };\n};\n\ntemplate bool operator==(const mi_heap_destroy_stl_allocator& x, const mi_heap_destroy_stl_allocator& y) mi_attr_noexcept { return (x.is_equal(y)); }\ntemplate bool operator!=(const mi_heap_destroy_stl_allocator& x, const mi_heap_destroy_stl_allocator& y) mi_attr_noexcept { return (!x.is_equal(y)); }\n\n#endif // C++11\n\n#endif // __cplusplus\n\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/include/mimalloc/atomic.h", "sha256": "e9512b521a82cc0cbc262f6060dd70bb4243ac4a7d69d6ab8b1ef8e56f181a21", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2024 Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MI_ATOMIC_H\n#define MI_ATOMIC_H\n\n// include windows.h or pthreads.h\n#if defined(_WIN32)\n#ifndef WIN32_LEAN_AND_MEAN\n#define WIN32_LEAN_AND_MEAN\n#endif\n#include \n#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))\n#define MI_USE_PTHREADS\n#include \n#endif\n\n// --------------------------------------------------------------------------------------------\n// Atomics\n// We need to be portable between C, C++, and MSVC.\n// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode.\n// This is why we try to use only `uintptr_t` and `*` as atomic types.\n// To gain better insight in the range of used atomics, we use explicitly named memory order operations\n// instead of passing the memory order as a parameter.\n// -----------------------------------------------------------------------------------------------\n\n#if defined(__cplusplus)\n// Use C++ atomics\n#include \n#define _Atomic(tp) std::atomic\n#define mi_atomic(name) std::atomic_##name\n#define mi_memory_order(name) std::memory_order_##name\n#if (__cplusplus >= 202002L) // c++20, see issue #571\n #define MI_ATOMIC_VAR_INIT(x) x\n#elif !defined(ATOMIC_VAR_INIT)\n #define MI_ATOMIC_VAR_INIT(x) x\n#else\n #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)\n#endif\n#elif defined(_MSC_VER)\n// Use MSVC C wrapper for C11 atomics\n#define _Atomic(tp) tp\n#define MI_ATOMIC_VAR_INIT(x) x\n#define mi_atomic(name) mi_atomic_##name\n#define mi_memory_order(name) mi_memory_order_##name\n#else\n// Use C11 atomics\n#include \n#define mi_atomic(name) atomic_##name\n#define mi_memory_order(name) memory_order_##name\n#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735\n #define MI_ATOMIC_VAR_INIT(x) x\n#elif !defined(ATOMIC_VAR_INIT)\n #define MI_ATOMIC_VAR_INIT(x) x\n#else\n #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x)\n#endif\n#endif\n\n// Various defines for all used memory orders in mimalloc\n#define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail) \\\n mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail)\n\n#define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail) \\\n mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail)\n\n#define mi_atomic_load_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))\n#define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))\n#define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))\n#define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))\n#define mi_atomic_exchange_relaxed(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(relaxed))\n#define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release))\n#define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel))\n\n#define mi_atomic_cas_weak_relaxed(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed))\n#define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))\n#define mi_atomic_cas_weak_acq_rel(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))\n#define mi_atomic_cas_strong_relaxed(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(relaxed),mi_memory_order(relaxed))\n#define mi_atomic_cas_strong_release(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))\n#define mi_atomic_cas_strong_acq_rel(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire))\n\n#define mi_atomic_add_relaxed(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed))\n#define mi_atomic_add_acq_rel(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel))\n#define mi_atomic_sub_relaxed(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed))\n#define mi_atomic_sub_acq_rel(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel))\n#define mi_atomic_and_relaxed(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(relaxed))\n#define mi_atomic_and_acq_rel(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel))\n#define mi_atomic_or_relaxed(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(relaxed))\n#define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel))\n\n#define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(uintptr_t)1)\n#define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(uintptr_t)1)\n#define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1)\n#define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1)\n\nstatic inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add);\nstatic inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub);\n\n\n#if defined(__cplusplus) || !defined(_MSC_VER)\n\n// In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value)\n// We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well\n#define mi_atomic_load_ptr_acquire(tp,p) mi_atomic_load_acquire(p)\n#define mi_atomic_load_ptr_relaxed(tp,p) mi_atomic_load_relaxed(p)\n\n// In C++ we need to add casts to help resolve templates if NULL is passed\n#if defined(__cplusplus)\n#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,(tp*)x)\n#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,(tp*)x)\n#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des)\n#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des)\n#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des)\n#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,(tp*)des)\n#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,(tp*)x)\n#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x)\n#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x)\n#else\n#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,x)\n#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,x)\n#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des)\n#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des)\n#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des)\n#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel(p,exp,des)\n#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,x)\n#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x)\n#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x)\n#endif\n\n// These are used by the statistics\nstatic inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) {\n return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));\n}\nstatic inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {\n const int64_t add = mi_atomic_load_relaxed((_Atomic(int64_t)*)padd);\n if (add != 0) {\n mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed));\n }\n}\nstatic inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {\n int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p);\n while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, ¤t, x)) { /* nothing */ };\n}\n\n// Used by timers\n#define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire))\n#define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))\n#define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))\n#define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))\n\n#define mi_atomic_casi64_strong_acq_rel(p,e,d) mi_atomic_cas_strong_acq_rel(p,e,d)\n#define mi_atomic_addi64_acq_rel(p,i) mi_atomic_add_acq_rel(p,i)\n\n\n#elif defined(_MSC_VER)\n\n// Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.\n// It is recommended to always compile as C++ when using MSVC.\n\n#include \n#ifdef _WIN64\ntypedef LONG64 msc_intptr_t;\n#define MI_MSC_64(f) f##64\n#define MI_MSC_XX(f) f##64\n#else\ntypedef LONG msc_intptr_t;\n#define MI_MSC_64(f) f\n#define MI_MSC_XX(f) f##32\n#endif\n\ntypedef enum mi_memory_order_e {\n mi_memory_order_relaxed,\n mi_memory_order_consume,\n mi_memory_order_acquire,\n mi_memory_order_release,\n mi_memory_order_acq_rel,\n mi_memory_order_seq_cst\n} mi_memory_order;\n\nstatic inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) {\n (void)(mo);\n return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add);\n}\nstatic inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) {\n (void)(mo);\n return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub));\n}\nstatic inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {\n (void)(mo);\n return (uintptr_t)MI_MSC_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x);\n}\nstatic inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {\n (void)(mo);\n return (uintptr_t)MI_MSC_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x);\n}\nstatic inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {\n (void)(mo1); (void)(mo2);\n const uintptr_t read = (uintptr_t)MI_MSC_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected));\n if (read == *expected) {\n return true;\n }\n else {\n *expected = read;\n return false;\n }\n}\nstatic inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {\n return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2);\n}\nstatic inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) {\n (void)(mo);\n return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);\n}\nstatic inline void mi_atomic_thread_fence(mi_memory_order mo) {\n (void)(mo);\n _Atomic(uintptr_t) x = 0;\n mi_atomic_exchange_explicit(&x, 1, mo);\n}\n\nstatic inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) {\n (void)(mo);\n // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc\n #if defined(_M_IX86) || defined(_M_X64)\n return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);\n #elif defined(_M_ARM) || defined(_M_ARM64)\n if (mo == mi_memory_order_relaxed) {\n return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);\n }\n else if (mo <= mi_memory_order_acquire) {\n return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p);\n }\n else {\n const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);\n __dmb(15); // _ARM(64)_BARRIER_SY\n return u;\n }\n #else\n #warning \"define mi_atomic_load_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)\"\n return MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);\n #endif\n}\nstatic inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {\n (void)(mo);\n // assert(mo<=mi_memory_order_release); // others are not used by mimalloc\n #if defined(_M_IX86) || defined(_M_X64)\n MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);\n #elif defined(_M_ARM) || defined(_M_ARM64)\n if (mo == mi_memory_order_relaxed) {\n MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);\n }\n else if (mo <= mi_memory_order_release) {\n MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x);\n }\n else {\n mi_atomic_exchange_explicit(p, x, mo);\n }\n #else\n mi_atomic_exchange_explicit(p, x, mo);\n #endif\n}\n\nstatic inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) {\n (void)(mo);\n // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc\n #if defined(_M_IX86) || defined(_M_X64)\n return __iso_volatile_load64((volatile const int64_t*)p);\n #elif defined(_M_ARM) || defined(_M_ARM64)\n if (mo == mi_memory_order_relaxed) {\n return __iso_volatile_load64((volatile const int64_t*)p);\n }\n #if defined(_M_ARM64)\n else if (mo <= mi_memory_order_acquire) {\n return __ldar64((volatile const uintptr_t*)p);\n }\n #endif\n else {\n const int64_t i = __iso_volatile_load64((volatile const int64_t*)p);\n __dmb(15); // _ARM(64)_BARRIER_SY\n return i;\n }\n #else\n #warning \"define mi_atomic_loadi64_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)\"\n return __iso_volatile_load64((volatile const int64_t*)p);\n #endif\n}\n\nstatic inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) {\n (void)(mo);\n // assert(mo<=mi_memory_order_release); // others are not used by mimalloc\n #if defined(_M_IX86) || defined(_M_X64)\n __iso_volatile_store64((volatile int64_t*)p,x);\n #elif defined(_M_ARM) || defined(_M_ARM64)\n if (mo == mi_memory_order_relaxed) {\n __iso_volatile_store64((volatile int64_t*)p,x);\n }\n #if defined(_M_ARM64)\n else if (mo == mi_memory_order_release) {\n __stlr64((volatile uint64_t*)p, (uint64_t)x);\n }\n #endif\n else {\n InterlockedExchange64(p, x);\n }\n #else\n InterlockedExchange64(p, x);\n #endif\n}\n\n// These are used by the statistics\nstatic inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) {\n #ifdef _WIN64\n return (int64_t)mi_atomic_addi((int64_t*)p, add);\n #elif defined(_M_ARM)\n return _InterlockedExchangeAdd64(p, add);\n #else\n // x86\n int64_t current;\n int64_t sum;\n do {\n current = __iso_volatile_load64((volatile const int64_t*)p);\n sum = current + add;\n } while (_InterlockedCompareExchange64(p, sum, current) != current);\n return current;\n #endif\n}\n\nstatic inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {\n const int64_t add = *padd;\n if (add != 0) {\n mi_atomic_addi64_relaxed((volatile _Atomic(int64_t)*)p, add);\n }\n}\n\nstatic inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) {\n int64_t current;\n do {\n current = *p;\n } while (current < x && _InterlockedCompareExchange64(p, x, current) != current);\n}\n\nstatic inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) {\n mi_atomic_addi64_relaxed(p, i);\n}\n\nstatic inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) {\n const int64_t read = _InterlockedCompareExchange64(p, des, *exp);\n if (read == *exp) {\n return true;\n }\n else {\n *exp = read;\n return false;\n }\n}\n\n// The pointer macros cast to `uintptr_t`.\n#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))\n#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))\n#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))\n#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))\n#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)\n#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)\n#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)\n#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)\n#define mi_atomic_exchange_ptr_relaxed(tp,p,x) (tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)\n#define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)\n#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)\n\n#define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire))\n#define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed))\n#define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release))\n#define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed))\n\n\n#endif\n\n\n// Atomically add a signed value; returns the previous value.\nstatic inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) {\n return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add);\n}\n\n// Atomically subtract a signed value; returns the previous value.\nstatic inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) {\n return (intptr_t)mi_atomic_addi(p, -sub);\n}\n\n\n// ----------------------------------------------------------------------\n// Guard\n// ----------------------------------------------------------------------\n\ntypedef _Atomic(uintptr_t) mi_atomic_guard_t;\n\n// Allows only one thread to execute at a time (without blocking anyone)\n#define mi_atomic_guard(guard) \\\n uintptr_t _mi_guard_expected = 0; \\\n for(bool _mi_guard_once = true; \\\n _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \\\n (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) )\n\n\n// ----------------------------------------------------------------------\n// Locks\n// These should be light-weight in-process only locks.\n// Only used for reserving arena's and to maintain the abandoned list.\n// ----------------------------------------------------------------------\n#if _MSC_VER\n#pragma warning(disable:26110) // unlock with holding lock\n#endif\n\n#define mi_lock(lock) for(bool _mi_go = (mi_lock_acquire(lock),true); _mi_go; (mi_lock_release(lock), _mi_go=false) )\n#define mi_lock_maybe(lock,acquire) for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) )\n\n\n#if defined(_WIN32)\n\ntypedef struct mi_lock_s {\n SRWLOCK mutex; // slim reader-writer lock\n} mi_lock_t;\n\n#define MI_LOCK_INITIALIZER { SRWLOCK_INIT }\n\nstatic inline bool mi_lock_try_acquire(mi_lock_t* lock) {\n return TryAcquireSRWLockExclusive(&lock->mutex);\n}\nstatic inline void mi_lock_acquire(mi_lock_t* lock) {\n AcquireSRWLockExclusive(&lock->mutex);\n}\nstatic inline void mi_lock_release(mi_lock_t* lock) {\n ReleaseSRWLockExclusive(&lock->mutex);\n}\nstatic inline void mi_lock_init(mi_lock_t* lock) {\n InitializeSRWLock(&lock->mutex);\n}\nstatic inline void mi_lock_done(mi_lock_t* lock) {\n (void)(lock);\n}\n\n#elif defined(MI_USE_PTHREADS)\n\n#include // memcpy\nvoid _mi_error_message(int err, const char* fmt, ...);\n\ntypedef struct mi_lock_s {\n pthread_mutex_t mutex;\n} mi_lock_t;\n\n#define MI_LOCK_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }\n\nstatic inline bool mi_lock_try_acquire(mi_lock_t* lock) {\n return (pthread_mutex_trylock(&lock->mutex) == 0);\n}\nstatic inline void mi_lock_acquire(mi_lock_t* lock) {\n const int err = pthread_mutex_lock(&lock->mutex);\n if (err != 0) {\n _mi_error_message(err, \"internal error: lock cannot be acquired (err %i)\\n\", err);\n }\n}\nstatic inline void mi_lock_release(mi_lock_t* lock) {\n pthread_mutex_unlock(&lock->mutex);\n}\nstatic inline void mi_lock_init(mi_lock_t* lock) {\n if(lock==NULL) return;\n // use this instead of pthread_mutex_init since that can cause allocation on some platforms (and recursively initialize)\n const pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;\n memcpy(&lock->mutex,&mutex,sizeof(mutex));\n}\nstatic inline void mi_lock_done(mi_lock_t* lock) {\n pthread_mutex_destroy(&lock->mutex);\n}\n\n#elif defined(__cplusplus)\n\n#include \n#include \n#include \n\ntypedef struct mi_lock_s {\n std::mutex mutex;\n} mi_lock_t;\n\n#define MI_LOCK_INITIALIZER { }\n\nstatic inline bool mi_lock_try_acquire(mi_lock_t* lock) {\n return lock->mutex.try_lock();\n}\nstatic inline void mi_lock_acquire(mi_lock_t* lock) {\n lock->mutex.lock();\n}\nstatic inline void mi_lock_release(mi_lock_t* lock) {\n lock->mutex.unlock();\n}\nstatic inline void mi_lock_init(mi_lock_t* lock) {\n new(&lock->mutex) std::mutex();\n}\nstatic inline void mi_lock_done(mi_lock_t* lock) {\n (void)(lock);\n}\n\n#else\n\n// fall back to poor man's locks.\n// this should only be the case in a single-threaded environment (like __wasi__)\n#include \n#ifndef EFAULT\n#define EFAULT (14)\n#endif\nvoid _mi_error_message(int err, const char* fmt, ...);\nvoid _mi_prim_thread_yield(void);\n\ntypedef struct mi_lock_s {\n _Atomic(uintptr_t) mutex;\n} mi_lock_t;\n\n#define MI_LOCK_INITIALIZER { MI_ATOMIC_VAR_INIT(0) }\n\nstatic inline bool mi_lock_try_acquire(mi_lock_t* lock) {\n uintptr_t expected = 0;\n return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1);\n}\nstatic inline void mi_lock_acquire(mi_lock_t* lock) {\n for (int i = 0; i < 10000; i++) { // for at most 10000 tries?\n if (mi_lock_try_acquire(lock)) return;\n _mi_prim_thread_yield();\n }\n _mi_error_message(EFAULT, \"internal error: lock cannot be acquired (due to lack of native lock primitives)\\n\");\n}\nstatic inline void mi_lock_release(mi_lock_t* lock) {\n mi_atomic_store_release(&lock->mutex, (uintptr_t)0);\n}\nstatic inline void mi_lock_init(mi_lock_t* lock) {\n mi_lock_release(lock);\n}\nstatic inline void mi_lock_done(mi_lock_t* lock) {\n (void)(lock);\n}\n\n#endif\n\n\ntypedef struct mi_atomic_once_s {\n _Atomic(uintptr_t) tid;\n mi_lock_t lock;\n} mi_atomic_once_t;\n\n// Returns `true` only on the first invocation, signifying we can execute an action once.\n// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.\n// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.\nbool _mi_atomic_once_enter(mi_atomic_once_t* once); // defined in `libc.c`\nvoid _mi_atomic_once_release(mi_atomic_once_t* once); // defined in `libc.c`\n\n#define mi_atomic_do_once \\\n static mi_atomic_once_t _mi_once = { MI_ATOMIC_VAR_INIT(0), MI_LOCK_INITIALIZER }; \\\n for(bool _mi_exec = _mi_atomic_once_enter(&_mi_once); _mi_exec; (_mi_atomic_once_release(&_mi_once),_mi_exec=false))\n\n\n#endif // __MIMALLOC_ATOMIC_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/include/mimalloc/bits.h", "sha256": "57a7297e18ae464a7bfd3fc25a46229008602385697305c69ed289b89ece7631", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2024 Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* ----------------------------------------------------------------------------\n Bit operation, and platform dependent definition (MI_INTPTR_SIZE etc)\n---------------------------------------------------------------------------- */\n\n#pragma once\n#ifndef MI_BITS_H\n#define MI_BITS_H\n\n#include // size_t\n#include // int64_t etc\n#include // bool\n\n// ------------------------------------------------------\n// Size of a pointer.\n// We assume that `sizeof(void*)==sizeof(intptr_t)`\n// and it holds for all platforms we know of.\n//\n// However, the C standard only requires that:\n// p == (void*)((intptr_t)p))\n// but we also need:\n// i == (intptr_t)((void*)i)\n// or otherwise one might define an intptr_t type that is larger than a pointer...\n// ------------------------------------------------------\n\n#if INTPTR_MAX > INT64_MAX\n# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example)\n#elif INTPTR_MAX == INT64_MAX\n# define MI_INTPTR_SHIFT (3)\n#elif INTPTR_MAX == INT32_MAX\n# define MI_INTPTR_SHIFT (2)\n#else\n#error platform pointers must be 32, 64, or 128 bits\n#endif\n\n#if (INTPTR_MAX) > LONG_MAX\n# define MI_PU(x) x##ULL\n#else\n# define MI_PU(x) x##UL\n#endif\n\n#if SIZE_MAX == UINT64_MAX\n# define MI_SIZE_SHIFT (3)\ntypedef int64_t mi_ssize_t;\n#elif SIZE_MAX == UINT32_MAX\n# define MI_SIZE_SHIFT (2)\ntypedef int32_t mi_ssize_t;\n#else\n#error platform objects must be 32 or 64 bits in size\n#endif\n\n#if (SIZE_MAX/2) > LONG_MAX\n# define MI_ZU(x) x##ULL\n#else\n# define MI_ZU(x) x##UL\n#endif\n\n#define MI_INTPTR_SIZE (1<\n#elif MI_ARCH_ARM64 && MI_OPT_SIMD\n#include \n#endif\n#if defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)\n#include \n#endif\n\n#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // msvc\n#define __BMI2__ 1\n#endif\n#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc\n#define __BMI1__ 1\n#endif\n#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // msvc\n#define __LZCNT__ 1\n#endif\n\n// Define big endian if needed\n// #define MI_BIG_ENDIAN 1\n\n// maximum virtual address bits in a user-space pointer\n#if MI_DEFAULT_VIRTUAL_ADDRESS_BITS > 0 \n#define MI_MAX_VABITS MI_DEFAULT_VIRTUAL_ADDRESS_BITS\n#elif MI_ARCH_X64\n#define MI_MAX_VABITS (47)\n#elif MI_INTPTR_SIZE > 4\n#define MI_MAX_VABITS (48)\n#else\n#define MI_MAX_VABITS (32)\n#endif\n\n// use a flat page-map or a 2-level one\n#ifndef MI_PAGE_MAP_FLAT\n#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 && !MI_PAGE_META_IS_SEPARATED\n#define MI_PAGE_MAP_FLAT 1\n#else\n#define MI_PAGE_MAP_FLAT 0\n#endif\n#endif\n\n\n/* --------------------------------------------------------------------------------\n Builtin's\n-------------------------------------------------------------------------------- */\n\n#ifndef __has_builtin\n#define __has_builtin(x) 0\n#endif\n\n#define mi_builtin(name) __builtin_##name\n#define mi_has_builtin(name) __has_builtin(__builtin_##name)\n\n#if (LONG_MAX == INT32_MAX)\n#define mi_builtin32(name) mi_builtin(name##l)\n#define mi_has_builtin32(name) mi_has_builtin(name##l)\n#else\n#define mi_builtin32(name) mi_builtin(name)\n#define mi_has_builtin32(name) mi_has_builtin(name)\n#endif\n#if (LONG_MAX == INT64_MAX)\n#define mi_builtin64(name) mi_builtin(name##l)\n#define mi_has_builtin64(name) mi_has_builtin(name##l)\n#else\n#define mi_builtin64(name) mi_builtin(name##ll)\n#define mi_has_builtin64(name) mi_has_builtin(name##ll)\n#endif\n\n#if (MI_SIZE_BITS == 32)\n#define mi_builtinz(name) mi_builtin32(name)\n#define mi_has_builtinz(name) mi_has_builtin32(name)\n#define mi_msc_builtinz(name) name\n#elif (MI_SIZE_BITS == 64)\n#define mi_builtinz(name) mi_builtin64(name)\n#define mi_has_builtinz(name) mi_has_builtin64(name)\n#define mi_msc_builtinz(name) name##64\n#endif\n\n/* --------------------------------------------------------------------------------\n Popcount and count trailing/leading zero's\n-------------------------------------------------------------------------------- */\n\nsize_t _mi_popcount_generic(size_t x);\n\nstatic inline size_t mi_popcount(size_t x) {\n #if mi_has_builtinz(popcount)\n return mi_builtinz(popcount)(x);\n #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)\n return mi_msc_builtinz(__popcnt)(x);\n #elif MI_ARCH_X64 && defined(__BMI1__)\n return (size_t)_mm_popcnt_u64(x);\n #else\n #define MI_HAS_FAST_POPCOUNT 0\n return (x<=1 ? x : _mi_popcount_generic(x));\n #endif\n}\n\n#ifndef MI_HAS_FAST_POPCOUNT\n#define MI_HAS_FAST_POPCOUNT 1\n#endif\n\n\n\nsize_t _mi_clz_generic(size_t x);\nsize_t _mi_ctz_generic(size_t x);\n\nstatic inline size_t mi_ctz(size_t x) {\n #if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) // on x64 tzcnt is defined for 0\n size_t r;\n __asm (\"tzcnt\\t%1, %0\" : \"=r\"(r) : \"r\"(x) : \"cc\");\n return r;\n #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__BMI1__) \n return _tzcnt_u64(x);\n #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)\n unsigned long idx;\n return (mi_msc_builtinz(_BitScanForward)(&idx, x) ? (size_t)idx : MI_SIZE_BITS);\n #elif mi_has_builtinz(ctz)\n return (x!=0 ? (size_t)mi_builtinz(ctz)(x) : MI_SIZE_BITS);\n #elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86)\n size_t r = MI_SIZE_BITS; // bsf leaves destination unmodified if the argument is 0 (see )\n __asm (\"bsf\\t%1, %0\" : \"+r\"(r) : \"r\"(x) : \"cc\");\n return r;\n #elif MI_HAS_FAST_POPCOUNT\n return (x!=0 ? (mi_popcount(x^(x-1))-1) : MI_SIZE_BITS);\n #else\n #define MI_HAS_FAST_BITSCAN 0\n return (x!=0 ? _mi_ctz_generic(x) : MI_SIZE_BITS);\n #endif\n}\n\nstatic inline size_t mi_clz(size_t x) {\n #if defined(__GNUC__) && MI_ARCH_X64 && defined(__LZCNT__) // on x64 lzcnt is defined for 0\n size_t r;\n __asm (\"lzcnt\\t%1, %0\" : \"=r\"(r) : \"r\"(x) : \"cc\");\n return r;\n #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__LZCNT__) \n return _lzcnt_u64(x);\n #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)\n unsigned long idx;\n return (mi_msc_builtinz(_BitScanReverse)(&idx, x) ? MI_SIZE_BITS - 1 - (size_t)idx : MI_SIZE_BITS);\n #elif mi_has_builtinz(clz)\n return (x!=0 ? (size_t)mi_builtinz(clz)(x) : MI_SIZE_BITS);\n #elif defined(__GNUC__) && (MI_ARCH_X64 || MI_ARCH_X86)\n if (x==0) return MI_SIZE_BITS;\n size_t r;\n __asm (\"bsr\\t%1, %0\" : \"=r\"(r) : \"r\"(x) : \"cc\");\n return (MI_SIZE_BITS - 1 - r);\n #else\n #define MI_HAS_FAST_BITSCAN 0\n return (x!=0 ? _mi_clz_generic(x) : MI_SIZE_BITS);\n #endif\n}\n\n#ifndef MI_HAS_FAST_BITSCAN\n#define MI_HAS_FAST_BITSCAN 1\n#endif\n\n/* --------------------------------------------------------------------------------\n find trailing/leading zero (bit scan forward/reverse)\n-------------------------------------------------------------------------------- */\n\n// Bit scan forward: find the least significant bit that is set (i.e. count trailing zero's)\n// return false if `x==0` (with `*idx` undefined) and true otherwise,\n// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).\nstatic inline bool mi_bsf(size_t x, size_t* idx) {\n #if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) && (!defined(__clang_major__) || __clang_major__ >= 9)\n // on x64 the carry flag is set on zero which gives better codegen\n bool is_zero;\n __asm ( \"tzcnt\\t%2, %1\" : \"=@ccc\"(is_zero), \"=r\"(*idx) : \"r\"(x) : \"cc\" );\n return !is_zero;\n #elif 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)\n unsigned long i;\n return (mi_msc_builtinz(_BitScanForward)(&i, x) ? (*idx = (size_t)i, true) : false);\n #else\n return (x!=0 ? (*idx = mi_ctz(x), true) : false);\n #endif\n}\n\n// Bit scan reverse: find the most significant bit that is set\n// return false if `x==0` (with `*idx` undefined) and true otherwise,\n// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).\nstatic inline bool mi_bsr(size_t x, size_t* idx) {\n #if 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)\n unsigned long i;\n return (mi_msc_builtinz(_BitScanReverse)(&i, x) ? (*idx = (size_t)i, true) : false);\n #else\n return (x!=0 ? (*idx = MI_SIZE_BITS - 1 - mi_clz(x), true) : false);\n #endif\n}\n\n\n/* --------------------------------------------------------------------------------\n rotate\n-------------------------------------------------------------------------------- */\n\nstatic inline size_t mi_rotr(size_t x, size_t r) {\n #if (mi_has_builtin(rotateright64) && MI_SIZE_BITS==64)\n return mi_builtin(rotateright64)(x,r);\n #elif (mi_has_builtin(rotateright32) && MI_SIZE_BITS==32)\n return mi_builtin(rotateright32)(x,r);\n #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64)\n return _rotr64(x, (int)r);\n #elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32)\n return _lrotr(x,(int)r);\n #else\n // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to\n // avoid UB when `rshift==0`. See \n const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);\n return ((x >> rshift) | (x << ((-rshift) & (MI_SIZE_BITS-1))));\n #endif\n}\n\nstatic inline size_t mi_rotl(size_t x, size_t r) {\n #if (mi_has_builtin(rotateleft64) && MI_SIZE_BITS==64)\n return mi_builtin(rotateleft64)(x,r);\n #elif (mi_has_builtin(rotateleft32) && MI_SIZE_BITS==32)\n return mi_builtin(rotateleft32)(x,r);\n #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_ARM64)\n return _rotl64(x, (int)r);\n #elif defined(_MSC_VER) && (MI_ARCH_X86 || MI_ARCH_ARM32)\n return _lrotl(x, (int)r);\n #else\n // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to\n // avoid UB when `rshift==0`. See \n const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);\n return ((x << rshift) | (x >> ((-rshift) & (MI_SIZE_BITS-1))));\n #endif\n}\n\nstatic inline uint32_t mi_rotl32(uint32_t x, uint32_t r) {\n #if mi_has_builtin(rotateleft32)\n return mi_builtin(rotateleft32)(x,r);\n #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)\n return _lrotl(x, (int)r);\n #else\n // The term `(-rshift)&(BITS-1)` is written instead of `BITS - rshift` to\n // avoid UB when `rshift==0`. See \n const unsigned int rshift = (unsigned int)(r) & 31;\n return ((x << rshift) | (x >> ((-rshift) & 31)));\n #endif\n}\n\n\n#endif // MI_BITS_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/include/mimalloc/internal.h", "sha256": "5aeeff21d667daf4aadcc66f634aa957ecd688d15b277828f367c8c2274251a8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MI_INTERNAL_H\n#define MI_INTERNAL_H\n\n// --------------------------------------------------------------------------\n// This file contains the internal API's of mimalloc and various utility\n// functions and macros.\n// --------------------------------------------------------------------------\n\n#include \"types.h\"\n#include \"track.h\"\n#include \"bits.h\"\n\n\n// --------------------------------------------------------------------------\n// Compiler defines\n// --------------------------------------------------------------------------\n\n#if (MI_DEBUG>0)\n#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__)\n#else\n#define mi_trace_message(...)\n#endif\n\n#define mi_decl_cache_align mi_decl_align(64)\n\n#if defined(_MSC_VER)\n#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths)\n#pragma warning(disable:26812) // unscoped enum warning\n#define mi_decl_forceinline __forceinline\n#define mi_decl_noinline __declspec(noinline)\n#define mi_decl_thread __declspec(thread)\n#define mi_decl_noreturn __declspec(noreturn)\n#define mi_decl_weak\n#define mi_decl_hidden\n#define mi_decl_cold\n#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc\n#if !MI_TRACK_ASAN\n#define mi_decl_forceinline __attribute__((always_inline)) inline\n#else\n#define mi_decl_forceinline inline\n#endif\n#define mi_decl_noinline __attribute__((noinline))\n#define mi_decl_thread __thread\n#define mi_decl_noreturn __attribute__((noreturn))\n#define mi_decl_weak __attribute__((weak))\n#define mi_decl_hidden __attribute__((visibility(\"hidden\")))\n#if (__GNUC__ >= 4) || defined(__clang__)\n#define mi_decl_cold __attribute__((cold))\n#else\n#define mi_decl_cold\n#endif\n#elif __cplusplus >= 201103L // c++11\n#define mi_decl_forceinline inline\n#define mi_decl_noinline\n#define mi_decl_thread thread_local\n#define mi_decl_noreturn [[noreturn]]\n#define mi_decl_weak\n#define mi_decl_hidden\n#define mi_decl_cold\n#else\n#define mi_decl_forceinline inline\n#define mi_decl_noinline\n#define mi_decl_thread __thread // hope for the best :-)\n#define mi_decl_noreturn\n#define mi_decl_weak\n#define mi_decl_hidden\n#define mi_decl_cold\n#endif\n\n#if defined(__GNUC__) || defined(__clang__)\n#define mi_unlikely(x) (__builtin_expect(!!(x),false))\n#define mi_likely(x) (__builtin_expect(!!(x),true))\n#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L)\n#define mi_unlikely(x) (x) [[unlikely]]\n#define mi_likely(x) (x) [[likely]]\n#else\n#define mi_unlikely(x) (x)\n#define mi_likely(x) (x)\n#endif\n\n#ifndef __has_builtin\n#define __has_builtin(x) 0\n#endif\n\n#if defined(__cplusplus)\n#define mi_decl_externc extern \"C\"\n#else\n#define mi_decl_externc\n#endif\n\n#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc\n#define mi_decl_maybe_unused __attribute__((unused))\n#elif __cplusplus >= 201703L // c++17\n#define mi_decl_maybe_unused [[maybe_unused]]\n#else\n#define mi_decl_maybe_unused\n#endif\n\n#if defined(__cplusplus)\n#define mi_decl_externc extern \"C\"\n#else\n#define mi_decl_externc\n#endif\n\n\n#if defined(__EMSCRIPTEN__) && !defined(__wasi__)\n#define __wasi__\n#endif\n\n\n// --------------------------------------------------------------------------\n// Internal functions\n// --------------------------------------------------------------------------\n\n\n// \"libc.c\"\n#include \nint _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args);\nint _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...);\nchar _mi_toupper(char c);\nint _mi_strnicmp(const char* s, const char* t, size_t n);\nvoid _mi_strlcpy(char* dest, const char* src, size_t dest_size);\nvoid _mi_strlcat(char* dest, const char* src, size_t dest_size);\nsize_t _mi_strlen(const char* s);\nsize_t _mi_strnlen(const char* s, size_t max_len);\nchar* _mi_strnstr(char* s, size_t max_len, const char* pat);\nbool _mi_streq(const char* s, const char* t);\nbool _mi_getenv(const char* name, char* result, size_t result_size);\n\n// \"options.c\"\nvoid _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);\nvoid _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);\nvoid _mi_raw_message(const char* fmt, ...);\nvoid _mi_message(const char* fmt, ...);\nvoid _mi_warning_message(const char* fmt, ...);\nvoid _mi_verbose_message(const char* fmt, ...);\nvoid _mi_trace_message(const char* fmt, ...);\nvoid _mi_options_init(void);\nvoid _mi_options_post_init(void);\nlong _mi_option_get_fast(mi_option_t option);\nvoid _mi_error_message(int err, const char* fmt, ...);\n\n// random.c\nvoid _mi_random_init(mi_random_ctx_t* ctx);\nvoid _mi_random_init_weak(mi_random_ctx_t* ctx);\nvoid _mi_random_reinit_if_weak(mi_random_ctx_t * ctx);\nvoid _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx);\nuintptr_t _mi_random_next(mi_random_ctx_t* ctx);\nuintptr_t _mi_theap_random_next(mi_theap_t* theap);\nuintptr_t _mi_os_random_weak(uintptr_t extra_seed);\nstatic inline uintptr_t _mi_random_shuffle(uintptr_t x);\n\n// init.c\nextern mi_decl_hidden mi_decl_cache_align const mi_page_t _mi_page_empty;\nvoid _mi_auto_process_init(void);\nvoid mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept;\nbool _mi_is_redirected(void);\nbool _mi_allocator_init(const char** message);\nvoid _mi_allocator_done(void);\nbool _mi_is_main_thread(void);\nbool _mi_preloading(void); // true while the C runtime is not initialized yet\nvoid _mi_thread_done(mi_theap_t* theap);\n\nmi_subproc_t* _mi_subproc(void);\nmi_subproc_t* _mi_subproc_main(void);\nmi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc);\nmi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);\n\nmi_threadid_t _mi_thread_id(void) mi_attr_noexcept;\nsize_t _mi_thread_seq_id(void) mi_attr_noexcept;\nbool _mi_is_heap_main(const mi_heap_t* heap);\nbool _mi_is_theap_main(const mi_theap_t* theap);\nvoid _mi_theap_guarded_init(mi_theap_t* theap);\nvoid _mi_theap_options_init(mi_theap_t* theap);\nmi_theap_t* _mi_theap_default_safe(void); // ensure the returned theap is initialized\nmi_theap_t* _mi_theap_main_safe(void);\n \n// os.c\nvoid _mi_os_init(void); // called from process init\nvoid* _mi_os_alloc(size_t size, mi_memid_t* memid);\nvoid* _mi_os_zalloc(size_t size, mi_memid_t* memid);\nvoid _mi_os_free(void* p, size_t size, mi_memid_t memid);\nvoid _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc );\n\nsize_t _mi_os_page_size(void);\nsize_t _mi_os_guard_page_size(void);\nsize_t _mi_os_good_alloc_size(size_t size);\nbool _mi_os_has_overcommit(void);\nbool _mi_os_has_virtual_reserve(void);\nsize_t _mi_os_virtual_address_bits(void);\nsize_t _mi_os_minimal_purge_size(void);\n\nbool _mi_os_reset(void* addr, size_t size);\nbool _mi_os_decommit(void* addr, size_t size);\nvoid _mi_os_reuse(void* p, size_t size);\nmi_decl_nodiscard bool _mi_os_commit(void* p, size_t size, bool* is_zero);\nmi_decl_nodiscard bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size);\nmi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size);\nbool _mi_os_unprotect(void* addr, size_t size);\nbool _mi_os_purge(void* p, size_t size);\nbool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg);\n\nsize_t _mi_os_secure_guard_page_size(void);\nbool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid);\nbool _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid);\nbool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid);\nbool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid);\n\nint _mi_os_numa_node(void);\nint _mi_os_numa_node_count(void);\n\nvoid* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid);\nvoid* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid);\n\nvoid* _mi_os_get_aligned_hint(size_t try_alignment, size_t size);\nbool _mi_os_canuse_large_page(size_t size, size_t alignment);\nsize_t _mi_os_large_page_size(void);\nvoid* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);\n\n// threadlocal.c\n\nmi_thread_local_t _mi_thread_local_create(void);\nvoid _mi_thread_local_free( mi_thread_local_t key );\nbool _mi_thread_local_set( mi_thread_local_t key, void* val );\nvoid* _mi_thread_local_get( mi_thread_local_t key );\nvoid _mi_thread_locals_init(void);\nvoid _mi_thread_locals_done(void);\nvoid _mi_thread_locals_thread_done(void);\n\n// arena.c\nmi_arena_id_t _mi_arena_id_none(void);\nmi_arena_t* _mi_arena_from_id(mi_arena_id_t id);\nbool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena);\n\nvoid* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid);\nvoid* _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid);\nvoid _mi_arenas_free(void* p, size_t size, mi_memid_t memid);\nbool _mi_arenas_contain(const void* p);\nvoid _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld);\nvoid _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc);\n\nmi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t page_alignment);\nvoid _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */);\nvoid _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap);\nvoid _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */);\nbool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page);\n\n// arena-meta.c\nvoid* _mi_meta_zalloc( size_t size, mi_memid_t* memid );\nvoid _mi_meta_free(void* p, size_t size, mi_memid_t memid);\nbool _mi_meta_is_meta_page(void* p);\n\n// \"page-map.c\"\nbool _mi_page_map_init(void);\nmi_decl_nodiscard bool _mi_page_map_register(mi_page_t* page);\nvoid _mi_page_map_unregister(mi_page_t* page);\nvoid _mi_page_map_unregister_range(void* start, size_t size);\nmi_page_t* _mi_safe_ptr_page(const void* p);\nvoid _mi_page_map_unsafe_destroy(mi_subproc_t* subproc);\n\n// \"page.c\"\nvoid* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept mi_attr_malloc;\n\nvoid _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks\nvoid _mi_page_unfull(mi_page_t* page);\nvoid _mi_page_free(mi_page_t* page, mi_page_queue_t* pq); // free the page\nvoid _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread...\n\nsize_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append);\nvoid _mi_deferred_free(mi_theap_t* theap, bool force);\n\nvoid _mi_page_free_collect(mi_page_t* page, bool force);\nvoid _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head);\nmi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page);\nbool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq);\n\nsize_t _mi_page_stats_bin(const mi_page_t* page); // for stats\nsize_t _mi_bin_size(size_t bin); // for stats\nsize_t _mi_bin(size_t size); // for stats\n\n// \"theap.c\"\nmi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld);\nvoid _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock);\nvoid _mi_theap_default_set(mi_theap_t* theap);\nvoid _mi_theap_cached_set(mi_theap_t* theap);\nvoid _mi_theap_collect_retired(mi_theap_t* theap, bool force);\nvoid _mi_theap_collect_abandon(mi_theap_t* theap);\nbool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg);\nvoid _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page);\nbool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock);\nvoid _mi_theap_incref(mi_theap_t* theap);\nvoid _mi_theap_decref(mi_theap_t* theap);\n\n// \"heap.c\"\nvoid _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page);\nmi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap); // get (and possible create) the theap belonging to a heap\nmi_decl_cold mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap); // get the theap for a heap without initializing (and return NULL in that case)\nvoid _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to); // in \"arena.c\"\nvoid _mi_heap_destroy_pages(mi_heap_t* heap_from); // in \"arena.c\"\nvoid _mi_heap_force_destroy(mi_heap_t* heap); // allow destroying the main heap\n\n// \"stats.c\"\nvoid _mi_stats_init(void);\nvoid _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from);\n\nmi_msecs_t _mi_clock_now(void);\nmi_msecs_t _mi_clock_end(mi_msecs_t start);\nmi_msecs_t _mi_clock_start(void);\n\n// \"alloc.c\"\nvoid* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned`\nvoid* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept;\nvoid* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned`\nvoid* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept;\nmi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p);\nvoid _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size);\n\n#if MI_DEBUG>1\nbool _mi_page_is_valid(mi_page_t* page);\n#endif\n\n\n// ------------------------------------------------------\n// Assertions\n// ------------------------------------------------------\n\n#if (MI_DEBUG)\n// use our own assertion to print without memory allocation\nmi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_attr_noexcept;\n#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__))\n#else\n#define mi_assert(x)\n#endif\n\n#if (MI_DEBUG>1)\n#define mi_assert_internal mi_assert\n#else\n#define mi_assert_internal(x)\n#endif\n\n#if (MI_DEBUG>2)\n#define mi_assert_expensive mi_assert\n#else\n#define mi_assert_expensive(x)\n#endif\n\n\n/* -----------------------------------------------------------\n Statistics (in `stats.c`)\n----------------------------------------------------------- */\n\n// add to stat keeping track of the peak\nvoid __mi_stat_increase(mi_stat_count_t* stat, size_t amount);\nvoid __mi_stat_decrease(mi_stat_count_t* stat, size_t amount);\nvoid __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount);\nvoid __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount);\n\n// adjust stat in special cases to compensate for double counting (and does not adjust peak values and can decrease the total)\nvoid __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount);\nvoid __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount);\nvoid __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount);\nvoid __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount);\n\n// counters can just be increased\nvoid __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);\nvoid __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);\n\n#define mi_heap_stat_counter_increase(heap,stat,amount) __mi_stat_counter_increase_mt( &(heap)->stats.stat, amount)\n#define mi_heap_stat_increase(heap,stat,amount) __mi_stat_increase_mt( &(heap)->stats.stat, amount)\n#define mi_heap_stat_decrease(heap,stat,amount) __mi_stat_decrease_mt( &(heap)->stats.stat, amount)\n#define mi_heap_stat_adjust_increase(heap,stat,amnt) __mi_stat_adjust_increase_mt( &(heap)->stats.stat, amnt)\n#define mi_heap_stat_adjust_decrease(heap,stat,amnt) __mi_stat_adjust_decrease_mt( &(heap)->stats.stat, amnt)\n\n#define mi_subproc_stat_counter_increase(subproc,stat,amount) __mi_stat_counter_increase_mt( &(subproc)->stats.stat, amount)\n#define mi_subproc_stat_increase(subproc,stat,amount) __mi_stat_increase_mt( &(subproc)->stats.stat, amount)\n#define mi_subproc_stat_decrease(subproc,stat,amount) __mi_stat_decrease_mt( &(subproc)->stats.stat, amount)\n#define mi_subproc_stat_adjust_increase(subproc,stat,amount) __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amount)\n#define mi_subproc_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount)\n\n#define mi_os_stat_counter_increase(stat,amount) mi_subproc_stat_counter_increase(_mi_subproc(),stat,amount)\n#define mi_os_stat_increase(stat,amount) mi_subproc_stat_increase(_mi_subproc(),stat,amount)\n#define mi_os_stat_decrease(stat,amount) mi_subproc_stat_decrease(_mi_subproc(),stat,amount)\n\n#define mi_theap_stat_counter_increase(theap,stat,amount) __mi_stat_counter_increase( &(theap)->stats.stat, amount)\n#define mi_theap_stat_increase(theap,stat,amount) __mi_stat_increase( &(theap)->stats.stat, amount)\n#define mi_theap_stat_decrease(theap,stat,amount) __mi_stat_decrease( &(theap)->stats.stat, amount)\n#define mi_theap_stat_adjust_increase(theap,stat,amnt) __mi_stat_adjust_increase( &(theap)->stats.stat, amnt)\n#define mi_theap_stat_adjust_decrease(theap,stat,amnt) __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt)\n\n\n/* -----------------------------------------------------------\n Options (exposed for the debugger)\n----------------------------------------------------------- */\ntypedef enum mi_option_init_e {\n MI_OPTION_UNINIT, // not yet initialized\n MI_OPTION_DEFAULTED, // not found in the environment, use default value\n MI_OPTION_INITIALIZED // found in environment or set explicitly\n} mi_option_init_t;\n\ntypedef struct mi_option_desc_s {\n long value; // the value\n mi_option_init_t init; // is it initialized yet? (from the environment)\n mi_option_t option; // for debugging: the option index should match the option\n const char* name; // option name without `mimalloc_` prefix\n const char* legacy_name; // potential legacy option name\n} mi_option_desc_t;\n\n\n\n/* -----------------------------------------------------------\n Inlined definitions\n----------------------------------------------------------- */\n#define MI_UNUSED(x) (void)(x)\n#if (MI_DEBUG>1)\n#define MI_UNUSED_RELEASE(x)\n#else\n#define MI_UNUSED_RELEASE(x) MI_UNUSED(x)\n#endif\n\n#define MI_INIT4(x) x(),x(),x(),x()\n#define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x)\n#define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x)\n#define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x)\n#define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x)\n#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x)\n#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x)\n\n#define MI_INIT74(x) MI_INIT64(x),MI_INIT8(x),x(),x()\n#define MI_INIT5(x) MI_INIT4(x),x()\n#define MI_INIT6(x) MI_INIT4(x),x(),x()\n\n#include \n// initialize a local variable to zero; use memset as compilers optimize constant sized memset's\n#define _mi_memzero_var(x) memset(&x,0,sizeof(x))\n\n// Is `x` a power of two? (0 is considered a power of two)\nstatic inline bool _mi_is_power_of_two(uintptr_t x) {\n return ((x & (x - 1)) == 0);\n}\n\n// Is a pointer aligned?\nstatic inline bool _mi_is_aligned(const void* p, size_t alignment) {\n return (alignment==0 || ((uintptr_t)p % alignment) == 0);\n}\n\n// Align upwards\nstatic inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {\n mi_assert_internal(alignment != 0);\n uintptr_t mask = alignment - 1;\n if ((alignment & mask) == 0) { // power of two?\n return ((sz + mask) & ~mask);\n }\n else {\n return (((sz + mask)/alignment)*alignment);\n }\n}\n\n// Align a pointer upwards\nstatic inline void* _mi_align_up_ptr(const void* p, size_t alignment) {\n return (void*)_mi_align_up((uintptr_t)p, alignment);\n}\n\nstatic inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {\n mi_assert_internal(alignment != 0);\n uintptr_t mask = alignment - 1;\n if ((alignment & mask) == 0) { // power of two?\n return (sz & ~mask);\n }\n else {\n return ((sz / alignment) * alignment);\n }\n}\n\n// align a pointer downwards\nstatic inline void* _mi_align_down_ptr(const void* p, size_t alignment) {\n return (void*)_mi_align_down((uintptr_t)p, alignment);\n}\n\n// Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`.\nstatic inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) {\n mi_assert_internal(divider != 0);\n return (divider == 0 ? size : ((size + divider - 1) / divider));\n}\n\n\n// clamp an integer\nstatic inline size_t _mi_clamp(size_t sz, size_t min, size_t max) {\n if (sz < min) return min;\n else if (sz > max) return max;\n else return sz;\n}\n\n// Is memory zero initialized?\nstatic inline bool mi_mem_is_zero(const void* p, size_t size) {\n for (size_t i = 0; i < size; i++) {\n if (((uint8_t*)p)[i] != 0) return false;\n }\n return true;\n}\n\n// Align a byte size to a size in _machine words_,\n// i.e. byte size == `wsize*sizeof(void*)`.\nstatic inline size_t _mi_wsize_from_size(size_t size) {\n mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t));\n return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);\n}\n\n// Overflow detecting multiply\n#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5))\n#include // UINT_MAX, ULONG_MAX\n#if defined(_CLOCK_T) // for Illumos\n#undef _CLOCK_T\n#endif\nstatic inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {\n #if (SIZE_MAX == ULONG_MAX)\n return __builtin_umull_overflow(count, size, (unsigned long *)total);\n #elif (SIZE_MAX == UINT_MAX)\n return __builtin_umul_overflow(count, size, (unsigned int *)total);\n #else\n return __builtin_umulll_overflow(count, size, (unsigned long long *)total);\n #endif\n}\n#else /* __builtin_umul_overflow is unavailable */\nstatic inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {\n *total = count*size;\n if mi_likely(((size|count)>>(4*MI_SIZE_SIZE))==0) { // did size and count fit both in the lower half bits of a size_t?\n return false;\n }\n else {\n return (size!=0 && (SIZE_MAX / size) < count);\n }\n}\n#endif\n\n// Safe multiply `count*size` into `total`; return `true` on overflow.\nstatic inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) {\n if (count==1) { // quick check for the case where count is one (common for C++ allocators)\n *total = size;\n return false;\n }\n else if mi_likely(!mi_mul_overflow(count, size, total)) {\n return false;\n }\n else {\n #if MI_DEBUG > 0\n _mi_error_message(EOVERFLOW, \"allocation request is too large (%zu * %zu bytes)\\n\", count, size);\n #endif\n *total = SIZE_MAX;\n return true;\n }\n}\n\n\n/*----------------------------------------------------------------------------------------\n Heap functions\n------------------------------------------------------------------------------------------- */\n\nextern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_THREAD_LOCAL)\nextern mi_decl_hidden const mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated\n\n\nstatic inline mi_heap_t* _mi_theap_heap(const mi_theap_t* theap) {\n return mi_atomic_load_ptr_acquire(mi_heap_t,&theap->heap);\n}\n\nstatic inline bool mi_theap_is_initialized(const mi_theap_t* theap) {\n return (theap != NULL && _mi_theap_heap(theap) != NULL);\n}\n\nstatic inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) {\n mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));\n const size_t idx = _mi_wsize_from_size(size);\n mi_assert_internal(idx < MI_PAGES_DIRECT);\n return theap->pages_free_direct[idx];\n}\n\n\n//static inline uintptr_t _mi_ptr_cookie(const void* p) {\n// extern mi_theap_t _mi_theap_main;\n// mi_assert_internal(_mi_theap_main.cookie != 0);\n// return ((uintptr_t)p ^ _mi_theap_main.cookie);\n//}\n\n\n/* -----------------------------------------------------------\n The page map maps addresses to `mi_page_t` pointers\n----------------------------------------------------------- */\n\n#if MI_PAGE_MAP_FLAT\n\n// flat page-map committed on demand, using one byte per slice (64 KiB).\n// single indirection and low commit, but large initial virtual reserve (4 GiB with 48 bit virtual addresses)\n// used by default on <= 40 bit virtual address spaces.\nextern mi_decl_hidden uint8_t* _mi_page_map;\n\nstatic inline size_t _mi_page_map_index(const void* p) {\n return (size_t)((uintptr_t)p >> MI_ARENA_SLICE_SHIFT);\n}\n\nstatic inline mi_page_t* _mi_ptr_page_ex(const void* p, bool* valid) {\n const size_t idx = _mi_page_map_index(p);\n const size_t ofs = _mi_page_map[idx];\n if (valid != NULL) { *valid = (ofs != 0); }\n return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) + 1 - ofs) << MI_ARENA_SLICE_SHIFT);\n}\n\nstatic inline mi_page_t* _mi_checked_ptr_page(const void* p) {\n bool valid;\n mi_page_t* const page = _mi_ptr_page_ex(p, &valid);\n return (valid ? page : NULL);\n}\n\nstatic inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {\n return _mi_ptr_page_ex(p, NULL);\n}\n\n#else\n\n// 2-level page map:\n// double indirection, but low commit and low virtual reserve.\n//\n// the page-map is usually 4 MiB (for 48 bit virtual addresses) and points to sub maps of 64 KiB.\n// the page-map is committed on-demand (in 64 KiB parts) (and sub-maps are committed on-demand as well)\n// one sub page-map = 64 KiB => covers 2^(16-3) * 2^16 = 2^29 = 512 MiB address space\n// the page-map needs 48-(16+13) = 19 bits => 2^19 sub map pointers = 2^22 bytes = 4 MiB reserved size.\n#define MI_PAGE_MAP_SUB_SHIFT (13)\n#define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT)\n#define MI_PAGE_MAP_SHIFT (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)\n#define MI_PAGE_MAP_COUNT (MI_ZU(1) << MI_PAGE_MAP_SHIFT)\n\ntypedef mi_page_t** mi_submap_t;\nextern mi_decl_hidden _Atomic(mi_submap_t)* _mi_page_map;\n\nstatic inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) {\n const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE);\n if (sub_idx != NULL) { *sub_idx = u % MI_PAGE_MAP_SUB_COUNT; }\n return (u / MI_PAGE_MAP_SUB_COUNT);\n}\n\nstatic inline mi_submap_t _mi_page_map_at(size_t idx) {\n return mi_atomic_load_ptr_relaxed(mi_page_t*, &_mi_page_map[idx]);\n}\n\nstatic inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {\n size_t sub_idx;\n const size_t idx = _mi_page_map_index(p, &sub_idx);\n return (_mi_page_map_at(idx))[sub_idx]; // NULL if p==NULL\n}\n\nstatic inline mi_page_t* _mi_checked_ptr_page(const void* p) {\n size_t sub_idx;\n const size_t idx = _mi_page_map_index(p, &sub_idx);\n mi_submap_t const sub = _mi_page_map_at(idx);\n if mi_unlikely(sub == NULL) return NULL;\n return sub[sub_idx];\n}\n\n#endif\n\n\nstatic inline mi_page_t* _mi_ptr_page(const void* p) {\n mi_assert_internal(p==NULL || mi_is_in_heap_region(p));\n #if MI_DEBUG || MI_SECURE || defined(__APPLE__)\n return _mi_checked_ptr_page(p);\n #else\n return _mi_unchecked_ptr_page(p);\n #endif\n}\n\n\n// Get the block size of a page\nstatic inline size_t mi_page_block_size(const mi_page_t* page) {\n mi_assert_internal(page->block_size > 0);\n return page->block_size;\n}\n\n// Page start\nstatic inline uint8_t* mi_page_start(const mi_page_t* page) {\n return page->page_start;\n}\n\nstatic inline size_t mi_page_size(const mi_page_t* page) {\n return mi_page_block_size(page) * page->reserved;\n}\n\nstatic inline uint8_t* mi_page_area(const mi_page_t* page, size_t* size) {\n if (size) { *size = mi_page_size(page); }\n return mi_page_start(page);\n}\n\nstatic inline size_t mi_page_info_size(void) {\n return _mi_align_up(sizeof(mi_page_t), MI_MAX_ALIGN_SIZE);\n}\n\nstatic inline bool mi_page_contains_address(const mi_page_t* page, const void* p) {\n size_t psize;\n uint8_t* start = mi_page_area(page, &psize);\n return (start <= (uint8_t*)p && (uint8_t*)p < start + psize);\n}\n\nstatic inline bool mi_page_is_in_arena(const mi_page_t* page) {\n return (page->memid.memkind == MI_MEM_ARENA);\n}\n\nstatic inline bool mi_page_is_singleton(const mi_page_t* page) {\n return (page->reserved == 1);\n}\n\n// Get the usable block size of a page without fixed padding.\n// This may still include internal padding due to alignment and rounding up size classes.\nstatic inline size_t mi_page_usable_block_size(const mi_page_t* page) {\n return mi_page_block_size(page) - MI_PADDING_SIZE;\n}\n\nstatic inline bool mi_page_meta_is_separated(const mi_page_t* page) {\n #if MI_PAGE_META_IS_SEPARATED\n // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_ALIGNED_FREE_SMALL\n return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN));\n #else\n MI_UNUSED(page);\n return false; \n #endif\n}\n\nstatic inline uint8_t* mi_page_slice_start(const mi_page_t* page) {\n if (mi_page_meta_is_separated(page)) { \n // page meta info is at a separate location (at `arena->pages`)\n return (uint8_t*)_mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN);\n }\n else {\n // page meta info is at the start of the page slices\n return (uint8_t*)page;\n }\n}\n\n// This gives the offset relative to the start slice of a page. \nstatic inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offset_relative_to_page_start) {\n return (page->page_start - mi_page_slice_start(page)) + offset_relative_to_page_start;\n}\n\n// Currently committed part of a page\nstatic inline size_t mi_page_committed(const mi_page_t* page) {\n return (page->slice_committed == 0 ? mi_page_size(page) : page->slice_committed - mi_page_slice_offset_of(page,0));\n}\n\n// are all blocks in a page freed?\n// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`.\nstatic inline bool mi_page_all_free(const mi_page_t* page) {\n mi_assert_internal(page != NULL);\n return (page->used == 0);\n}\n\n// are there immediately available blocks, i.e. blocks available on the free list.\nstatic inline bool mi_page_immediate_available(const mi_page_t* page) {\n mi_assert_internal(page != NULL);\n return (page->free != NULL);\n}\n\n\n// is the page not yet used up to its reserved space?\nstatic inline bool mi_page_is_expandable(const mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_assert_internal(page->capacity <= page->reserved);\n return (page->capacity < page->reserved);\n}\n\n\nstatic inline bool mi_page_is_full(const mi_page_t* page) {\n const bool full = (page->reserved == page->used);\n mi_assert_internal(!full || page->free == NULL);\n return full;\n}\n\n// is more than 7/8th of a page in use?\nstatic inline bool mi_page_is_mostly_used(const mi_page_t* page) {\n if (page==NULL) return true;\n uint16_t frac = page->reserved / 8U;\n return (page->reserved - page->used <= frac);\n}\n\n// is more than (n-1)/n'th of a page in use?\nstatic inline bool mi_page_is_used_at_frac(const mi_page_t* page, uint16_t n) {\n if (page==NULL) return true;\n uint16_t frac = page->reserved / n;\n return (page->reserved - page->used <= frac);\n}\n\n\nstatic inline bool mi_page_is_huge(const mi_page_t* page) {\n return (mi_page_is_singleton(page) &&\n (page->block_size > MI_LARGE_MAX_OBJ_SIZE ||\n (mi_memkind_is_os(page->memid.memkind) && page->memid.mem.os.base < (void*)page)));\n}\n\nstatic inline mi_page_queue_t* mi_page_queue(const mi_theap_t* theap, size_t size) {\n mi_page_queue_t* const pq = &((mi_theap_t*)theap)->pages[_mi_bin(size)];\n if (size <= MI_LARGE_MAX_OBJ_SIZE) { mi_assert_internal(pq->block_size <= MI_LARGE_MAX_OBJ_SIZE); }\n return pq;\n}\n\n\n//-----------------------------------------------------------\n// Page thread id and flags\n//-----------------------------------------------------------\n\n// Thread id of thread that owns this page (with flags in the bottom 2 bits)\nstatic inline mi_threadid_t mi_page_xthread_id(const mi_page_t* page) {\n return mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id);\n}\n\n// Plain thread id of the thread that owns this page\nstatic inline mi_threadid_t mi_page_thread_id(const mi_page_t* page) {\n return (mi_page_xthread_id(page) & ~MI_PAGE_FLAG_MASK);\n}\n\nstatic inline mi_page_flags_t mi_page_flags(const mi_page_t* page) {\n return (mi_page_xthread_id(page) & MI_PAGE_FLAG_MASK);\n}\n\nstatic inline bool mi_page_flags_set(mi_page_t* page, bool set, mi_page_flags_t newflag) {\n mi_page_flags_t old;\n if (set) { old = mi_atomic_or_relaxed(&page->xthread_id, newflag); }\n else { old = mi_atomic_and_relaxed(&page->xthread_id, ~newflag); }\n return ((old & newflag) == newflag);\n}\n\nstatic inline bool mi_page_is_in_full(const mi_page_t* page) {\n return ((mi_page_flags(page) & MI_PAGE_IN_FULL_QUEUE) != 0);\n}\n\nstatic inline void mi_page_set_in_full(mi_page_t* page, bool in_full) {\n const bool was_in_full = mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE);\n if (was_in_full != in_full) {\n // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220)\n mi_theap_t* const theap = page->theap;\n mi_assert_internal(theap!=NULL);\n if (theap != NULL) {\n const size_t size = page->capacity * mi_page_block_size(page);\n if (in_full) { theap->pages_full_size += size; }\n else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; }\n }\n }\n}\n\nstatic inline bool mi_page_has_interior_pointers(const mi_page_t* page) {\n return ((mi_page_flags(page) & MI_PAGE_HAS_INTERIOR_POINTERS) != 0);\n}\n\nstatic inline void mi_page_set_has_interior_pointers(mi_page_t* page, bool has_aligned) {\n mi_page_flags_set(page, has_aligned, MI_PAGE_HAS_INTERIOR_POINTERS);\n}\n\nstatic inline void mi_page_set_theap(mi_page_t* page, mi_theap_t* theap) {\n // mi_assert_internal(!mi_page_is_in_full(page)); // can happen when destroying pages on theap_destroy\n page->theap = theap;\n const mi_threadid_t tid = (theap == NULL ? MI_THREADID_ABANDONED : theap->tld->thread_id);\n mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0);\n\n // we need to use an atomic cas since a concurrent thread may still set the MI_PAGE_HAS_INTERIOR_POINTERS flag (see `alloc_aligned.c`).\n mi_threadid_t xtid_old = mi_page_xthread_id(page);\n mi_threadid_t xtid;\n do {\n xtid = tid | (xtid_old & MI_PAGE_FLAG_MASK);\n } while (!mi_atomic_cas_weak_release(&page->xthread_id, &xtid_old, xtid));\n}\n\nstatic inline bool mi_page_is_abandoned(const mi_page_t* page) {\n // note: the xtheap field of an abandoned theap is set to the subproc (for fast reclaim-on-free)\n return (mi_page_thread_id(page) <= MI_THREADID_ABANDONED_MAPPED);\n}\n\nstatic inline bool mi_page_is_abandoned_mapped(const mi_page_t* page) {\n return (mi_page_thread_id(page) == MI_THREADID_ABANDONED_MAPPED);\n}\n\nstatic inline void mi_page_set_abandoned_mapped(mi_page_t* page) {\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_atomic_or_relaxed(&page->xthread_id, (mi_threadid_t)MI_THREADID_ABANDONED_MAPPED);\n}\n\nstatic inline void mi_page_clear_abandoned_mapped(mi_page_t* page) {\n mi_assert_internal(mi_page_is_abandoned_mapped(page));\n mi_atomic_and_relaxed(&page->xthread_id, (mi_threadid_t)MI_PAGE_FLAG_MASK);\n}\n\n\nstatic inline mi_theap_t* mi_page_theap(const mi_page_t* page) {\n mi_assert_internal(!mi_page_is_abandoned(page));\n mi_assert_internal(page->theap != NULL);\n return page->theap;\n}\n\nstatic inline mi_tld_t* mi_page_tld(const mi_page_t* page) {\n mi_assert_internal(!mi_page_is_abandoned(page));\n mi_assert_internal(page->theap != NULL);\n return page->theap->tld;\n}\n\n\nstatic inline mi_heap_t* mi_page_heap(const mi_page_t* page) {\n mi_heap_t* heap = page->heap;\n // we use NULL for the main heap to make `_mi_page_get_associated_theap` fast in `free.c:mi_abandoned_page_try_reclaim`.\n if mi_likely(heap==NULL) heap = mi_heap_main();\n mi_assert_internal(heap != NULL);\n return heap;\n}\n\n//-----------------------------------------------------------\n// Thread free list and ownership\n//-----------------------------------------------------------\n\n// Thread free flag helpers\nstatic inline mi_block_t* mi_tf_block(mi_thread_free_t tf) {\n return (mi_block_t*)(tf & ~1);\n}\nstatic inline bool mi_tf_is_owned(mi_thread_free_t tf) {\n return ((tf & 1) == 1);\n}\nstatic inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned) {\n return (mi_thread_free_t)((uintptr_t)block | (owned ? 1 : 0));\n}\n\n// Thread free access\nstatic inline mi_block_t* mi_page_thread_free(const mi_page_t* page) {\n return mi_tf_block(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free));\n}\n\n// are there any available blocks?\nstatic inline bool mi_page_has_any_available(const mi_page_t* page) {\n mi_assert_internal(page != NULL && page->reserved > 0);\n return (page->used < page->reserved || (mi_page_thread_free(page) != NULL));\n}\n\n// Owned?\nstatic inline bool mi_page_is_owned(const mi_page_t* page) {\n return mi_tf_is_owned(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free));\n}\n\n// get ownership; returns true if the page was not owned before.\nstatic inline bool mi_page_claim_ownership(mi_page_t* page) {\n const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, (uintptr_t)1);\n return ((old&1)==0);\n}\n\n\n/* -------------------------------------------------------------------\n Guarded objects\n------------------------------------------------------------------- */\n#if MI_GUARDED\n// we always align guarded pointers in a block at an offset\n// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones\n#define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0))\n#define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED)\n#endif\n\nstatic inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) {\n#if MI_GUARDED\n const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block;\n return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED);\n#else\n MI_UNUSED(block); MI_UNUSED(p);\n return false;\n#endif \n}\n\n#if MI_GUARDED\nstatic inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) {\n // this code is written to result in fast assembly as it is on the hot path for allocation\n const size_t count = theap->guarded_sample_count - 1; // if the rate was 0, this will underflow and count for a long time..\n if mi_likely(count != 0) {\n // no sample\n theap->guarded_sample_count = count;\n return false;\n }\n else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) {\n // use guarded allocation\n theap->guarded_sample_count = theap->guarded_sample_rate; // reset\n return (theap->guarded_sample_rate != 0);\n }\n else {\n // failed size criteria, rewind count (but don't write to an empty theap)\n if (theap->guarded_sample_rate != 0) { theap->guarded_sample_count = 1; }\n return false;\n }\n}\n\nmi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept;\n\n#endif\n\n\n/* -------------------------------------------------------------------\nEncoding/Decoding the free list next pointers\n\nThis is to protect against buffer overflow exploits where the\nfree list is mutated. Many hardened allocators xor the next pointer `p`\nwith a secret key `k1`, as `p^k1`. This prevents overwriting with known\nvalues but might be still too weak: if the attacker can guess\nthe pointer `p` this can reveal `k1` (since `p^k1^p == k1`).\nMoreover, if multiple blocks can be read as well, the attacker can\nxor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot\nabout the pointers (and subsequently `k1`).\n\nInstead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<next, keys);\n #else\n MI_UNUSED(keys); MI_UNUSED(null);\n next = (mi_block_t*)block->next;\n #endif\n mi_track_mem_noaccess(block,sizeof(mi_block_t));\n return next;\n}\n\nstatic inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) {\n mi_track_mem_undefined(block,sizeof(mi_block_t));\n #if MI_ENCODE_FREELIST\n block->next = mi_ptr_encode(null, next, keys);\n #else\n MI_UNUSED(keys); MI_UNUSED(null);\n block->next = (mi_encoded_t)next;\n #endif\n mi_track_mem_noaccess(block,sizeof(mi_block_t));\n}\n\nstatic inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) {\n #if MI_ENCODE_FREELIST\n mi_block_t* next = mi_block_nextx(page,block,page->keys);\n // check for free list corruption: is `next` at least in the same page?\n // TODO: check if `next` is `page->block_size` aligned?\n if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) {\n _mi_error_message(EFAULT, \"corrupted free list entry of size %zub at %p: value 0x%zx\\n\", mi_page_block_size(page), block, (uintptr_t)next);\n next = NULL;\n }\n return next;\n #else\n MI_UNUSED(page);\n return mi_block_nextx(page,block,NULL);\n #endif\n}\n\nstatic inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) {\n #if MI_ENCODE_FREELIST\n mi_block_set_nextx(page,block,next, page->keys);\n #else\n MI_UNUSED(page);\n mi_block_set_nextx(page,block,next,NULL);\n #endif\n}\n\n/* -----------------------------------------------------------\n arena blocks\n----------------------------------------------------------- */\n\n// Blocks needed for a given byte size\nstatic inline size_t mi_slice_count_of_size(size_t size) {\n return _mi_divide_up(size, MI_ARENA_SLICE_SIZE);\n}\n\n// Byte size of a number of blocks\nstatic inline size_t mi_size_of_slices(size_t bcount) {\n return (bcount * MI_ARENA_SLICE_SIZE);\n}\n\n\n/* -----------------------------------------------------------\n memory id's\n----------------------------------------------------------- */\n\nstatic inline mi_memid_t _mi_memid_create(mi_memkind_t memkind) {\n mi_memid_t memid;\n _mi_memzero_var(memid);\n memid.memkind = memkind;\n return memid;\n}\n\nstatic inline mi_memid_t _mi_memid_none(void) {\n return _mi_memid_create(MI_MEM_NONE);\n}\n\nstatic inline mi_memid_t _mi_memid_create_os(void* base, size_t size, bool committed, bool is_zero, bool is_large) {\n mi_memid_t memid = _mi_memid_create(MI_MEM_OS);\n memid.mem.os.base = base;\n memid.mem.os.size = size;\n memid.initially_committed = committed;\n memid.initially_zero = is_zero;\n memid.is_pinned = is_large;\n return memid;\n}\n\nstatic inline mi_memid_t _mi_memid_create_meta(void* mpage, size_t block_idx, size_t block_count) {\n mi_memid_t memid = _mi_memid_create(MI_MEM_META);\n memid.mem.meta.meta_page = mpage;\n memid.mem.meta.block_index = (uint32_t)block_idx;\n memid.mem.meta.block_count = (uint32_t)block_count;\n memid.initially_committed = true;\n memid.initially_zero = true;\n memid.is_pinned = true;\n return memid;\n}\n\n\n// -------------------------------------------------------------------\n// Fast \"random\" shuffle\n// -------------------------------------------------------------------\n\nstatic inline uintptr_t _mi_random_shuffle(uintptr_t x) {\n if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros\n#if (MI_INTPTR_SIZE>=8)\n // by Sebastiano Vigna, see: \n x ^= x >> 30;\n x *= 0xbf58476d1ce4e5b9UL;\n x ^= x >> 27;\n x *= 0x94d049bb133111ebUL;\n x ^= x >> 31;\n#elif (MI_INTPTR_SIZE==4)\n // by Chris Wellons, see: \n x ^= x >> 16;\n x *= 0x7feb352dUL;\n x ^= x >> 15;\n x *= 0x846ca68bUL;\n x ^= x >> 16;\n#endif\n return x;\n}\n\n\n// ---------------------------------------------------------------------------------\n// Provide our own `_mi_memcpy/set` for potential performance optimizations.\n//\n// For now, only on x64/x86 we optimize to `rep movsb/stosb`.\n// Generally, we check for \"fast short rep movsb/stosb\" (FSRM/FSRS) or \"fast enhanced rep movsb\" (ERMS) support\n// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253.\n// Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows.\n// ---------------------------------------------------------------------------------\n\n#if !MI_TRACK_ENABLED && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__))\n\nextern mi_decl_hidden size_t _mi_cpu_movsb_max; // in init.c\nextern mi_decl_hidden size_t _mi_cpu_stosb_max;\n\nstatic inline void mi_rep_movsb(void* dst, const void* src, size_t n) {\n #if defined(__GNUC__)\n __asm volatile(\"rep movsb\" : \"+D\"(dst), \"+c\"(n), \"+S\"(src) : : \"memory\");\n #else\n __movsb((unsigned char*)dst, (const unsigned char*)src, n);\n #endif\n}\n\nstatic inline void mi_rep_stosb(void* dst, uint8_t val, size_t n) {\n #if defined(__GNUC__)\n __asm volatile(\"rep stosb\" : \"+D\"(dst), \"+c\"(n) : \"a\"(val) : \"memory\");\n #else\n __stosb((unsigned char*)dst, val, n);\n #endif\n}\n\nstatic inline void _mi_memcpy(void* dst, const void* src, size_t n) {\n if mi_likely(n <= _mi_cpu_movsb_max) { // has fsrm && n <= 127 (todo: and maybe has erms?)\n mi_rep_movsb(dst, src, n);\n }\n else {\n memcpy(dst, src, n);\n }\n}\n\nstatic inline void _mi_memset(void* dst, int val, size_t n) {\n if mi_likely(n <= _mi_cpu_stosb_max) { // has fsrs && n <= 127\n mi_rep_stosb(dst, (uint8_t)val, n);\n }\n else {\n memset(dst, val, n);\n }\n}\n\n#else\n\nstatic inline void _mi_memcpy(void* dst, const void* src, size_t n) {\n memcpy(dst, src, n);\n}\n\nstatic inline void _mi_memset(void* dst, int val, size_t n) {\n memset(dst, val, n);\n}\n\n#endif\n\n// -------------------------------------------------------------------------------\n// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned\n// This is used for example in `mi_realloc`.\n// -------------------------------------------------------------------------------\n\n#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)\n\n// On GCC/CLang we provide a hint that the pointers are word aligned.\nstatic inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {\n mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));\n void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);\n const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE);\n _mi_memcpy(adst, asrc, n);\n}\n\nstatic inline void _mi_memset_aligned(void* dst, int val, size_t n) {\n mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);\n void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);\n _mi_memset(adst, val, n);\n}\n\n#else\n\n// Default fallback on `_mi_memcpy`\nstatic inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {\n mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));\n _mi_memcpy(dst, src, n);\n}\n\nstatic inline void _mi_memset_aligned(void* dst, int val, size_t n) {\n mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);\n _mi_memset(dst, val, n);\n}\n\n#endif\n\nstatic inline void _mi_memzero(void* dst, size_t n) {\n _mi_memset(dst, 0, n);\n}\n\nstatic inline void _mi_memzero_aligned(void* dst, size_t n) {\n _mi_memset_aligned(dst, 0, n);\n}\n\n\n\n#endif // MI_INTERNAL_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/include/mimalloc/prim.h", "sha256": "555302d2fcd05e974441d0c9c9208c26e860db4b3f17b9025a08f42e0f4807ba", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MIMALLOC_PRIM_H\n#define MIMALLOC_PRIM_H\n#include \"internal.h\" // mi_decl_hidden\n\n// --------------------------------------------------------------------------\n// This file specifies the primitive portability API.\n// Each OS/host needs to implement these primitives, see `src/prim`\n// for implementations on Window, macOS, WASI, and Linux/Unix.\n//\n// note: on all primitive functions, we always have result parameters != NULL, and:\n// addr != NULL and page aligned\n// size > 0 and page aligned\n// the return value is an error code as an `int` where 0 is success\n// --------------------------------------------------------------------------\n\n// OS memory configuration\ntypedef struct mi_os_mem_config_s {\n size_t page_size; // default to 4KiB\n size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows)\n size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB)\n size_t physical_memory_in_kib; // physical memory size in KiB\n size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization)\n bool has_overcommit; // can we reserve more memory than can be actually committed?\n bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc)\n bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory)\n bool has_transparent_huge_pages; // true if transparent huge pages are enabled (on Linux)\n} mi_os_mem_config_t;\n\n// Initialize\nvoid _mi_prim_mem_init( mi_os_mem_config_t* config );\n\n// Free OS memory\nint _mi_prim_free(void* addr, size_t size );\n\n// Allocate OS memory. Return NULL on error.\n// The `try_alignment` is just a hint and the returned pointer does not have to be aligned.\n// If `commit` is false, the virtual memory range only needs to be reserved (with no access)\n// which will later be committed explicitly using `_mi_prim_commit`.\n// `is_zero` is set to true if the memory was zero initialized (as on most OS's)\n// The `hint_addr` address is either `NULL` or a preferred allocation address but can be ignored.\n// pre: !commit => !allow_large\n// try_alignment >= _mi_os_page_size() and a power of 2\nint _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr);\n\n// Commit memory. Returns error code or 0 on success.\n// For example, on Linux this would make the memory PROT_READ|PROT_WRITE.\n// `is_zero` is set to true if the memory was zero initialized (e.g. on Windows)\nint _mi_prim_commit(void* addr, size_t size, bool* is_zero);\n\n// Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true\n// if the memory would need to be re-committed. For example, on Windows this is always true,\n// but on Linux we could use MADV_DONTNEED to decommit which does not need a recommit.\n// pre: needs_recommit != NULL\nint _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit);\n\n// Reset memory. The range keeps being accessible but the content might be reset to zero at any moment.\n// Returns error code or 0 on success.\nint _mi_prim_reset(void* addr, size_t size);\n\n// Reuse memory. This is called for memory that is already committed but\n// may have been reset (`_mi_prim_reset`) or decommitted (`_mi_prim_decommit`) where `needs_recommit` was false.\n// Returns error code or 0 on success. On most platforms this is a no-op.\nint _mi_prim_reuse(void* addr, size_t size);\n\n// Protect memory. Returns error code or 0 on success.\nint _mi_prim_protect(void* addr, size_t size, bool protect);\n\n// Allocate huge (1GiB) pages possibly associated with a NUMA node.\n// `is_zero` is set to true if the memory was zero initialized (as on most OS's)\n// pre: size > 0 and a multiple of 1GiB.\n// numa_node is either negative (don't care), or a numa node number.\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr);\n\n// Return the current NUMA node\nsize_t _mi_prim_numa_node(void);\n\n// Return the number of logical NUMA nodes\nsize_t _mi_prim_numa_node_count(void);\n\n// Clock ticks\nmi_msecs_t _mi_prim_clock_now(void);\n\n// Return process information (only for statistics)\ntypedef struct mi_process_info_s {\n mi_msecs_t elapsed;\n mi_msecs_t utime;\n mi_msecs_t stime;\n size_t current_rss;\n size_t peak_rss;\n size_t current_commit;\n size_t peak_commit;\n size_t page_faults;\n} mi_process_info_t;\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo);\n\n// Default stderr output. (only for warnings etc. with verbose enabled)\n// msg != NULL && _mi_strlen(msg) > 0\nvoid _mi_prim_out_stderr( const char* msg );\n\n// Get an environment variable. (only for options)\n// name != NULL, result != NULL, result_size >= 64\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size);\n\n\n// Fill a buffer with strong randomness; return `false` on error or if\n// there is no strong randomization available.\nbool _mi_prim_random_buf(void* buf, size_t buf_len);\n\n// Called on the first thread start, and should ensure `_mi_thread_done` is called on thread termination.\nvoid _mi_prim_thread_init_auto_done(void);\n\n// Called on process exit and may take action to clean up resources associated with the thread auto done.\nvoid _mi_prim_thread_done_auto_done(void);\n\n// Called when the default theap for a thread changes\nvoid _mi_prim_thread_associate_default_theap(mi_theap_t* theap);\n\n// Is this thread part of a thread pool?\nbool _mi_prim_thread_is_in_threadpool(void);\n\n// Yield to other threads. Should be similar to `sleep(0)`. \n// Is called only in rare situations and does not have to be lightning fast.\nvoid _mi_prim_thread_yield(void);\n\n//-------------------------------------------------------------------\n// Access to TLS (thread local storage) slots.\n// We need fast access to both a unique thread id (in `free.c:mi_free`) and\n// to a thread-local theap pointer (in `alloc.c:mi_malloc`).\n// To achieve this we use specialized code for various platforms.\n//-------------------------------------------------------------------\n\n// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot.\n// The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform.\n// If you test on another platform and it works please send a PR :-)\n// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.\n//\n// Note: we would like to prefer `__builtin_thread_pointer()` nowadays instead of using assembly,\n// but unfortunately we can not detect support reliably (see issue #883)\n// We also use it on Apple OS as we use a TLS slot for the default theap there.\n#if (defined(_WIN32)) || \\\n (defined(__GNUC__) && ( \\\n (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \\\n || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \\\n || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \\\n || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \\\n || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \\\n ))\n\nstatic inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {\n void* res;\n const size_t ofs = (slot*sizeof(void*));\n #if defined(_WIN32)\n #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)\n res = (void*)__readgsqword((unsigned long)ofs); // direct load at offset from gs\n #elif _M_IX86 && !defined(_M_ARM64EC)\n res = (void*)__readfsdword((unsigned long)ofs); // direct load at offset from fs\n #else\n res = ((void**)NtCurrentTeb())[slot]; MI_UNUSED(ofs);\n #endif\n #elif defined(__i386__)\n __asm__(\"movl %%gs:%1, %0\" : \"=r\" (res) : \"m\" (*((void**)ofs)) : ); // x86 32-bit always uses GS\n #elif defined(__APPLE__) && defined(__x86_64__)\n __asm__(\"movq %%gs:%1, %0\" : \"=r\" (res) : \"m\" (*((void**)ofs)) : ); // x86_64 macOSX uses GS\n #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)\n __asm__(\"movl %%fs:%1, %0\" : \"=r\" (res) : \"m\" (*((void**)ofs)) : ); // x32 ABI\n #elif defined(__x86_64__)\n __asm__(\"movq %%fs:%1, %0\" : \"=r\" (res) : \"m\" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS\n #elif defined(__arm__)\n void** tcb; MI_UNUSED(ofs);\n __asm__ volatile (\"mrc p15, 0, %0, c13, c0, 3\\nbic %0, %0, #3\" : \"=r\" (tcb));\n res = tcb[slot];\n #elif defined(__aarch64__)\n void** tcb; MI_UNUSED(ofs);\n #if defined(__APPLE__) // M1, issue #343\n __asm__ volatile (\"mrs %0, tpidrro_el0\\nbic %0, %0, #7\" : \"=r\" (tcb));\n #else\n __asm__ volatile (\"mrs %0, tpidr_el0\" : \"=r\" (tcb));\n #endif\n res = tcb[slot];\n #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781\n MI_UNUSED(ofs);\n res = pthread_getspecific(slot);\n #else\n #define MI_HAS_TLS_SLOT 0\n MI_UNUSED(ofs);\n res = NULL;\n #endif\n return res;\n}\n\n#ifndef MI_HAS_TLS_SLOT\n#define MI_HAS_TLS_SLOT 1\n#endif\n\n// setting a tls slot is only used on macOS for now\nstatic inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {\n const size_t ofs = (slot*sizeof(void*));\n #if defined(_WIN32)\n ((void**)NtCurrentTeb())[slot] = value; MI_UNUSED(ofs);\n #elif defined(__i386__)\n __asm__(\"movl %1,%%gs:%0\" : \"=m\" (*((void**)ofs)) : \"rn\" (value) : ); // 32-bit always uses GS\n #elif defined(__APPLE__) && defined(__x86_64__)\n __asm__(\"movq %1,%%gs:%0\" : \"=m\" (*((void**)ofs)) : \"rn\" (value) : ); // x86_64 macOS uses GS\n #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)\n __asm__(\"movl %1,%%fs:%0\" : \"=m\" (*((void**)ofs)) : \"rn\" (value) : ); // x32 ABI\n #elif defined(__x86_64__)\n __asm__(\"movq %1,%%fs:%0\" : \"=m\" (*((void**)ofs)) : \"rn\" (value) : ); // x86_64 Linux, BSD uses FS\n #elif defined(__arm__)\n void** tcb; MI_UNUSED(ofs);\n __asm__ volatile (\"mrc p15, 0, %0, c13, c0, 3\\nbic %0, %0, #3\" : \"=r\" (tcb));\n tcb[slot] = value;\n #elif defined(__aarch64__)\n void** tcb; MI_UNUSED(ofs);\n #if defined(__APPLE__) // M1, issue #343\n __asm__ volatile (\"mrs %0, tpidrro_el0\\nbic %0, %0, #7\" : \"=r\" (tcb));\n #else\n __asm__ volatile (\"mrs %0, tpidr_el0\" : \"=r\" (tcb));\n #endif\n tcb[slot] = value;\n #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781\n MI_UNUSED(ofs);\n pthread_setspecific(slot, value);\n #else\n MI_UNUSED(ofs); MI_UNUSED(value);\n #endif\n}\n\n#endif\n\n\n// defined in `init.c`; do not use these directly\nextern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main; // theap belonging to the main heap\nextern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called?\n\n\n//-------------------------------------------------------------------\n// Get a fast unique thread id.\n//\n// Getting the thread id should be performant as it is called in the\n// fast path of `_mi_free` and we specialize for various platforms as\n// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.\n// We only require _mi_prim_thread_id() to return a unique id\n// for each thread (unequal to zero).\n//-------------------------------------------------------------------\n\n\n// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id\n// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)\n// Nevertheless, it seems needed on older graviton platforms (see issue #851).\n// For now, we only enable this for specific platforms.\n#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */\n #if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly ()*/ \\\n && !defined(__CYGWIN__) \\\n && !defined(MI_LIBC_MUSL) \\\n && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot () */\n #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \\\n || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \\\n || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))\n #define MI_USE_BUILTIN_THREAD_POINTER 1\n #endif\n #endif\n#endif\n\nstatic inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept;\n\nstatic inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {\n const mi_threadid_t tid = __mi_prim_thread_id();\n mi_assert_internal(tid > 1);\n mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear?\n return tid;\n}\n\n// Get a unique id for the current thread.\n#if defined(MI_PRIM_THREAD_ID)\n\nstatic inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {\n const mi_threadid_t tid = MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)\n mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits\n return tid;\n}\n\n#elif defined(_WIN32)\n\nstatic inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {\n // Windows: works on Intel and ARM in both 32- and 64-bit\n return (uintptr_t)NtCurrentTeb();\n}\n\n#elif MI_USE_BUILTIN_THREAD_POINTER\n\nstatic inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {\n // Works on most Unix based platforms with recent compilers\n return (uintptr_t)__builtin_thread_pointer();\n}\n\n#elif MI_HAS_TLS_SLOT\n\nstatic inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {\n #if defined(__BIONIC__)\n // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id\n // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86\n return (uintptr_t)mi_prim_tls_slot(1);\n #else\n // in all our other targets, slot 0 is the thread id\n // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h\n // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36\n return (uintptr_t)mi_prim_tls_slot(0);\n #endif\n}\n\n#else\n\n// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).\nstatic inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {\n return (uintptr_t)&__mi_theap_main;\n}\n\n#endif\n\n\n\n/* ----------------------------------------------------------------------------------------\nGet the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`).\n\nThis is inlined here as it is on the fast path for allocation functions.\nWe have 4 models:\n\n- MI_TLS_MODEL_THREAD_LOCAL: use regular thread local (default on Linux, FreeBSD, etc)\n On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a\n thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures\n that the storage will always be available and properly initialized (with an empty theap).\n\n On some platforms the underlying TLS implementation (or the loader) will call itself `malloc`\n on a first access to a thread local and recurse in the MI_TLS_MODEL_THREAD_LOCAL.\n A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process\n is initialized before accessing the thread-local. This is a check in the fast path though\n so this should be avoided.\n\n- MI_TLS_MODEL_FIXED_SLOT: use a fixed slot in the TLS block (default on macOS)\n This reserves an unused and fixed TLS slot. This is fast and avoids the problem\n where the underlying TLS implementation (or the loader) will call itself `malloc`\n on a first access to a thread local (and recurse in the MI_TLS_MODEL_THREAD_LOCAL).\n This goes wrong though if the OS or a library uses the same fixed slot.\n\n- MI_TLS_MODEL_DYNAMIC_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows)\n Windows has somewhat slow thread locals so by default we use TlsAlloc'd slots which\n can be more efficient. First tries to use one of the \"direct\" first 64 slots which \n are the fastest, but falls back to using \"expansion\" slots when needed (up to 1088 slots).\n (If the allocated slot happens to always be under 64 for a particular program,\n one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.)\n\n- MI_TLS_MODEL_DYNAMIC_PTHREADS: use `pthread_getspecific`. (default on OpenBSD, maybe good for Android as well?)\n Use pthread local storage. Somewhat slow but can work well depending on the platform.\n\nEach model should define `MI_THEAP_INITASNULL` to signify that the initial value\nreturned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap).\nThis incurs an extra check in the fast path (but can often be combined in an existing check).\n------------------------------------------------------------------------------------------- */\n\nstatic inline mi_theap_t* _mi_theap_default(void);\nstatic inline mi_theap_t* _mi_theap_cached(void);\n\n#if defined(_WIN32)\n #define MI_TLS_MODEL_DYNAMIC_WIN32 1 \n#elif defined(__APPLE__) && MI_HAS_TLS_SLOT && !defined(__POWERPC__) // macOS on arm64 or x64\n // #define MI_TLS_MODEL_DYNAMIC_PTHREADS 1 // also works but a bit slower\n #define MI_TLS_MODEL_FIXED_SLOT 1\n #define MI_TLS_MODEL_FIXED_SLOT_DEFAULT 108 // seems unused. @apple: it would be great to get 2 official slots for custom allocators :-)\n #define MI_TLS_MODEL_FIXED_SLOT_CACHED 109\n // we used before __PTK_FRAMEWORK_OLDGC_KEY9 (89) but that seems used now.\n // see \n#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__)\n #define MI_TLS_MODEL_DYNAMIC_PTHREADS 1\n // #define MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL 1\n#else\n #define MI_TLS_MODEL_THREAD_LOCAL 1\n#endif\n\n// Declared this way to optimize register spills and branches\nmi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void);\n\nstatic inline mi_theap_t* __mi_theap_empty(void) {\n #if __GNUC__\n __asm(\"\"); // prevent conditional load\n return (mi_theap_t*)&_mi_theap_empty;\n #else\n return _mi_theap_empty_get();\n #endif\n}\n\n#if MI_TLS_MODEL_THREAD_LOCAL\n// Thread local with an initial value (default on Linux). Very efficient.\n\nextern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from\nextern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap\n\nstatic inline mi_theap_t* _mi_theap_default(void) {\n #if defined(MI_TLS_RECURSE_GUARD)\n if (mi_unlikely(!_mi_process_is_initialized)) return _mi_theap_empty_get();\n #endif\n return __mi_theap_default;\n}\n\nstatic inline mi_theap_t* _mi_theap_cached(void) {\n return __mi_theap_cached;\n}\n\n#elif MI_TLS_MODEL_FIXED_SLOT\n// Fixed TLS slot (default on macOS).\n#define MI_THEAP_INITASNULL 1\n\nstatic inline mi_theap_t* _mi_theap_default(void) {\n return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_DEFAULT);\n}\n\nstatic inline mi_theap_t* _mi_theap_cached(void) {\n return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_CACHED);\n}\n\n#elif MI_TLS_MODEL_DYNAMIC_WIN32\n// Dynamic TLS slot (default on Windows)\n#define MI_THEAP_INITASNULL 1\n\n// We try to use direct slots (64), but can also use the expansion slots (upto 1024 extra available)\n// See for the offsets.\n#if MI_SIZE_SIZE==4\n#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_SIZE_SIZE)\n#else\n#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_SIZE_SIZE)\n#endif\n\nextern mi_decl_hidden size_t _mi_theap_default_slot;\nextern mi_decl_hidden size_t _mi_theap_cached_slot;\nextern mi_decl_hidden size_t _mi_theap_default_expansion_slot;\nextern mi_decl_hidden size_t _mi_theap_cached_expansion_slot;\n\nstatic inline mi_theap_t* _mi_theap_default(void) {\n const size_t slot = _mi_theap_default_slot;\n mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);\n #if !MI_WIN_DIRECT_TLS\n if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?\n if mi_likely(theap!=NULL) { // initialized (on this thread)?\n theap = ((mi_theap_t**)theap)[_mi_theap_default_expansion_slot];\n }\n }\n #endif\n return theap;\n}\n\nstatic inline mi_theap_t* _mi_theap_cached(void) {\n const size_t slot = _mi_theap_cached_slot;\n mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);\n #if !MI_WIN_DIRECT_TLS\n if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?\n if mi_likely(theap!=NULL) { // initialized (on this thread)?\n theap = ((mi_theap_t**)theap)[_mi_theap_cached_expansion_slot];\n }\n }\n #endif\n return theap;\n}\n\n#elif MI_TLS_MODEL_DYNAMIC_PTHREADS\n// Dynamic pthread slot on less common platforms. This is not too bad. (default on OpenBSD)\n#define MI_THEAP_INITASNULL 1\n\nextern mi_decl_hidden pthread_key_t _mi_theap_default_key;\nextern mi_decl_hidden pthread_key_t _mi_theap_cached_key;\n\nstatic inline mi_theap_t* _mi_theap_default(void) {\n #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL\n // we can skip this check if using the initial key will return NULL from pthread_getspecific\n if mi_unlikely(_mi_theap_default_key==0) { return NULL; }\n #endif\n return (mi_theap_t*)pthread_getspecific(_mi_theap_default_key);\n}\n\nstatic inline mi_theap_t* _mi_theap_cached(void) {\n #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL\n // we can skip this check if using the initial key will return NULL from pthread_getspecific\n if mi_unlikely(_mi_theap_cached_key==0) { return NULL; }\n #endif\n return (mi_theap_t*)pthread_getspecific(_mi_theap_cached_key);\n}\n\n#else\n#error \"no TLS model is defined for this platform?\"\n#endif\n\n\n// Check if a thread is initialized (without using a thread-local if using fixed slots)\nstatic inline bool _mi_thread_is_initialized(void) {\n return (mi_theap_is_initialized(_mi_theap_default()));\n}\n\n// Get (and possible create) the theap belonging to a heap\n// We cache the last accessed theap in `_mi_theap_cached` for better performance.\nstatic inline mi_theap_t* _mi_heap_theap(const mi_heap_t* heap) {\n mi_theap_t* theap = _mi_theap_cached(); \n #if MI_THEAP_INITASNULL\n if mi_likely(theap!=NULL && _mi_theap_heap(theap)==heap) return theap;\n #else\n if mi_likely(_mi_theap_heap(theap)==heap) return theap;\n #endif\n return _mi_heap_theap_get_or_init(heap);\n}\n\n// Get the theap belonging to a heap without creating in if it is not yet initialized.\nstatic inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) {\n mi_theap_t* theap = _mi_theap_cached();\n #if MI_THEAP_INITASNULL\n if mi_unlikely(theap==NULL || _mi_theap_heap(theap)!=heap)\n #else\n if mi_unlikely(_mi_theap_heap(theap)!=heap)\n #endif\n {\n theap = _mi_heap_theap_get_peek(heap); // don't update the cache on a query (?)\n }\n mi_assert(theap==NULL || _mi_theap_heap(theap)==heap);\n return theap;\n}\n\n// Find the associated theap or NULL if it does not exist (during shutdown)\n// Should be fast as it is called in `free.c:mi_free_try_collect`.\nstatic inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) {\n mi_heap_t* const heap = page->heap;\n mi_theap_t* theap;\n if mi_likely(heap==NULL) { theap = __mi_theap_main; } // note: on macOS accessing the thread_local can cause allocation during thread shutdown (and reinitialize the thread)!\n else { theap = _mi_heap_theap_peek(heap); }\n mi_assert_internal(theap==NULL || _mi_thread_id()==theap->tld->thread_id);\n return theap;\n}\n\n#endif // MI_PRIM_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/include/mimalloc/track.h", "sha256": "1f378b4b6063c078d2aeb5d916c1d30b5db787a44ab9118e7daf673b086882c6", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MI_TRACK_H\n#define MI_TRACK_H\n\n/* ------------------------------------------------------------------------------------------------------\nTrack memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers.\nThese can be defined for tracking allocation:\n\n #define mi_track_malloc_size(p,reqsize,size,zero)\n #define mi_track_free_size(p,_size)\n\nThe macros are set up such that the size passed to `mi_track_free_size`\nalways matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`).\nThe `reqsize` is what the user requested, and `size >= reqsize`.\nThe `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled,\nor otherwise it is the usable block size which may be larger than the original request.\nUse `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc).\nThe `zero` parameter is `true` if the allocated block is zero initialized.\n\nOptional:\n\n #define mi_track_align(p,alignedp,offset,size)\n #define mi_track_resize(p,oldsize,newsize)\n #define mi_track_init()\n\nThe `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block.\nThe corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`).\nThe `mi_track_resize` is currently unused but could be called on reallocations within a block.\n`mi_track_init` is called at program start.\n\nThe following macros are for tools like asan and valgrind to track whether memory is\ndefined, undefined, or not accessible at all:\n\n #define mi_track_mem_defined(p,size)\n #define mi_track_mem_undefined(p,size)\n #define mi_track_mem_noaccess(p,size)\n\n-------------------------------------------------------------------------------------------------------*/\n\n#if MI_TRACK_VALGRIND\n// valgrind tool\n\n#define MI_TRACK_ENABLED 1\n#define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on theap_destroy\n#define MI_TRACK_TOOL \"valgrind\"\n\n#include \n#include \n\n#define mi_track_malloc_size(p,reqsize,size,zero) VALGRIND_MALLOCLIKE_BLOCK(p,size,MI_PADDING_SIZE /*red zone*/,zero)\n#define mi_track_free_size(p,_size) VALGRIND_FREELIKE_BLOCK(p,MI_PADDING_SIZE /*red zone*/)\n#define mi_track_resize(p,oldsize,newsize) VALGRIND_RESIZEINPLACE_BLOCK(p,oldsize,newsize,MI_PADDING_SIZE /*red zone*/)\n#define mi_track_mem_defined(p,size) VALGRIND_MAKE_MEM_DEFINED(p,size)\n#define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size)\n#define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size)\n\n#elif MI_TRACK_ASAN\n// address sanitizer\n\n#define MI_TRACK_ENABLED 1\n#define MI_TRACK_HEAP_DESTROY 0\n#define MI_TRACK_TOOL \"asan\"\n\n#include \n\n#define mi_track_malloc_size(p,reqsize,size,zero) ASAN_UNPOISON_MEMORY_REGION(p,size)\n#define mi_track_free_size(p,size) ASAN_POISON_MEMORY_REGION(p,size)\n#define mi_track_mem_defined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size)\n#define mi_track_mem_undefined(p,size) ASAN_UNPOISON_MEMORY_REGION(p,size)\n#define mi_track_mem_noaccess(p,size) ASAN_POISON_MEMORY_REGION(p,size)\n\n#elif MI_TRACK_ETW\n// windows event tracing\n\n#define MI_TRACK_ENABLED 1\n#define MI_TRACK_HEAP_DESTROY 1\n#define MI_TRACK_TOOL \"ETW\"\n\n#include \"../src/prim/windows/etw.h\"\n\n#define mi_track_init() EventRegistermicrosoft_windows_mimalloc()\n#define mi_track_done() EventUnregistermicrosoft_windows_mimalloc()\n#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size)\n#define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)(p), size)\n\n#else\n// no tracking\n\n#define MI_TRACK_ENABLED 0\n#define MI_TRACK_HEAP_DESTROY 0\n#define MI_TRACK_TOOL \"none\"\n\n#define mi_track_malloc_size(p,reqsize,size,zero)\n#define mi_track_free_size(p,_size)\n\n#endif\n\n// -------------------\n// Utility definitions\n\n#ifndef mi_track_resize\n#define mi_track_resize(p,oldsize,newsize) do{ mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false); } while(0)\n#endif\n\n#ifndef mi_track_align\n#define mi_track_align(p,alignedp,offset,size) mi_track_mem_noaccess(p,offset)\n#endif\n\n#ifndef mi_track_init\n#define mi_track_init()\n#endif\n\n#ifndef mi_track_done\n#define mi_track_done()\n#endif\n\n#ifndef mi_track_mem_defined\n#define mi_track_mem_defined(p,size)\n#endif\n\n#ifndef mi_track_mem_undefined\n#define mi_track_mem_undefined(p,size)\n#endif\n\n#ifndef mi_track_mem_noaccess\n#define mi_track_mem_noaccess(p,size)\n#endif\n\n\n#if MI_PADDING\n#define mi_track_malloc(p,reqsize,zero) \\\n if ((p)!=NULL) { \\\n mi_assert_internal(mi_usable_size(p)==(reqsize)); \\\n mi_track_malloc_size(p,reqsize,reqsize,zero); \\\n }\n#else\n#define mi_track_malloc(p,reqsize,zero) \\\n if ((p)!=NULL) { \\\n mi_assert_internal(mi_usable_size(p)>=(reqsize)); \\\n mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \\\n }\n#endif\n\n#endif // MI_TRACK_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/include/mimalloc/types.h", "sha256": "b54e760da2a8d9d8a9cb708e280268c2d194c99bc518b21f72db70100578ad6a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#pragma once\n#ifndef MI_TYPES_H\n#define MI_TYPES_H\n\n// --------------------------------------------------------------------------\n// This file contains the main type definitions for mimalloc:\n// mi_heap_t : all data for a heap; usually there is just one main default heap.\n// mi_theap_t : a thread local heap belonging to a specific heap:\n// maintains lists of thread-local heap pages that have free space.\n// mi_page_t : a mimalloc page (usually 64KiB or 512KiB) from\n// where objects of a single size are allocated.\n// Note: we write \"OS page\" for OS memory pages while\n// using plain \"page\" for mimalloc pages (`mi_page_t`).\n// mi_arena_t : a large memory area where pages are allocated (process shared)\n// mi_tld_t : thread local data\n// mi_subproc_t : all heaps belong to a sub-process (usually just the main one)\n// --------------------------------------------------------------------------\n\n\n#include \n#include // ptrdiff_t\n#include // uintptr_t, uint16_t, etc\n#include // bool\n#include // SIZE_MAX etc.\n#include // error codes\n#include \"bits.h\" // size defines (MI_INTPTR_SIZE etc), bit operations\n#include \"atomic.h\" // _Atomic primitives\n\n// Minimal alignment necessary. On most platforms 16 bytes are needed\n// due to SSE registers for example. This must be at least `sizeof(void*)`\n#ifndef MI_MAX_ALIGN_SIZE\n#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t)\n#endif\n\n\n// ------------------------------------------------------\n// Variants\n// ------------------------------------------------------\n\n// Define NDEBUG in the release version to disable assertions.\n// #define NDEBUG\n\n// Define MI_TRACK_ to enable tracking support\n// #define MI_TRACK_VALGRIND 1\n// #define MI_TRACK_ASAN 1\n// #define MI_TRACK_ETW 1\n\n// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).\n// #define MI_STAT 1\n\n// Define MI_SECURE to enable security mitigations\n// #define MI_SECURE 1 // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption\n// #define MI_SECURE 2 // randomize relative allocation addresses (within mimalloc pages)\n// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)\n// #define MI_SECURE 4 // checks for double free (may be more expensive) (`-DMI_SECURE=ON`)\n// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`)\n\n#if !defined(MI_SECURE)\n#define MI_SECURE 0\n#endif\n\n// Define MI_DEBUG for assertion and invariant checking\n// #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON)\n// #define MI_DEBUG 2 // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON)\n// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON)\n#if !defined(MI_DEBUG)\n#if defined(MI_BUILD_RELEASE) || defined(NDEBUG)\n#define MI_DEBUG 0\n#else\n#define MI_DEBUG 2\n#endif\n#endif\n\n// Statistics (0=only essential, 1=normal, 2=more fine-grained (expensive) tracking)\n#ifndef MI_STAT\n#if (MI_DEBUG>0)\n#define MI_STAT 2\n#else\n#define MI_STAT 0\n#endif\n#endif\n\n// Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options)\n#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_PAGE_META_ALIGNED_FREE_SMALL \n#define MI_GUARDED 1\n#endif\n\n// Reserve extra padding at the end of each block to be more resilient against theap block overflows.\n// The padding can detect buffer overflow on free.\n#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW))\n#define MI_PADDING 1\n#endif\n\n// Check padding bytes; allows byte-precise buffer overflow detection\n#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1)\n#define MI_PADDING_CHECK 1\n#endif\n\n\n// Encoded free lists allow detection of corrupted free lists\n// and can detect buffer overflows, modify after free, and double `free`s.\n#if (MI_SECURE>=3 || MI_DEBUG>=1)\n#define MI_ENCODE_FREELIST 1\n#endif\n\n// Enable large pages for objects between 64KiB and 512KiB.\n// This should perhaps be disabled by default as for many workloads the block sizes above 64 KiB\n// are quite random which can lead to too many partially used large pages (but see issue #1104).\n#ifndef MI_ENABLE_LARGE_PAGES\n#define MI_ENABLE_LARGE_PAGES 1\n#endif\n\n// Place page meta info at the start of the page area or keep it separate? \n// Separate keeps the page info at the arena start (default) which is more secure \n// and reduces wasted space due to alignment and block sizes. \n// (but also reserves more memory up front (about 2MiB per GiB))\n#if !defined(MI_PAGE_META_IS_SEPARATED)\n#if MI_PAGE_MAP_FLAT\n#define MI_PAGE_META_IS_SEPARATED 0\n#else\n#define MI_PAGE_META_IS_SEPARATED 1\n#endif\n#endif\n\n// We can choose to only put page info of small pages at the start of the page area.\n// This can be used to have a slightly faster `mi_free_small` function for specialized\n// cases (like language runtime systems).\n#if !defined(MI_PAGE_META_ALIGNED_FREE_SMALL)\n#define MI_PAGE_META_ALIGNED_FREE_SMALL 0\n#endif\n\n// Configuration checks\n#if !MI_PAGE_META_IS_SEPARATED && MI_SECURE\n#error \"secure mode should use separated page infos\"\n#endif\n#if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE\n#error \"secure mode cannot use MI_PAGE_META_ALIGNED_FREE_SMALL\"\n#endif\n#if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT\n#error \"cannot have a flat page map with separated page infos\"\n#endif\n#if MI_DEBUG && NDEBUG\n#warning \"mimalloc assertions enabled in a release build\"\n#endif\n\n\n// --------------------------------------------------------------\n// Sizes of internal data-structures\n// (comments specify sizes on 64-bit, usually 32-bit is halved)\n// --------------------------------------------------------------\n\n// Main size parameter; determines max arena sizes and max arena object sizes etc.\n#ifndef MI_ARENA_SLICE_SHIFT\n #ifdef MI_SMALL_PAGE_SHIFT // backward compatibility\n #define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT\n #elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64\n #define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages\n #else\n #define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit)\n #endif\n#endif\n#if MI_ARENA_SLICE_SHIFT < 12\n#error Arena slices should be at least 4KiB\n#endif\n\n#ifndef MI_BCHUNK_BITS_SHIFT\n #if MI_ARENA_SLICE_SHIFT <= 13 // <= 8KiB\n #define MI_BCHUNK_BITS_SHIFT (7) // 128 bits\n #elif MI_ARENA_SLICE_SHIFT < 16 // <= 32KiB\n #define MI_BCHUNK_BITS_SHIFT (8) // 256 bits\n #else\n #define MI_BCHUNK_BITS_SHIFT (6 + MI_SIZE_SHIFT) // 512 bits (or 256 on 32-bit)\n #endif\n#endif\n\n#define MI_BCHUNK_BITS (1 << MI_BCHUNK_BITS_SHIFT) // sub-bitmaps in arena's are \"bchunks\" of 512 bits\n#define MI_ARENA_SLICE_SIZE (MI_ZU(1) << MI_ARENA_SLICE_SHIFT) // arena's allocate in slices of 64 KiB\n#define MI_ARENA_SLICE_ALIGN (MI_ARENA_SLICE_SIZE)\n\n#define MI_ARENA_MIN_OBJ_SLICES (1)\n#define MI_ARENA_MAX_CHUNK_OBJ_SLICES (MI_BCHUNK_BITS) // 32 MiB (or 8 MiB on 32-bit)\n\n#define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_MIN_OBJ_SLICES * MI_ARENA_SLICE_SIZE)\n#define MI_ARENA_MAX_CHUNK_OBJ_SIZE (MI_ARENA_MAX_CHUNK_OBJ_SLICES * MI_ARENA_SLICE_SIZE)\n\n#if MI_ARENA_MAX_CHUNK_OBJ_SIZE < MI_SIZE_SIZE*1024\n#error maximum object size may be too small to hold local thread data\n#endif\n\n#define MI_SMALL_PAGE_SIZE MI_ARENA_MIN_OBJ_SIZE // 64 KiB\n#define MI_MEDIUM_PAGE_SIZE (8*MI_SMALL_PAGE_SIZE) // 512 KiB (=byte in the bchunk bitmap)\n#define MI_LARGE_PAGE_SIZE (MI_SIZE_SIZE*MI_MEDIUM_PAGE_SIZE) // 4 MiB (=word in the bchunk bitmap)\n\n\n// Maximum number of size classes. (spaced exponentially in 12.5% increments)\n#if MI_BIN_HUGE != 73U\n#error \"mimalloc internal: expecting 73 bins\"\n#endif\n#define MI_BIN_FULL (MI_BIN_HUGE+1)\n#define MI_BIN_COUNT (MI_BIN_FULL+1)\n\n// We never allocate more than PTRDIFF_MAX (see also )\n#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX\n\n// Minimal commit for a page on-demand commit (should be >= OS page size)\n#define MI_PAGE_MIN_COMMIT_SIZE MI_ARENA_SLICE_SIZE\n\n\n// ------------------------------------------------------\n// Arena's are large reserved areas of memory allocated from\n// the OS that are managed by mimalloc to efficiently\n// allocate MI_ARENA_SLICE_SIZE slices of memory for the\n// mimalloc pages.\n// ------------------------------------------------------\n\n// A large memory arena where pages are allocated in.\ntypedef struct mi_arena_s mi_arena_t; // defined below\n\n\n// ------------------------------------------------------\n// Heaps contain allocated blocks. Heaps are self-contained\n// but share the (sub-process) memory in the arena's.\n// ------------------------------------------------------\n\n// A first-class heap.\ntypedef struct mi_heap_s mi_heap_t; // heaps\n\n// ------------------------------------------------------\n// We can have sub-processes that are fully separated\n// from each other (for running multiple Python interpreters\n// for example). A sub-process holds the memory arenas and heaps.\n// ------------------------------------------------------\n\n// A sub-process\ntypedef struct mi_subproc_s mi_subproc_t;\n\n\n// ---------------------------------------------------------------\n// a memory id tracks the provenance of arena/OS allocated memory\n// ---------------------------------------------------------------\n\n// Memory can reside in arena's, direct OS allocated, meta-data pages, or statically allocated.\n// The memid keeps track of this.\ntypedef enum mi_memkind_e {\n MI_MEM_NONE, // not allocated\n MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example)\n MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`init.c`))\n MI_MEM_META, // allocated with the meta data allocator (`arena-meta.c`)\n MI_MEM_OS, // allocated from the OS\n MI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory)\n MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`)\n MI_MEM_ARENA, // allocated from an arena (the usual case) (`arena.c`)\n MI_MEM_HEAP_MAIN // allocated in the main heap (for theaps)\n} mi_memkind_t;\n\nstatic inline bool mi_memkind_is_os(mi_memkind_t memkind) {\n return (memkind >= MI_MEM_OS && memkind <= MI_MEM_OS_REMAP);\n}\n\nstatic inline bool mi_memkind_needs_no_free(mi_memkind_t memkind) {\n return (memkind <= MI_MEM_STATIC);\n}\n\n\ntypedef struct mi_memid_os_info {\n void* base; // actual base address of the block (used for offset aligned allocations)\n size_t size; // allocated full size\n // size_t alignment; // alignment at allocation\n} mi_memid_os_info_t;\n\ntypedef struct mi_memid_arena_info {\n mi_arena_t* arena; // arena that contains this memory\n uint32_t slice_index; // slice index in the arena\n uint32_t slice_count; // allocated slices\n} mi_memid_arena_info_t;\n\ntypedef struct mi_memid_meta_info {\n void* meta_page; // meta-page that contains the block\n uint32_t block_index; // block index in the meta-data page\n uint32_t block_count; // allocated blocks\n} mi_memid_meta_info_t;\n\ntypedef struct mi_memid_s {\n union {\n mi_memid_os_info_t os; // only used for MI_MEM_OS\n mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA\n mi_memid_meta_info_t meta; // only used for MI_MEM_META\n } mem;\n mi_memkind_t memkind;\n bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages)\n bool initially_committed;// `true` if the memory was originally allocated as committed\n bool initially_zero; // `true` if the memory was originally zero initialized\n} mi_memid_t;\n\n\nstatic inline bool mi_memid_is_os(mi_memid_t memid) {\n return mi_memkind_is_os(memid.memkind);\n}\n\nstatic inline bool mi_memid_needs_no_free(mi_memid_t memid) {\n return mi_memkind_needs_no_free(memid.memkind);\n}\n\nstatic inline mi_arena_t* mi_memid_arena(mi_memid_t memid) {\n return (memid.memkind == MI_MEM_ARENA ? memid.mem.arena.arena : NULL);\n}\n\n\n// ------------------------------------------------------\n// Mimalloc pages contain allocated blocks\n// ------------------------------------------------------\n\n// The free lists use encoded next fields\n// (Only actually encodes when MI_ENCODED_FREELIST is defined.)\ntypedef uintptr_t mi_encoded_t;\n\n// thread id's\ntypedef size_t mi_threadid_t;\n\n// free lists contain blocks\ntypedef struct mi_block_s {\n mi_encoded_t next;\n} mi_block_t;\n\n\n// The page flags are put in the bottom 2 bits of the thread_id (for a fast test in `mi_free`)\n// If `has_interior_pointers` is true if the page has pointers at an offset in a block (so we have to unalign to the block start before free-ing)\n// `in_full_queue` is true if the page is full and resides in the full queue (so we move it to a regular queue on free-ing)\n#define MI_PAGE_IN_FULL_QUEUE MI_ZU(0x01)\n#define MI_PAGE_HAS_INTERIOR_POINTERS MI_ZU(0x02)\n#define MI_PAGE_FLAG_MASK MI_ZU(0x03)\ntypedef size_t mi_page_flags_t;\n\n// There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue),\n// and 4 for abandoned & mapped threads -- abandoned-mapped pages are abandoned but also mapped\n// in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse.\n// Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked)\n#define MI_THREADID_ABANDONED MI_ZU(0)\n#define MI_THREADID_ABANDONED_MAPPED (MI_PAGE_FLAG_MASK + 1)\n\n// Thread free list.\n// Points to a list of blocks that are freed by other threads.\n// The least-bit is set if the page is owned by the current thread. (`mi_page_is_owned`).\n// Ownership is required before we can read any non-atomic fields in the page.\n// This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`.\ntypedef uintptr_t mi_thread_free_t;\n\n// A page contains blocks of one specific size (`block_size`).\n// Each page has three list of free blocks:\n// `free` for blocks that can be allocated,\n// `local_free` for freed blocks that are not yet available to `mi_malloc`\n// `thread_free` for freed blocks by other threads\n// The `local_free` and `thread_free` lists are migrated to the `free` list\n// when it is exhausted. The separate `local_free` list is necessary to\n// implement a monotonic heartbeat. The `thread_free` list is needed for\n// avoiding atomic operations when allocating from the owning thread.\n//\n// `used - |thread_free|` == actual blocks that are in use (alive)\n// `used - |thread_free| + |free| + |local_free| == capacity`\n//\n// We don't count \"freed\" (as |free|) but use only the `used` field to reduce\n// the number of memory accesses in the `mi_page_all_free` function(s).\n// Use `_mi_page_free_collect` to collect the thread_free list and update the `used` count.\n//\n// Notes:\n// - Non-atomic fields can only be accessed if having _ownership_ (low bit of `xthread_free` is 1).\n// Combining the `thread_free` list with an ownership bit allows a concurrent `free` to atomically\n// free an object and (re)claim ownership if the page was abandoned.\n// - If a page is not part of a theap it is called \"abandoned\" (`theap==NULL`) -- in\n// that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that\n// are in the `pages_abandoned` lists of an arena, these are called \"mapped\" abandoned pages).\n// - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`.\n// - The layout is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc`\n// - Using `uint16_t` does not seem to slow things down\n\ntypedef struct mi_page_s {\n _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`)\n\n mi_block_t* free; // list of available free blocks (`malloc` allocates from this list)\n uint16_t used; // number of blocks in use (including blocks in `thread_free`)\n uint16_t capacity; // number of blocks committed\n uint16_t reserved; // number of blocks reserved in memory\n uint8_t retire_expire; // expiration count for retired blocks\n bool free_is_zero; // `true` if the blocks in the free list are zero initialized\n\n mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)\n _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`)\n\n size_t block_size; // const: size available in each block (always `>0`)\n uint8_t* page_start; // const: start of the blocks\n\n #if (MI_ENCODE_FREELIST || MI_PADDING)\n uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary\n #endif\n\n mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages)\n mi_heap_t* heap; // const: the heap owning this page\n\n struct mi_page_s* next; // next page owned by the theap with the same `block_size`\n struct mi_page_s* prev; // previous page owned by the theap with the same `block_size`\n size_t slice_committed; // committed size relative to the first arena slice of the page data (or 0 if the page is fully committed already)\n mi_memid_t memid; // const: provenance of the page memory\n} mi_page_t;\n\n\n// ------------------------------------------------------\n// Object sizes\n// ------------------------------------------------------\n\n#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // pages must be aligned on this for the page map.\n#define MI_PAGE_MIN_START_BLOCK_ALIGN MI_MAX_ALIGN_SIZE // minimal block alignment for the first block in a page (16b)\n#define MI_PAGE_MAX_START_BLOCK_ALIGN2 (4*MI_KiB) // maximal block alignment for \"power of 2\"-sized blocks (such that we guarantee natural alignment)\n#define MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4*MI_KiB) // also aligns any multiple of this size to avoid TLB misses.\n#define MI_PAGE_MAX_OVERALLOC_ALIGN MI_ARENA_SLICE_SIZE // (64 KiB) limit for which we overallocate in arena pages, beyond this use OS allocation\n\n// The max object sizes are intended to not waste more than ~ 12.5% internally over the page sizes.\n#define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // = 10 KiB\n#if MI_ENABLE_LARGE_PAGES\n#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // ~ 84 KiB\n#define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/8) // <= 512 KiB // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`\n#else\n#define MI_MEDIUM_MAX_OBJ_SIZE (MI_MEDIUM_PAGE_SIZE/8) // <= 64 KiB\n#define MI_LARGE_MAX_OBJ_SIZE MI_MEDIUM_MAX_OBJ_SIZE // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`\n#endif\n#define MI_LARGE_MAX_OBJ_WSIZE (MI_LARGE_MAX_OBJ_SIZE/MI_SIZE_SIZE)\n\n#if (MI_LARGE_MAX_OBJ_WSIZE >= 655360)\n#error \"mimalloc internal: define more bins\"\n#endif\n\n\n// ------------------------------------------------------\n// Page kinds\n// ------------------------------------------------------\n\ntypedef enum mi_page_kind_e {\n MI_PAGE_SMALL, // small blocks go into 64KiB pages\n MI_PAGE_MEDIUM, // medium blocks go into 512KiB pages\n MI_PAGE_LARGE, // larger blocks go into 4MiB pages (if `MI_ENABLE_LARGE_PAGES==1`)\n MI_PAGE_SINGLETON // page containing a single block.\n // used for blocks `> MI_LARGE_MAX_OBJ_SIZE` or an alignment `> MI_PAGE_MAX_OVERALLOC_ALIGN`.\n} mi_page_kind_t;\n\n\n\n// ------------------------------------------------------\n// A \"theap\" is a thread local heap which owns pages.\n// (making them thread-local avoids atomic operations)\n//\n// All theaps belong to a (non-thread-local) heap.\n// A theap just owns a set of pages for allocation and\n// can only be allocate/reallocate from the thread that created it.\n// Freeing blocks can be done from any thread though.\n//\n// Per thread, there is always a default theap that belongs\n// to the default heap. It is initialized to statically\n// point initially to an empty theap to avoid initialization\n// checks in the fast path.\n// ------------------------------------------------------\n\n// Thread local data\ntypedef struct mi_tld_s mi_tld_t; // defined below\n\n// Pages of a certain block size are held in a queue.\ntypedef struct mi_page_queue_s {\n mi_page_t* first;\n mi_page_t* last;\n size_t count;\n size_t block_size;\n} mi_page_queue_t;\n\n// Random context\ntypedef struct mi_random_cxt_s {\n uint32_t input[16];\n uint32_t output[16];\n int output_available;\n bool weak;\n} mi_random_ctx_t;\n\n\n// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows\n#if MI_PADDING\ntypedef struct mi_padding_s {\n uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)\n uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)\n} mi_padding_t;\n#define MI_PADDING_SIZE (sizeof(mi_padding_t))\n#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE)\n#else\n#define MI_PADDING_SIZE 0\n#define MI_PADDING_WSIZE 0\n#endif\n\n#define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1)\n\n\n// A thread-local heap (\"theap\") owns a set of thread-local pages.\nstruct mi_theap_s {\n mi_tld_t* tld; // thread-local data\n _Atomic(mi_heap_t*) heap; // the heap this theap belongs to.\n _Atomic(size_t) refcount; // reference count\n unsigned long long heartbeat; // monotonic heartbeat count\n uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`)\n mi_random_ctx_t random; // random number context used for secure allocation\n size_t page_count; // total number of pages in the `pages` queues.\n size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues)\n size_t page_retired_max; // largest retired index into the `pages` array.\n size_t pages_full_size; // optimization: total size of blocks in the pages of the full queue (issue #1220)\n long generic_count; // how often is `_mi_malloc_generic` called?\n long generic_collect_count; // how often is `_mi_malloc_generic` called without collecting?\n\n mi_theap_t* tnext; // list of theaps in this thread\n mi_theap_t* tprev; \n mi_theap_t* hnext; // list of theaps of the owning `heap`\n mi_theap_t* hprev;\n \n long page_full_retain; // how many full pages can be retained per queue (before abandoning them)\n bool allow_page_reclaim; // `true` if this theap should not reclaim abandoned pages\n bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint\n #if MI_GUARDED\n size_t guarded_size_min; // minimal size for guarded objects\n size_t guarded_size_max; // maximal size for guarded objects\n size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages)\n size_t guarded_sample_count; // current sample count (counting down to 0)\n #endif\n mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.\n mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or \"bin\")\n mi_memid_t memid; // provenance of the theap struct itself (meta or os)\n mi_stats_t stats; // thread-local statistics\n};\n\n\n\n\n// ------------------------------------------------------\n// Heaps contain allocated blocks. Heaps are self-contained\n// but share the (sub-process) memory in the arena's.\n// ------------------------------------------------------\n\n// Keep track of all owned and abandoned pages in the arena's\nstruct mi_arena_pages_s;\ntypedef struct mi_arena_pages_s mi_arena_pages_t;\n\n#define MI_MAX_ARENAS (160) // Limited for now (and takes up .bss).. but arena's scale up exponentially (see `mi_arena_reserve`)\n // 160 arenas is enough for ~2 TiB memory\n\n// A dynamic thread-local variable; 0 for an invalid thread-local\ntypedef size_t mi_thread_local_t;\n\ntypedef struct mi_heap_s {\n mi_subproc_t* subproc; // a heap belongs to a subprocess\n size_t heap_seq; // unique sequence number for heaps in this subprocess\n mi_heap_t* next; // list of heaps in this subprocess\n mi_heap_t* prev;\n mi_thread_local_t theap; // dynamic thread local for the thread-local theaps of this heap\n\n mi_arena_t* exclusive_arena; // if the heap should only allocate from a specific arena (or NULL)\n int numa_node; // if >=0, prefer this numa node for allocations\n\n mi_theap_t* theaps; // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields)\n mi_lock_t theaps_lock; // lock for the theaps list operations\n\n _Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heap\n mi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arena\n mi_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations)\n\n _Atomic(mi_arena_pages_t*) arena_pages[MI_MAX_ARENAS]; // track owned and abandoned pages in the arenas (entries can be NULL)\n mi_lock_t arena_pages_lock; // lock to update the arena_pages array\n\n mi_stats_t stats; // statistics for this heap; periodically updated by merging from each theap\n} mi_heap_t;\n\n\n// ------------------------------------------------------\n// Sub processes do not reclaim or visit pages from other sub processes.\n// These are essentially the static variables of a process, and\n// usually there is only one subprocess. This can be used for example\n// by CPython to have separate interpreters within one process.\n// Each thread can only belong to one subprocess\n// (and needs to call `mi_subproc_add_current_thread` before any allocations).\n// ------------------------------------------------------\n\nstruct mi_subproc_s {\n size_t subproc_seq; // unique id for sub-processes\n mi_subproc_t* next; // list of all sub-processes\n mi_subproc_t* prev;\n\n _Atomic(size_t) arena_count; // current count of arena's\n _Atomic(mi_arena_t*) arenas[MI_MAX_ARENAS]; // arena's of this sub-process\n mi_lock_t arena_reserve_lock; // lock to ensure arena's get reserved one at a time\n mi_decl_align(8) // needed on some 32-bit platforms\n _Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged\n\n _Atomic(mi_heap_t*) heap_main; // main heap for this sub process\n mi_heap_t* heaps; // heaps belonging to this sub-process\n mi_lock_t heaps_lock;\n\n _Atomic(size_t) thread_count; // current threads associated with this sub-process\n _Atomic(size_t) thread_total_count; // total created threads associated with this sub-process\n _Atomic(size_t) heap_count; // current heaps in this sub-process (== |heaps|)\n _Atomic(size_t) heap_total_count; // total created heaps in this sub-process\n\n mi_memid_t memid; // provenance of this memory block (meta or static)\n mi_decl_align(8) // needed on some 32-bit platforms\n mi_stats_t stats; // subprocess statistics; updated for arena/OS stats like committed,\n // and otherwise merged with heap stats when those are deleted\n};\n\n\n// ------------------------------------------------------\n// Thread Local data\n// ------------------------------------------------------\n\n// Milliseconds as in `int64_t` to avoid overflows\ntypedef int64_t mi_msecs_t;\n\n// Thread local data\nstruct mi_tld_s {\n mi_threadid_t thread_id; // thread id of this thread\n size_t thread_seq; // thread sequence id (linear count of created threads)\n int numa_node; // thread preferred numa node\n mi_subproc_t* subproc; // sub-process this thread belongs to.\n mi_theap_t* theaps; // list of theaps in this thread (so we can abandon all when the thread terminates)\n mi_lock_t theaps_lock; // lock as the theaps list is sometimes accessed from another thread (on `mi_heap_free`)\n bool recurse; // true if deferred was called; used to prevent infinite recursion.\n bool is_in_threadpool; // true if this thread is part of a threadpool (and can run arbitrary tasks)\n mi_memid_t memid; // provenance of the tld memory itself (meta or OS)\n};\n\n\n/* ----------------------------------------------------------------------------\n Arenas are fixed area's of OS memory from which we can allocate\n large blocks (>= MI_ARENA_MIN_BLOCK_SIZE).\n In contrast to the rest of mimalloc, the arenas are shared between\n threads and need to be accessed using atomic operations (using atomic `mi_bitmap_t`'s).\n\n Arenas are also used to for huge OS page (1GiB) reservations or for reserving\n OS memory upfront which can be improve performance or is sometimes needed\n on embedded devices. We can also employ this with WASI or `sbrk` systems\n to reserve large arenas upfront and be able to reuse the memory more effectively.\n-----------------------------------------------------------------------------*/\n\n#define MI_ARENA_BIN_COUNT (MI_BIN_COUNT)\n#define MI_ARENA_MIN_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) // 32 MiB (or 8 MiB on 32-bit)\n#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE)\n\ntypedef struct mi_bitmap_s mi_bitmap_t; // atomic bitmap (defined in `src/bitmap.h`)\ntypedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in `src/bitmap.h`)\n\ntypedef struct mi_arena_pages_s {\n mi_bitmap_t* pages; // all registered pages (abandoned and owned)\n mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page)\n // followed by the bitmaps (whose siz`es depend on the arena size)\n} mi_arena_pages_t;\n\n\n// A memory arena\ntypedef struct mi_arena_s {\n mi_memid_t memid; // provenance of the memory area\n mi_subproc_t* subproc; // subprocess this arena belongs to (`this 'element-of' this->subproc->arenas`)\n size_t arena_idx; // index in the arenas array\n\n size_t slice_count; // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`)\n size_t info_slices; // initial slices reserved for the arena bitmaps\n int numa_node; // associated NUMA node\n bool is_exclusive; // only allow allocations if specifically for this arena\n mi_decl_align(8) // needed on some 32-bit platforms\n _Atomic(mi_msecs_t) purge_expire; // expiration time when slices can be purged from `slices_purge`.\n mi_commit_fun_t* commit_fun; // custom commit/decommit memory\n void* commit_fun_arg; // user argument for a custom commit function\n\n size_t total_size; // for (user given) memory more than MI_ARENA_MAX_SIZE, we use N arena's to cover it. The first (parent) has the total size (and the other sub-arena's 0).\n mi_arena_t* parent; // if this is a sub arena, this points to the first one in the memory area.\n\n mi_bbitmap_t* slices_free; // is the slice free? (a binned bitmap with size classes)\n mi_bitmap_t* slices_committed; // is the slice committed? (i.e. accessible)\n mi_bitmap_t* slices_dirty; // is the slice potentially non-zero?\n mi_bitmap_t* slices_purge; // slices that can be purged\n mi_page_t* pages_meta; // pre-allocated `slice_count` page meta info -- only used if `MI_PAGE_META_IS_SEPARATED!=0`\n mi_arena_pages_t pages_main; // arena page bitmaps for the main heap are allocated up front as well\n\n // followed by the bitmaps (whose sizes depend on the arena size)\n // note: when adding bitmaps revise `mi_arena_info_slices_needed`\n} mi_arena_t;\n\n\n\n/* -----------------------------------------------------------\n Error codes passed to `_mi_fatal_error`\n All are recoverable but EFAULT is a serious error and aborts by default in secure mode.\n For portability define undefined error codes using common Unix codes:\n \n----------------------------------------------------------- */\n\n#ifndef EAGAIN // double free\n#define EAGAIN (11)\n#endif\n#ifndef ENOMEM // out of memory\n#define ENOMEM (12)\n#endif\n#ifndef EFAULT // corrupted free-list or meta-data\n#define EFAULT (14)\n#endif\n#ifndef EINVAL // trying to free an invalid pointer\n#define EINVAL (22)\n#endif\n#ifndef EOVERFLOW // count*size overflow\n#define EOVERFLOW (75)\n#endif\n\n/* -----------------------------------------------------------\n Debug constants\n----------------------------------------------------------- */\n\n#if !defined(MI_DEBUG_UNINIT)\n#define MI_DEBUG_UNINIT (0xD0)\n#endif\n#if !defined(MI_DEBUG_FREED)\n#define MI_DEBUG_FREED (0xDF)\n#endif\n#if !defined(MI_DEBUG_PADDING)\n#define MI_DEBUG_PADDING (0xDE)\n#endif\n\n\n#endif // MI_TYPES_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/alloc-aligned.c", "sha256": "70420b4430da4bb6b5197fd27c6c0df485249c53a986bc9536c9fb643fc7613b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\" // _mi_theap_default\n\n#include // memset\n\n// ------------------------------------------------------\n// Aligned Allocation\n// ------------------------------------------------------\n\nstatic bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) {\n // certain blocks are always allocated at a certain natural alignment.\n // (see also `arena.c:mi_arenas_page_alloc_fresh`).\n mi_assert_internal(_mi_is_power_of_two(alignment) && (alignment > 0));\n if (alignment > size) return false;\n const size_t bsize = mi_good_size(size);\n const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two(bsize)) || // power-of-two under N\n (alignment==MI_PAGE_OSPAGE_BLOCK_ALIGN2 && (bsize % MI_PAGE_OSPAGE_BLOCK_ALIGN2)==0); // or multiple of N\n if (ok) { mi_assert_internal((bsize & (alignment-1)) == 0); } // since both power of 2 and alignment <= size\n return ok;\n}\n\n#if MI_GUARDED\nstatic mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero) mi_attr_noexcept {\n // use over allocation for guarded blocksl\n #if MI_THEAP_INITASNULL\n if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }\n #endif\n mi_assert_internal(alignment > 0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN);\n if mi_unlikely(alignment >= MI_PAGE_MAX_OVERALLOC_ALIGN || size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE - alignment)) {\n _mi_error_message(EOVERFLOW, \"(guarded) aligned allocation request is too large (size %zu, alignment %zu)\\n\", size, alignment);\n return NULL;\n }\n const size_t oversize = size + alignment - 1;\n void* const base = _mi_theap_malloc_guarded(theap, oversize, zero);\n if (base==NULL) return NULL;\n void* const p = _mi_align_up_ptr(base, alignment);\n mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size);\n mi_assert_internal(mi_usable_size(p) >= size);\n mi_assert_internal(_mi_is_aligned(p, alignment));\n return p;\n}\n\nstatic void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {\n #if MI_THEAP_INITASNULL\n if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }\n #endif\n const size_t rate = theap->guarded_sample_rate;\n // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty`\n if (rate != 0) { theap->guarded_sample_rate = 0; }\n void* p = _mi_theap_malloc_zero(theap, size, zero, usable);\n if (rate != 0) { theap->guarded_sample_rate = rate; }\n return p;\n}\n#else\nstatic void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {\n return _mi_theap_malloc_zero(theap, size, zero, usable);\n}\n#endif\n\n// Fallback aligned allocation that over-allocates -- split out for better codegen\nstatic mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept\n{\n mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE));\n mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));\n\n void* p;\n size_t oversize;\n if mi_unlikely(alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {\n // use OS allocation for large alignments and allocate inside a singleton page (not in an arena)\n // This can support alignments >= MI_PAGE_ALIGN by ensuring the object can be aligned\n // in the first (and single) page such that the page info is `MI_PAGE_ALIGN` bytes before it (and can be found in the _mi_page_map).\n if mi_unlikely(offset != 0) {\n // todo: cannot support offset alignment for very large alignments yet\n _mi_error_message(EOVERFLOW, \"aligned allocation with a large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\\n\", size, alignment, offset);\n return NULL;\n }\n oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size);\n // note: no guarded as alignment > 0\n p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, usable); // the page block size should be large enough to align in the single huge page block\n if (p == NULL) return NULL;\n }\n else {\n // otherwise over-allocate\n mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN);\n mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow\n oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that.\n p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, usable);\n if (p == NULL) return NULL;\n }\n\n // .. and align within the allocation\n const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)`\n const uintptr_t poffset = ((uintptr_t)p + offset) & align_mask;\n const uintptr_t adjust = (poffset == 0 ? 0 : alignment - poffset);\n mi_assert_internal(adjust < alignment);\n void* aligned_p = (void*)((uintptr_t)p + adjust);\n\n // note: after the above allocation, the page may be abandoned now (as it became full, see `page.c:_mi_malloc_generic`)\n // and we no longer own it. We should be careful to only read constant fields in the page,\n // or use safe atomic access as in `mi_page_set_has_interior_pointers`.\n // (we can access the page though since the just allocated pointer keeps it alive)\n mi_page_t* page = _mi_ptr_page(p);\n if (aligned_p != p) {\n mi_page_set_has_interior_pointers(page, true);\n #if MI_GUARDED\n // set tag to aligned so mi_usable_size works with guard pages\n if (adjust >= sizeof(mi_block_t)) {\n mi_block_t* const block = (mi_block_t*)p;\n block->next = MI_BLOCK_TAG_ALIGNED;\n }\n #endif\n _mi_padding_shrink(page, (mi_block_t*)p, adjust + size);\n }\n // todo: expand padding if overallocated ?\n\n mi_assert_internal(mi_page_usable_block_size(page) >= adjust + size);\n mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0);\n mi_assert_internal(mi_usable_size(aligned_p)>=size);\n mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust);\n #if MI_DEBUG > 1\n mi_page_t* const apage = _mi_ptr_page(aligned_p);\n void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p);\n mi_assert_internal(p == unalign_p);\n #endif\n\n // now zero the block if needed\n //if (alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {\n // // for the tracker, on huge aligned allocations only from the start of the large block is defined\n // mi_track_mem_undefined(aligned_p, size);\n // if (zero) {\n // _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p));\n // }\n //}\n\n if (p != aligned_p) {\n mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p));\n #if MI_GUARDED\n mi_track_mem_defined(p, sizeof(mi_block_t));\n #endif\n }\n return aligned_p;\n}\n\n// Generic primitive aligned allocation -- split out for better codegen\nstatic mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept\n{\n mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));\n // we don't allocate more than MI_MAX_ALLOC_SIZE (see )\n if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) {\n _mi_error_message(EOVERFLOW, \"aligned allocation request is too large (size %zu, alignment %zu)\\n\", size, alignment);\n return NULL;\n }\n\n // use regular allocation if it is guaranteed to fit the alignment constraints.\n // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist\n // a page with the right block size, and if we always use the over-alloc fallback that would never happen.\n if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) {\n void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, usable);\n mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);\n const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0;\n if mi_likely(is_aligned_or_null) {\n return p;\n }\n else {\n // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct..\n mi_assert(false);\n mi_free(p);\n }\n }\n\n // fall back to over-allocation\n return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,usable);\n}\n\n\n// Primitive aligned allocation\nstatic inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept\n{\n // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size.\n if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see )\n #if MI_DEBUG > 0\n _mi_error_message(EOVERFLOW, \"aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\\n\", size, alignment);\n #endif\n return NULL;\n }\n\n #if MI_GUARDED\n #if MI_THEAP_INITASNULL\n if mi_likely(theap!=NULL)\n #endif\n if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) {\n return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero);\n }\n #endif\n\n // try first if there happens to be a small block available with just the right alignment\n // since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already\n // naturally aligned this can be often the case.\n #if MI_THEAP_INITASNULL\n if mi_likely(theap!=NULL)\n #endif\n {\n if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) {\n const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`\n const size_t padsize = size + MI_PADDING_SIZE;\n mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize);\n if mi_likely(page->free != NULL) {\n const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;\n if mi_likely(is_aligned)\n {\n if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }\n void* p = _mi_page_malloc_zero(theap, page, padsize, zero);\n mi_assert_internal(p != NULL);\n mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);\n mi_track_malloc(p, size, zero);\n return p;\n }\n }\n }\n }\n\n // fallback to generic aligned allocation\n return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, usable);\n}\n\n\n// ------------------------------------------------------\n// Internal mi_theap_malloc_aligned / mi_malloc_aligned\n// ------------------------------------------------------\n\nstatic mi_decl_restrict void* mi_theap_malloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, false, NULL);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_theap_malloc_aligned_at(theap, size, alignment, 0);\n}\n\nstatic mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* theap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, true, NULL);\n}\n\nstatic mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_theap_zalloc_aligned_at(theap, size, alignment, 0);\n}\n\nstatic mi_decl_restrict void* mi_theap_calloc_aligned_at(mi_theap_t* theap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count, size, &total)) return NULL;\n return mi_theap_zalloc_aligned_at(theap, total, alignment, offset);\n}\n\nstatic mi_decl_restrict void* mi_theap_calloc_aligned(mi_theap_t* theap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_theap_calloc_aligned_at(theap, count, size, alignment, 0);\n}\n\n\n// ------------------------------------------------------\n// Aligned Allocation\n// ------------------------------------------------------\n\nmi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_malloc_aligned_at(_mi_theap_default(), size, alignment, offset);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {\n return mi_theap_malloc_aligned(_mi_theap_default(), size, alignment);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {\n return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, block_size);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_zalloc_aligned_at(_mi_theap_default(), size, alignment, offset);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {\n return mi_theap_zalloc_aligned(_mi_theap_default(), size, alignment);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {\n return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, block_size);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_calloc_aligned_at(_mi_theap_default(), count, size, alignment, offset);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_theap_calloc_aligned(_mi_theap_default(), count, size, alignment);\n}\n\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_malloc_aligned_at(_mi_heap_theap(heap), size, alignment, offset);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_theap_malloc_aligned(_mi_heap_theap(heap), size, alignment);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_zalloc_aligned_at(_mi_heap_theap(heap), size, alignment, offset);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_theap_zalloc_aligned(_mi_heap_theap(heap), size, alignment);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_calloc_aligned_at(_mi_heap_theap(heap), count, size, alignment, offset);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_theap_calloc_aligned(_mi_heap_theap(heap), count, size, alignment);\n}\n\n\n// ------------------------------------------------------\n// Aligned re-allocation\n// ------------------------------------------------------\n\nstatic void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept {\n mi_assert(alignment > 0);\n if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);\n if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL);\n size_t size = mi_usable_size(p);\n if (newsize <= size && newsize >= (size - (size / 2))\n && (((uintptr_t)p + offset) % alignment) == 0) {\n return p; // reallocation still fits, is aligned and not more than 25% waste\n }\n else {\n // note: we don't zero allocate upfront so we only zero initialize the expanded part\n void* newp = mi_theap_malloc_aligned_at(theap,newsize,alignment,offset);\n if (newp != NULL) {\n if (zero && newsize > size) {\n // also set last word in the previous allocation to zero to ensure any padding is zero-initialized\n size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);\n _mi_memzero((uint8_t*)newp + start, newsize - start);\n }\n _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));\n mi_free(p); // only free if successful\n }\n return newp;\n }\n}\n\nstatic void* mi_theap_realloc_zero_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept {\n mi_assert(alignment > 0);\n if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);\n return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,0,zero);\n}\n\nstatic void* mi_theap_realloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,offset,false);\n}\n\nstatic void* mi_theap_realloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {\n return mi_theap_realloc_zero_aligned(theap,p,newsize,alignment,false);\n}\n\nstatic void* mi_theap_rezalloc_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_realloc_zero_aligned_at(theap, p, newsize, alignment, offset, true);\n}\n\nstatic void* mi_theap_rezalloc_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {\n return mi_theap_realloc_zero_aligned(theap, p, newsize, alignment, true);\n}\n\nstatic void* mi_theap_recalloc_aligned_at(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(newcount, size, &total)) return NULL;\n return mi_theap_rezalloc_aligned_at(theap, p, total, alignment, offset);\n}\n\nstatic void* mi_theap_recalloc_aligned(mi_theap_t* theap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(newcount, size, &total)) return NULL;\n return mi_theap_rezalloc_aligned(theap, p, total, alignment);\n}\n\n\nmi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_realloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {\n return mi_theap_realloc_aligned(_mi_theap_default(), p, newsize, alignment);\n}\n\nmi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_rezalloc_aligned_at(_mi_theap_default(), p, newsize, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {\n return mi_theap_rezalloc_aligned(_mi_theap_default(), p, newsize, alignment);\n}\n\nmi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_recalloc_aligned_at(_mi_theap_default(), p, newcount, size, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_theap_recalloc_aligned(_mi_theap_default(), p, newcount, size, alignment);\n}\n\n\nmi_decl_nodiscard void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_realloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {\n return mi_theap_realloc_aligned(_mi_heap_theap(heap), p, newsize, alignment);\n}\n\nmi_decl_nodiscard void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_rezalloc_aligned_at(_mi_heap_theap(heap), p, newsize, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {\n return mi_theap_rezalloc_aligned(_mi_heap_theap(heap), p, newsize, alignment);\n}\n\nmi_decl_nodiscard void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {\n return mi_theap_recalloc_aligned_at(_mi_heap_theap(heap), p, newcount, size, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {\n return mi_theap_recalloc_aligned(_mi_heap_theap(heap), p, newcount, size, alignment);\n}\n\n\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/alloc-override.c", "sha256": "6c8e9b0d042481c55791edb7330daec64bedaf26f0e70c9ca40de3cc17a690fe", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2026, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n#if !defined(MI_IN_ALLOC_C)\n#error \"this file should be included from 'alloc.c' (so aliases can work)\"\n#endif\n\n\n#if defined(MI_MALLOC_OVERRIDE) && !defined(_DLL)\n\n#if defined(__APPLE__)\n#include \nmi_decl_externc void vfree(void* p);\nmi_decl_externc size_t malloc_size(const void* p);\nmi_decl_externc size_t malloc_good_size(size_t size);\n#endif\n\n// helper definition for C override of C++ new\ntypedef void* mi_nothrow_t;\n\n// ------------------------------------------------------\n// Override system malloc\n// ------------------------------------------------------\n\n#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !MI_TRACK_ENABLED\n // gcc, clang: use aliasing to alias the exported function to one of our `mi_` functions\n #if (defined(__GNUC__) && __GNUC__ >= 9)\n #pragma GCC diagnostic ignored \"-Wattributes\" // or we get warnings that nodiscard is ignored on a forward\n #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility(\"default\"), copy(fun)));\n #else\n #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility(\"default\")));\n #endif\n #define MI_FORWARD1(fun,x) MI_FORWARD(fun)\n #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun)\n #define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun)\n #define MI_FORWARD0(fun,x) MI_FORWARD(fun)\n #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun)\n#else\n // otherwise use forwarding by calling our `mi_` function\n #define MI_FORWARD1(fun,x) { return fun(x); }\n #define MI_FORWARD2(fun,x,y) { return fun(x,y); }\n #define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); }\n #define MI_FORWARD0(fun,x) { fun(x); }\n #define MI_FORWARD02(fun,x,y) { fun(x,y); }\n#endif\n\n#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE)\n // define MI_OSX_IS_INTERPOSED as we should not provide forwarding definitions for\n // functions that are interposed (or the interposing does not work)\n #define MI_OSX_IS_INTERPOSED\n\n mi_decl_externc size_t mi_malloc_size_checked(void *p) {\n if (!mi_is_in_heap_region(p)) return 0;\n return mi_usable_size(p);\n }\n\n // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1`\n // See: \n struct mi_interpose_s {\n const void* replacement;\n const void* target;\n };\n #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun }\n #define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun)\n\n #define MI_INTERPOSE_DECLS(name) __attribute__((used)) static struct mi_interpose_s name[] __attribute__((section(\"__DATA, __interpose\")))\n\n MI_INTERPOSE_DECLS(_mi_interposes) =\n {\n MI_INTERPOSE_MI(malloc),\n MI_INTERPOSE_MI(calloc),\n MI_INTERPOSE_MI(realloc),\n MI_INTERPOSE_MI(strdup),\n MI_INTERPOSE_MI(realpath),\n MI_INTERPOSE_MI(posix_memalign),\n MI_INTERPOSE_MI(reallocf),\n MI_INTERPOSE_MI(valloc),\n MI_INTERPOSE_FUN(malloc_size,mi_malloc_size_checked),\n MI_INTERPOSE_MI(malloc_good_size),\n #ifdef MI_OSX_ZONE\n // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely\n MI_INTERPOSE_MI(free),\n MI_INTERPOSE_FUN(vfree,mi_free),\n #else\n // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity )\n MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us\n MI_INTERPOSE_FUN(vfree,mi_cfree),\n #endif\n };\n #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)\n MI_INTERPOSE_DECLS(_mi_interposes_10_7) = { MI_INTERPOSE_MI(strndup) };\n #endif\n #if defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15)\n MI_INTERPOSE_DECLS(_mi_interposes_10_15) = { MI_INTERPOSE_MI(aligned_alloc) };\n #endif\n\n #ifdef __cplusplus\n extern \"C\" {\n #endif\n void _ZdlPv(void* p); // delete\n void _ZdaPv(void* p); // delete[]\n void _ZdlPvm(void* p, size_t n); // delete\n void _ZdaPvm(void* p, size_t n); // delete[]\n void* _Znwm(size_t n); // new\n void* _Znam(size_t n); // new[]\n void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow\n void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow\n #ifdef __cplusplus\n }\n #endif\n __attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[] __attribute__((section(\"__DATA, __interpose\"))) =\n {\n MI_INTERPOSE_FUN(_ZdlPv,mi_free),\n MI_INTERPOSE_FUN(_ZdaPv,mi_free),\n MI_INTERPOSE_FUN(_ZdlPvm,mi_free_size),\n MI_INTERPOSE_FUN(_ZdaPvm,mi_free_size),\n MI_INTERPOSE_FUN(_Znwm,mi_new),\n MI_INTERPOSE_FUN(_Znam,mi_new),\n MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow),\n MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow),\n };\n\n#elif defined(_MSC_VER)\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_HYBRIDPATCHABLE\n void* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) {\n return mi_expand(_Block, _Size);\n }\n _Check_return_ _ACRTIMP \n size_t __cdecl _msize_base(_Pre_notnull_ void* _Block) _CRT_NOEXCEPT {\n return mi_malloc_size(_Block);\n }\n _Check_return_ _ACRTIMP _CRT_HYBRIDPATCHABLE \n size_t __cdecl _msize(_Pre_notnull_ void* _Block) {\n return mi_malloc_size(_Block);\n }\n _ACRTIMP \n void __cdecl _free_base(_Pre_maybenull_ _Post_invalid_ void* _Block) {\n mi_free(_Block);\n }\n _ACRTIMP _CRT_HYBRIDPATCHABLE \n void __cdecl free(_Pre_maybenull_ _Post_invalid_ void* _Block) {\n mi_free(_Block);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_JIT_INTRINSIC _CRTRESTRICT _CRT_HYBRIDPATCHABLE\n void* __cdecl malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size) {\n return mi_malloc(_Size);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _malloc_base(_In_ size_t _Size) {\n return mi_malloc(_Size);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _realloc_base(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ size_t _Size) {\n return mi_realloc(_Block, _Size);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT _CRT_HYBRIDPATCHABLE\n void* __cdecl realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) {\n return mi_realloc(_Block, _Size);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT \n void* __cdecl _calloc_base(_In_ size_t _Count, _In_ size_t _Size) {\n return mi_calloc(_Count, _Size);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRT_JIT_INTRINSIC _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl calloc(_In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) {\n return mi_calloc(_Count, _Size);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _recalloc_base(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ size_t _Count, _In_ size_t _Size) {\n return mi_recalloc(_Block, _Count, _Size);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size) {\n return mi_recalloc(_Block, _Count, _Size);\n }\n _ACRTIMP \n void __cdecl _aligned_free(_Pre_maybenull_ _Post_invalid_ void* _Block) {\n mi_free(_Block);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {\n return mi_malloc_aligned(_Size, _Alignment);\n }\n _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_offset_malloc(_In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {\n return mi_malloc_aligned_at(_Size, _Alignment, _Offset);\n }\n _Check_return_ _ACRTIMP \n size_t __cdecl _aligned_msize(_Pre_notnull_ void* _Block, _In_ size_t _Alignment, _In_ size_t _Offset) {\n MI_UNUSED(_Alignment); MI_UNUSED(_Offset); return mi_malloc_size(_Block);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_offset_realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {\n return mi_realloc_aligned_at(_Block, _Size, _Alignment, _Offset);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_offset_recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment, _In_ size_t _Offset) {\n return mi_recalloc_aligned_at(_Block, _Count, _Size, _Alignment, _Offset);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_realloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {\n return mi_realloc_aligned(_Block, _Size, _Alignment);\n }\n _Success_(return != 0) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Count * _Size) _ACRTIMP _CRTALLOCATOR _CRTRESTRICT\n void* __cdecl _aligned_recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) {\n return mi_recalloc_aligned(_Block, _Count, _Size, _Alignment);\n }\n#else\n // On all other systems forward allocation primitives to our API\n mi_decl_export void* malloc(size_t size) MI_FORWARD1(mi_malloc, size)\n mi_decl_export void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n)\n mi_decl_export void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize)\n mi_decl_export void free(void* p) MI_FORWARD0(mi_free, p)\n // In principle we do not need to forward `strdup`/`strndup` but on some systems these do not use `malloc` internally (but a more primitive call)\n // We only override if `strdup` is not a macro (as on some older libc's, see issue #885)\n #if !defined(strdup)\n mi_decl_export char* strdup(const char* str) MI_FORWARD1(mi_strdup, str)\n #endif\n #if !defined(strndup) && (!defined(__APPLE__) || (defined(MAC_OS_X_VERSION_10_7) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7))\n mi_decl_export char* strndup(const char* str, size_t n) MI_FORWARD2(mi_strndup, str, n)\n #endif\n#endif\n\n#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)\n#pragma GCC visibility push(default)\n#endif\n\n// ------------------------------------------------------\n// Override new/delete\n// This is not really necessary as they usually call\n// malloc/free anyway, but it improves performance.\n// ------------------------------------------------------\n#ifdef __cplusplus\n // ------------------------------------------------------\n // With a C++ compiler we override the new/delete operators.\n // see \n // ------------------------------------------------------\n #include \n\n #ifndef MI_OSX_IS_INTERPOSED\n void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p)\n void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p)\n\n void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n)\n void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n)\n\n void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); }\n void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); }\n\n #if (__cplusplus >= 201402L || _MSC_VER >= 1916)\n void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)\n void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)\n #endif\n #endif\n\n #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5))\n void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); };\n void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); };\n void operator delete (void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); }\n void operator delete[](void* p, std::align_val_t al, const std::nothrow_t&) noexcept { mi_free_aligned(p, static_cast(al)); }\n\n void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); }\n void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); }\n void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); }\n void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); }\n #endif\n\n#elif (defined(__GNUC__) || defined(__clang__))\n // ------------------------------------------------------\n // Override by defining the mangled C++ names of the operators (as\n // used by GCC and CLang).\n // See \n // ------------------------------------------------------\n\n void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete\n void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[]\n void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)\n void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)\n\n void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); }\n void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); }\n void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }\n void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }\n\n void _ZdlPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete(void*, std::nothrow_t const&)\n void _ZdaPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete[](void*, std::nothrow_t const&)\n void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&)\n void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&)\n\n #if (MI_INTPTR_SIZE==8) || (MI_INTPTR_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257\n void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit\n void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit\n void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }\n void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }\n void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)\n void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)\n void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }\n void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }\n #elif (MI_INTPTR_SIZE==4)\n void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit\n void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit\n void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }\n void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }\n void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)\n void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)\n void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }\n void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }\n #else\n #error \"define overloads for new/delete for this platform (just for performance, can be skipped)\"\n #endif\n#endif // __cplusplus\n\n// ------------------------------------------------------\n// Further Posix & Unix functions definitions\n// ------------------------------------------------------\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#ifndef MI_OSX_IS_INTERPOSED\n // Forward Posix/Unix calls as well\n void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize)\n size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p)\n #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__)\n size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p)\n #else\n size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p)\n #endif\n\n // No forwarding here due to aliasing/name mangling issues\n void* valloc(size_t size) { return mi_valloc(size); }\n void vfree(void* p) { mi_free(p); }\n size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); }\n int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); }\n\n // `aligned_alloc` is only available when __USE_ISOC11 is defined.\n // Note: it seems __USE_ISOC11 is not defined in musl (and perhaps other libc's) so we only check\n // for it if using glibc.\n // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot\n // override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9.\n // Fortunately, in the case where `aligned_alloc` is declared as `static inline` it\n // uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves.\n #if !defined(__GLIBC__) || __USE_ISOC11\n void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); }\n #endif\n#endif\n\n// no forwarding here due to aliasing/name mangling issues\nvoid cfree(void* p) { mi_free(p); }\nvoid* pvalloc(size_t size) { return mi_pvalloc(size); }\nvoid* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); }\n#if !defined(_WIN32)\nvoid* _aligned_malloc(size_t size, size_t alignment) { return mi_malloc_aligned(size,alignment); }\n#endif\nvoid* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); }\n// some systems define reallocarr so mark it as a weak symbol (#751)\nmi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); }\n\n#if defined(__wasi__)\n // forward __libc interface (see PR #667)\n void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc, size)\n void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc, count, size)\n void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc, p, size)\n void __libc_free(void* p) MI_FORWARD0(mi_free, p)\n void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); }\n\n#elif defined(__linux__)\n // forward __libc interface (needed for glibc-based and musl-based Linux distributions)\n void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size)\n void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size)\n void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size)\n void __libc_free(void* p) MI_FORWARD0(mi_free,p)\n void __libc_cfree(void* p) MI_FORWARD0(mi_free,p)\n\n void* __libc_valloc(size_t size) { return mi_valloc(size); }\n void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); }\n void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); }\n int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); }\n#endif\n\n#ifdef __cplusplus\n}\n#endif\n\n#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)\n#pragma GCC visibility pop\n#endif\n\n#endif // MI_MALLOC_OVERRIDE\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/alloc-posix.c", "sha256": "98d0eb877d7b9cdf7e9f857aaf3c739b69f746b91221e0e427e82bdd7cec90b3", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2021, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// ------------------------------------------------------------------------\n// mi prefixed publi definitions of various Posix, Unix, and C++ functions\n// for convenience and used when overriding these functions.\n// ------------------------------------------------------------------------\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n\n// ------------------------------------------------------\n// Posix & Unix functions definitions\n// ------------------------------------------------------\n\n#include \n#include // memset\n#include // getenv\n\n#ifdef _MSC_VER\n#pragma warning(disable:4996) // getenv _wgetenv\n#endif\n\n#ifndef EINVAL\n#define EINVAL 22\n#endif\n#ifndef ENOMEM\n#define ENOMEM 12\n#endif\n\n\nmi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept {\n // if (!mi_is_in_heap_region(p)) return 0;\n return mi_usable_size(p);\n}\n\nmi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {\n // if (!mi_is_in_heap_region(p)) return 0;\n return mi_usable_size(p);\n}\n\nmi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept {\n return mi_good_size(size);\n}\n\nvoid mi_cfree(void* p) mi_attr_noexcept {\n if (mi_is_in_heap_region(p)) {\n mi_free(p);\n }\n}\n\nint mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept {\n // Note: The spec dictates we should not modify `*p` on an error. (issue#27)\n // \n if (p == NULL) return EINVAL;\n if ((alignment % sizeof(void*)) != 0) return EINVAL; // natural alignment\n // it is also required that alignment is a power of 2 and > 0; this is checked in `mi_malloc_aligned`\n if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2\n void* q = mi_malloc_aligned(size, alignment);\n if (q==NULL && size != 0) return ENOMEM;\n mi_assert_internal(_mi_is_aligned(q,alignment));\n *p = q;\n return 0;\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {\n void* p = mi_malloc_aligned(size, alignment);\n mi_assert_internal(_mi_is_aligned(p,alignment));\n return p;\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept {\n return mi_memalign( _mi_os_page_size(), size );\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept {\n size_t psize = _mi_os_page_size();\n if (size >= SIZE_MAX - psize) return NULL; // overflow\n size_t asize = _mi_align_up(size, psize);\n return mi_malloc_aligned(asize, psize);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {\n // C11 requires the size to be an integral multiple of the alignment, see .\n // unfortunately, it turns out quite some programs pass a size that is not an integral multiple so skip this check..\n /* if mi_unlikely((size & (alignment - 1)) != 0) { // C11 requires alignment>0 && integral multiple, see \n #if MI_DEBUG > 0\n _mi_error_message(EOVERFLOW, \"(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\\n\", size, alignment);\n #endif\n return NULL;\n }\n */\n // C11 also requires alignment to be a power-of-two (and > 0) which is checked in mi_malloc_aligned\n void* p = mi_malloc_aligned(size, alignment);\n mi_assert_internal(_mi_is_aligned(p,alignment));\n return p;\n}\n\nmi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD \n size_t total;\n if mi_unlikely(mi_count_size_overflow(count, size, &total)) {\n errno = EOVERFLOW;\n return NULL;\n }\n void* newp = mi_realloc(p,total);\n if (newp==NULL) { errno = ENOMEM; }\n return newp;\n}\n\nmi_decl_nodiscard int mi_reallocarr( void* ptrp, size_t count, size_t size ) mi_attr_noexcept { // NetBSD \n mi_assert(size != 0);\n mi_assert(ptrp != NULL);\n if (ptrp == NULL || size == 0) {\n return (errno = EINVAL);\n }\n size_t total;\n if mi_unlikely(mi_count_size_overflow(count, size, &total)) {\n return (errno = EOVERFLOW);\n }\n void** op = (void**)ptrp;\n if (total == 0) {\n free(*op);\n *op = NULL;\n return 0;\n }\n else {\n void* newp = mi_realloc(*op,total);\n if (newp == NULL) { return (errno = ENOMEM); }\n *op = newp;\n return 0;\n }\n}\n\nvoid* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft\n void* res = mi_expand(p, newsize);\n if (res == NULL) { errno = ENOMEM; }\n return res;\n}\n\nmi_decl_nodiscard mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept {\n if (s==NULL) return NULL;\n size_t wlen;\n for(wlen = 0; s[wlen] != 0 && wlen < PTRDIFF_MAX; wlen++) { } // prevent overflow on wlen+1\n size_t size; \n if (mi_mul_overflow(wlen+1, sizeof(wchar_t), &size) || size > PTRDIFF_MAX) return NULL;\n wchar_t* p = (wchar_t*)mi_malloc(size);\n if (p != NULL) {\n _mi_memcpy(p,s,size);\n }\n return p;\n}\n\nmi_decl_nodiscard mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept {\n return (unsigned char*)mi_strdup((const char*)s);\n}\n\nint mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept {\n if (size != NULL) *size = 0;\n if (buf==NULL || name==NULL) return EINVAL;\n char* p = getenv(name);\n if (p==NULL) {\n *buf = NULL;\n }\n else {\n *buf = mi_strdup(p);\n if (*buf==NULL) return ENOMEM;\n if (size != NULL) { *size = _mi_strlen(p) + 1; } // cannot overflow as mi_strdup is limited to PTRDIFF_MAX\n }\n return 0;\n}\n\nint mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept {\n if (size != NULL) *size = 0;\n if (buf==NULL || name==NULL) return EINVAL; \n#if !defined(_WIN32) || (defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP))\n // not supported\n *buf = NULL;\n return EINVAL;\n#else\n wchar_t* p = (wchar_t*)_wgetenv(name);\n if (p==NULL) {\n *buf = NULL;\n }\n else {\n *buf = mi_wcsdup(p);\n if (*buf==NULL) return ENOMEM;\n if (size != NULL) { *size = wcslen(p) + 1; } // cannot overflow as wcsdup is limited to PTRDIFF_MAX\n }\n return 0;\n#endif\n}\n\nmi_decl_nodiscard void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft\n return mi_recalloc_aligned_at(p, newcount, size, alignment, offset);\n}\n\nmi_decl_nodiscard void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft\n return mi_recalloc_aligned(p, newcount, size, alignment);\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/alloc.c", "sha256": "a92d58e47220e5b6d138b83d92498771be4195ee59136a134e4461f6f726b9b8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "\n/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#ifndef _DEFAULT_SOURCE\n#define _DEFAULT_SOURCE // for realpath() on Linux\n#endif\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\" // _mi_prim_thread_id()\n\n#include // memset, strlen (for mi_strdup)\n#include // malloc, abort\n\n#define MI_IN_ALLOC_C\n#include \"alloc-override.c\"\n#include \"free.c\"\n#undef MI_IN_ALLOC_C\n\n// ------------------------------------------------------\n// Allocation\n// ------------------------------------------------------\n\n// Fast allocation in a page: just pop from the free list.\n// Fall back to generic allocation only if the list is empty.\n// Note: in release mode the (inlined) routine is about 7 instructions with a single test.\nstatic mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, size_t* usable) mi_attr_noexcept\n{\n if (page->block_size != 0) { // not the empty theap\n mi_assert_internal(mi_page_block_size(page) >= size);\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);\n }\n\n // check the free list\n mi_block_t* const block = page->free;\n if mi_unlikely(block == NULL) {\n return _mi_malloc_generic(theap, size, (zero ? 1 : 0), usable);\n }\n mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);\n if (usable != NULL) { *usable = mi_page_usable_block_size(page); };\n\n // pop from the free list\n page->free = mi_block_next(page, block);\n page->used++;\n mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);\n mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE));\n\n #if MI_DEBUG>3\n if (page->free_is_zero && size > sizeof(*block)) {\n mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block)));\n }\n #endif\n\n // allow use of the block internally\n // note: when tracking we need to avoid ever touching the MI_PADDING since\n // that is tracked by valgrind etc. as non-accessible (through the red-zone, see `mimalloc/track.h`)\n const size_t bsize = mi_page_usable_block_size(page);\n mi_track_mem_undefined(block, bsize);\n\n #if (MI_STAT>0)\n if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {\n mi_theap_stat_increase(theap, malloc_normal, bsize);\n #if (MI_STAT>1)\n mi_theap_stat_counter_increase(theap, malloc_normal_count, 1);\n const size_t bin = _mi_bin(bsize);\n mi_theap_stat_increase(theap, malloc_bins[bin], 1);\n mi_theap_stat_increase(theap, malloc_requested, size - MI_PADDING_SIZE);\n #endif\n }\n #endif\n\n // zero the block? note: we need to zero the full block size (issue #63)\n if mi_likely(!zero) {\n // #if MI_SECURE\n block->next = 0; // don't leak internal data\n // #endif\n #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN\n if (!mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); }\n #endif \n }\n else {\n if (!page->free_is_zero) {\n _mi_memzero_aligned(block, bsize);\n }\n else {\n block->next = 0;\n mi_track_mem_defined(block, bsize);\n } \n }\n\n #if MI_PADDING // && !MI_TRACK_ENABLED\n mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize);\n ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE));\n #if (MI_DEBUG>=2)\n mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta));\n #endif\n mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess\n padding->canary = mi_ptr_encode_canary(page,block,page->keys);\n padding->delta = (uint32_t)(delta);\n #if MI_PADDING_CHECK\n if (!mi_page_is_huge(page)) {\n uint8_t* fill = (uint8_t*)padding - delta;\n const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes\n for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }\n }\n #endif\n #endif\n\n return block;\n}\n\n// extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`.\nextern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept {\n return mi_page_malloc_zero(theap, page, size, zero, NULL);\n}\n\n#if MI_GUARDED\nmi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept;\n#endif\n\n// main allocation primitives for small and generic allocation\n\n// internal small size allocation\nstatic mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept\n{\n mi_assert(theap != NULL);\n mi_assert(size <= MI_SMALL_SIZE_MAX);\n #if MI_DEBUG\n const uintptr_t tid = _mi_thread_id();\n mi_assert(theap->tld->thread_id == 0 || theap->tld->thread_id == tid); // theaps are thread local\n #endif\n #if (MI_PADDING || MI_GUARDED)\n if mi_unlikely(size == 0) { size = sizeof(void*); }\n #endif\n #if MI_GUARDED\n if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) {\n return _mi_theap_malloc_guarded(theap, size, zero);\n }\n #endif\n\n // get page in constant time, and allocate from it\n mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE);\n void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, usable);\n mi_track_malloc(p,size,zero);\n\n #if MI_DEBUG>3\n if (p != NULL && zero) {\n mi_assert_expensive(mi_mem_is_zero(p, size));\n }\n #endif\n return p;\n}\n\n// internal generic allocation\nstatic mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept\n{\n #if MI_GUARDED\n #if MI_THEAP_INITASNULL\n if (theap!=NULL)\n #endif\n if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) {\n return _mi_theap_malloc_guarded(theap, size, zero);\n }\n #endif\n #if !MI_THEAP_INITASNULL\n mi_assert(theap!=NULL);\n #endif\n mi_assert(theap==NULL || theap->tld->thread_id == 0 || theap->tld->thread_id == _mi_thread_id()); // theaps are thread local\n mi_assert((huge_alignment & 1)==0);\n void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, usable); // note: size can overflow but it is detected in malloc_generic\n mi_track_malloc(p, size, zero);\n\n #if MI_DEBUG>3\n if (p != NULL && zero) {\n mi_assert_expensive(mi_mem_is_zero(p, size));\n }\n #endif\n return p;\n}\n\n// internal small allocation\nstatic mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {\n #if MI_THEAP_INITASNULL\n if (theap!=NULL) {\n return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);\n }\n else {\n return mi_theap_malloc_generic(theap, size, zero, 0, usable); // tailcall\n }\n #else\n return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);\n #endif\n}\n\n\n// allocate a small block\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {\n return mi_theap_malloc_small_zero(theap, size, false, NULL);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept {\n return mi_theap_malloc_small(_mi_theap_default(), size);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {\n return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, false, NULL);\n}\n\n// The main internal allocation functions\nstatic mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {\n // fast path for small objects\n if mi_likely(size <= MI_SMALL_SIZE_MAX) {\n mi_assert_internal(huge_alignment == 0);\n return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);\n }\n else {\n return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);\n }\n}\n\nextern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {\n // fast path for small objects\n #if MI_THEAP_INITASNULL\n if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX)\n #else\n if mi_likely(size <= MI_SMALL_SIZE_MAX)\n #endif\n {\n mi_assert_internal(huge_alignment == 0);\n return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);\n }\n else {\n return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);\n }\n}\n\nvoid* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {\n return _mi_theap_malloc_zero_ex(theap, size, zero, 0, usable);\n}\n\n\n// Main allocation functions\n\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {\n return _mi_theap_malloc_zero(theap, size, false, NULL);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept {\n return mi_theap_malloc(_mi_theap_default(), size);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {\n return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL);\n}\n\n\n// zero initialized small block\nmi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {\n return mi_theap_malloc_small_zero(_mi_theap_default(), size, true, NULL);\n}\n\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {\n return mi_theap_malloc_small_zero(theap, size, true, NULL);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {\n return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL);\n}\n\n\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {\n return _mi_theap_malloc_zero(theap, size, true, NULL);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept {\n return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {\n return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL);\n}\n\nmi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count,size,&total)) return NULL;\n return mi_theap_zalloc(theap,total);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept {\n return mi_theap_calloc(_mi_theap_default(),count,size);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count, size, &total)) return NULL;\n return mi_heap_zalloc(heap, total);\n}\n\n// Return usable size\nmi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* usable) mi_attr_noexcept {\n return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, usable);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_theap_umalloc(mi_theap_t* theap, size_t size, size_t* usable) mi_attr_noexcept {\n return _mi_theap_malloc_zero_ex(theap, size, false, 0, usable);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* usable) mi_attr_noexcept {\n return mi_theap_umalloc(_mi_theap_default(), size, usable);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* usable) mi_attr_noexcept {\n return _mi_theap_malloc_zero_ex(_mi_theap_default(), size, true, 0, usable);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* usable) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count,size,&total)) return NULL;\n return mi_uzalloc(total, usable);\n}\n\n// Uninitialized `calloc`\nstatic mi_decl_restrict void* mi_theap_mallocn(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count, size, &total)) return NULL;\n return mi_theap_malloc(theap, total);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {\n return mi_theap_mallocn(_mi_theap_default(),count,size);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count, size, &total)) return NULL;\n return mi_heap_malloc(heap, total);\n}\n\n\n// Expand (or shrink) in place (or fail)\nvoid* mi_expand(void* p, size_t newsize) mi_attr_noexcept {\n #if MI_PADDING\n // we do not shrink/expand with padding enabled\n MI_UNUSED(p); MI_UNUSED(newsize);\n return NULL;\n #else\n if (p == NULL) return NULL;\n const mi_page_t* const page = mi_validate_ptr_page(p,\"mi_expand\");\n const size_t size = _mi_usable_size(p,page);\n if (newsize > size) return NULL;\n return p; // it fits\n #endif\n}\n\nvoid* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {\n // if p == NULL then behave as malloc.\n // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).\n // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.)\n const mi_page_t* page;\n size_t size;\n if (p==NULL) {\n page = NULL;\n size = 0;\n if (usable_pre!=NULL) { *usable_pre = 0; }\n }\n else {\n page = mi_validate_ptr_page(p,\"mi_realloc\");\n size = _mi_usable_size(p,page);\n if (usable_pre!=NULL) { *usable_pre = mi_page_usable_block_size(page); }\n }\n if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0 // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)\n && mi_page_heap(page)==_mi_theap_heap(theap)) // and within the same heap\n {\n mi_assert_internal(p!=NULL);\n // todo: do not track as the usable size is still the same in the free; adjust potential padding?\n // mi_track_resize(p,size,newsize)\n // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }\n if (usable_post!=NULL) { *usable_post = mi_page_usable_block_size(page); }\n return p; // reallocation still fits and not more than 50% waste\n }\n void* newp = mi_theap_umalloc(theap,newsize,usable_post);\n if mi_likely(newp != NULL) {\n if (zero && newsize > size) {\n // also set last word in the previous allocation to zero to ensure any padding is zero-initialized\n const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);\n _mi_memzero((uint8_t*)newp + start, newsize - start);\n }\n else if (newsize == 0) {\n ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725)\n }\n if mi_likely(p != NULL) {\n const size_t copysize = (newsize > size ? size : newsize);\n mi_track_mem_defined(p,copysize); // _mi_useable_size may be too large for byte precise memory tracking..\n _mi_memcpy(newp, p, copysize);\n mi_free(p); // only free the original pointer if successful // todo: optimize since page is known?\n }\n }\n return newp;\n}\n\nmi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {\n return _mi_theap_realloc_zero(theap, p, newsize, false, NULL, NULL);\n}\n\nstatic void* mi_theap_reallocn(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count, size, &total)) return NULL;\n return mi_theap_realloc(theap, p, total);\n}\n\n\n// Reallocate but free `p` on errors\nstatic void* mi_theap_reallocf(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {\n void* newp = mi_theap_realloc(theap, p, newsize);\n if (newp==NULL && p!=NULL) mi_free(p);\n return newp;\n}\n\nstatic void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {\n return _mi_theap_realloc_zero(theap, p, newsize, true, NULL, NULL);\n}\n\nstatic void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {\n size_t total;\n if (mi_count_size_overflow(count, size, &total)) return NULL;\n return mi_theap_rezalloc(theap, p, total);\n}\n\n\nmi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept {\n return mi_theap_realloc(_mi_theap_default(),p,newsize);\n}\n\nmi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept {\n return mi_theap_reallocn(_mi_theap_default(),p,count,size);\n}\n\nmi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {\n return _mi_theap_realloc_zero(_mi_theap_default(),p,newsize, false, usable_pre, usable_post);\n}\n\n// Reallocate but free `p` on errors\nmi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {\n return mi_theap_reallocf(_mi_theap_default(),p,newsize);\n}\n\nmi_decl_nodiscard void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept {\n return mi_theap_rezalloc(_mi_theap_default(), p, newsize);\n}\n\nmi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept {\n return mi_theap_recalloc(_mi_theap_default(), p, count, size);\n}\n\n\nmi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {\n return mi_theap_realloc(_mi_heap_theap(heap), p, newsize);\n}\n\nmi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {\n return mi_theap_reallocn(_mi_heap_theap(heap), p, count, size);\n}\n\n// Reallocate but free `p` on errors\nmi_decl_nodiscard void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {\n return mi_theap_reallocf(_mi_heap_theap(heap), p, newsize);\n}\n\nmi_decl_nodiscard void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {\n return mi_theap_rezalloc(_mi_heap_theap(heap), p, newsize);\n}\n\nmi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {\n return mi_theap_recalloc(_mi_heap_theap(heap), p, count, size);\n}\n\n\n\n// ------------------------------------------------------\n// strdup, strndup, and realpath\n// ------------------------------------------------------\n\n// `strdup` using mi_malloc\nmi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* theap, const char* s) mi_attr_noexcept {\n if (s == NULL) return NULL;\n size_t len = _mi_strlen(s);\n if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1\n char* t = (char*)mi_theap_malloc(theap,len+1);\n if (t == NULL) return NULL;\n _mi_memcpy(t, s, len);\n t[len] = 0;\n return t;\n}\n\nmi_decl_nodiscard mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept {\n return mi_theap_strdup(_mi_theap_default(), s);\n}\n\nmi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept {\n return mi_theap_strdup(_mi_heap_theap(heap), s);\n}\n\n// `strndup` using mi_malloc\nmi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* theap, const char* s, size_t n) mi_attr_noexcept {\n if (s == NULL) return NULL;\n const size_t len = _mi_strnlen(s,n); // len <= n\n if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1\n char* t = (char*)mi_theap_malloc(theap, len+1);\n if (t == NULL) return NULL;\n _mi_memcpy(t, s, len);\n t[len] = 0;\n return t;\n}\n\nmi_decl_nodiscard mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept {\n return mi_theap_strndup(_mi_theap_default(),s,n);\n}\n\nmi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept {\n return mi_theap_strndup(_mi_heap_theap(heap), s, n);\n}\n\n#ifndef __wasi__\n// `realpath` using mi_malloc\n#ifdef _WIN32\n#ifndef PATH_MAX\n#define PATH_MAX MAX_PATH\n#endif\n\nmi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {\n // todo: use GetFullPathNameW to allow longer file names\n char buf[PATH_MAX];\n DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL);\n if (res == 0) {\n errno = GetLastError(); return NULL;\n }\n else if (res > PATH_MAX) {\n errno = EINVAL; return NULL;\n }\n else if (resolved_name != NULL) {\n return resolved_name;\n }\n else {\n return mi_theap_strndup(theap, buf, PATH_MAX);\n }\n}\n\n#else\n\n#include // pathconf\n\nstatic size_t mi_path_max(void) {\n static _Atomic(size_t) path_max = 0;\n size_t pmax = mi_atomic_load_acquire(&path_max);\n if (pmax == 0) {\n long m = 0;\n #ifdef _PC_PATH_MAX\n m = pathconf(\"/\",_PC_PATH_MAX);\n #endif\n if (m <= 0) pmax = 4096; // guess\n else if (m < 256) pmax = 256; // at least 256\n else if (m > 64*1024) pmax = 64*1024; // at most 64 KiB\n else pmax = m;\n size_t expected = 0;\n mi_atomic_cas_strong_acq_rel(&path_max, &expected, pmax);\n }\n return pmax;\n}\n\nchar* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {\n if (resolved_name != NULL) {\n return realpath(fname,resolved_name);\n }\n else {\n /*\n char* rname = realpath(fname, NULL);\n if (rname == NULL) return NULL;\n char* result = mi_heap_strdup(heap, rname);\n mi_cfree(rname); // note: may leak the original pointer if allocated internally with the system allocator \n // note: with ASAN realpath is intercepted and mi_cfree may leak the returned pointer :-(\n return result; \n */\n const size_t n = mi_path_max();\n char* const buf = (char*)mi_zalloc(n+1);\n if (buf == NULL) {\n errno = ENOMEM;\n return NULL;\n }\n char* rname = realpath(fname,buf);\n char* result = mi_theap_strndup(theap,rname,n); // ok if `rname==NULL`\n mi_free(buf);\n return result;\n }\n}\n#endif\n\nmi_decl_nodiscard mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept {\n return mi_theap_realpath(_mi_theap_default(),fname,resolved_name);\n}\n\nmi_decl_nodiscard mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept {\n return mi_theap_realpath(_mi_heap_theap(heap), fname, resolved_name);\n}\n#endif\n\n/*-------------------------------------------------------\nC++ new and new_aligned\nThe standard requires calling into `get_new_handler` and\nthrowing the bad_alloc exception on failure. If we compile\nwith a C++ compiler we can implement this precisely. If we\nuse a C compiler we cannot throw a `bad_alloc` exception\nbut we call `exit` instead (i.e. not returning).\n-------------------------------------------------------*/\n\n#ifdef __cplusplus\n#include \nstatic bool mi_try_new_handler(bool nothrow) {\n #if defined(_MSC_VER) || (__cplusplus >= 201103L)\n std::new_handler h = std::get_new_handler();\n #else\n std::new_handler h = std::set_new_handler();\n std::set_new_handler(h);\n #endif\n if (h==NULL) {\n _mi_error_message(ENOMEM, \"out of memory in 'new'\");\n #if defined(_CPPUNWIND) || defined(__cpp_exceptions) // exceptions are not always enabled\n if (!nothrow) {\n throw std::bad_alloc();\n }\n #else\n MI_UNUSED(nothrow);\n #endif\n return false;\n }\n else {\n h();\n return true;\n }\n}\n#else\ntypedef void (*std_new_handler_t)(void);\n\n#if (defined(__GNUC__) || (defined(__clang__) && !defined(_MSC_VER))) // exclude clang-cl, see issue #631\nstd_new_handler_t __attribute__((weak)) _ZSt15get_new_handlerv(void) {\n return NULL;\n}\nstatic std_new_handler_t mi_get_new_handler(void) {\n return _ZSt15get_new_handlerv();\n}\n#else\n// note: on windows we could dynamically link to `?get_new_handler@std@@YAP6AXXZXZ`.\nstatic std_new_handler_t mi_get_new_handler(void) {\n return NULL;\n}\n#endif\n\nstatic bool mi_try_new_handler(bool nothrow) {\n std_new_handler_t h = mi_get_new_handler();\n if (h==NULL) {\n _mi_error_message(ENOMEM, \"out of memory in 'new'\");\n if (!nothrow) {\n abort(); // cannot throw in plain C, use abort\n }\n return false;\n }\n else {\n h();\n return true;\n }\n}\n#endif\n\nstatic mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) {\n void* p = NULL;\n while(p == NULL && mi_try_new_handler(nothrow)) {\n p = mi_theap_malloc(theap,size);\n }\n return p;\n}\n\nstatic mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) {\n return mi_theap_try_new(_mi_theap_default(), size, nothrow);\n}\n\nstatic mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) {\n return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow);\n}\n\n\nmi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) {\n void* p = mi_theap_malloc(theap,size);\n if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false);\n return p;\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) {\n return mi_theap_alloc_new(_mi_theap_default(), size);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) {\n void* p = mi_heap_malloc(heap, size);\n if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false);\n return p;\n}\n\n\nmi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) {\n size_t total;\n if mi_unlikely(mi_count_size_overflow(count, size, &total)) {\n mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc\n return NULL;\n }\n else {\n return mi_theap_alloc_new(theap,total);\n }\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) {\n return mi_theap_alloc_new_n(_mi_theap_default(), count, size);\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) {\n return mi_theap_alloc_new_n(_mi_heap_theap(heap), count, size);\n}\n\n\nmi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept {\n void* p = mi_malloc(size);\n if mi_unlikely(p == NULL) return mi_try_new(size, true);\n return p;\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) {\n void* p;\n do {\n p = mi_malloc_aligned(size, alignment);\n }\n while(p == NULL && mi_try_new_handler(false));\n return p;\n}\n\nmi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept {\n void* p;\n do {\n p = mi_malloc_aligned(size, alignment);\n }\n while(p == NULL && mi_try_new_handler(true));\n return p;\n}\n\nmi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) {\n void* q;\n do {\n q = mi_realloc(p, newsize);\n } while (q == NULL && mi_try_new_handler(false));\n return q;\n}\n\nmi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) {\n size_t total;\n if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) {\n mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc\n return NULL;\n }\n else {\n return mi_new_realloc(p, total);\n }\n}\n\n#if MI_GUARDED\n// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true).\n// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`)\n// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size`\n\nstatic void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) {\n // TODO: we can still make padding work by moving it out of the guard page area\n mi_page_t* const page = _mi_ptr_page(block);\n mi_page_set_has_interior_pointers(page, true);\n block->next = MI_BLOCK_TAG_GUARDED;\n\n // set guard page at the end of the block\n const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local`\n const size_t os_page_size = _mi_os_page_size();\n mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t));\n if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) {\n // should never happen\n mi_free(block);\n return NULL;\n }\n uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size;\n // note: the alignment of the guard page relies on blocks being os_page_size aligned which\n // is ensured in `mi_arena_page_alloc_fresh`. \n mi_assert_internal(_mi_is_aligned(block, os_page_size));\n mi_assert_internal(_mi_is_aligned(guard_page, os_page_size));\n if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) {\n const bool ok = _mi_os_protect(guard_page, os_page_size);\n if mi_unlikely(!ok) {\n _mi_warning_message(\"failed to set a guard page behind an object (object %p of size %zu)\\n\", block, block_size);\n }\n }\n else {\n _mi_warning_message(\"unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\\n\", block, block_size);\n }\n\n // align pointer just in front of the guard page\n size_t offset = block_size - os_page_size - obj_size;\n mi_assert_internal(offset > sizeof(mi_block_t));\n if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) {\n // give up to place it right in front of the guard page if the offset is too large for unalignment\n offset = MI_PAGE_MAX_OVERALLOC_ALIGN;\n }\n uint8_t* const p = (uint8_t*)block + offset;\n mi_assert_internal(p == guard_page - obj_size);\n mi_track_align(block, p, offset, obj_size);\n mi_track_mem_defined(block, sizeof(mi_block_t));\n return p;\n}\n\nmi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept\n{\n // allocate multiple of page size ending in a guard page\n // ensure minimal alignment requirement?\n if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow \n _mi_error_message(EOVERFLOW, \"(guarded) allocation request is too large (%zu bytes)\\n\", size);\n return NULL;\n }\n const size_t os_page_size = _mi_os_page_size();\n const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));\n const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);\n const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); \n mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, NULL);\n if (block==NULL) return NULL;\n void* const p = mi_block_ptr_set_guarded(block, obj_size);\n if (p == NULL) return p;\n if (zero) { \n _mi_memzero_aligned(p,obj_size); // we have to zero here as padding might have written here (if the blocksize > reqsize + os_page_size)\n }\n\n // stats\n mi_track_malloc(p, obj_size, zero); \n if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); }\n mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1);\n #if MI_STAT>1\n // adjust stats to only count the allocated size of the block (and not the guard page)\n mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size);\n mi_theap_stat_increase(theap, malloc_requested, size);\n #endif\n #if MI_DEBUG>3\n if (zero) {\n mi_assert_expensive(mi_mem_is_zero(p, size));\n }\n #endif\n return p;\n}\n#endif\n\n// ------------------------------------------------------\n// ensure explicit external inline definitions are emitted!\n// ------------------------------------------------------\n\n#ifdef __cplusplus\nvoid* _mi_externs[] = {\n (void*)&_mi_page_malloc_zero,\n (void*)&_mi_theap_malloc_zero,\n (void*)&_mi_theap_malloc_zero_ex,\n (void*)&mi_theap_malloc,\n (void*)&mi_theap_zalloc,\n (void*)&mi_theap_malloc_small,\n (void*)&mi_malloc,\n (void*)&mi_malloc_small,\n (void*)&mi_zalloc,\n (void*)&mi_zalloc_small,\n (void*)&mi_heap_malloc,\n (void*)&mi_heap_malloc_small,\n (void*)&mi_malloc_aligned\n // (void*)&mi_theap_alloc_new,\n // (void*)&mi_theap_alloc_new_n\n};\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/arena-meta.c", "sha256": "86fde644ede2e8d582e53499568a1713516a20326e0350eaa06a28029cece91b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* ----------------------------------------------------------------------------\n We have a special \"mini\" allocator just for allocation of meta-data like\n the theap (`mi_theap_t`) or thread-local data (`mi_tld_t`).\n\n We reuse the bitmap of the arena's for allocation of 64b blocks inside\n an arena slice (64KiB).\n We always ensure that meta data is zero'd (we zero on `free`)\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"bitmap.h\"\n\n/* -----------------------------------------------------------\n Meta data allocation\n----------------------------------------------------------- */\n\n#define MI_META_PAGE_SIZE MI_ARENA_SLICE_SIZE\n#define MI_META_PAGE_ALIGN MI_ARENA_SLICE_ALIGN\n\n// large enough such that META_MAX_SIZE > 4k (even on 32-bit)\n#define MI_META_BLOCK_SIZE (1 << (16 - MI_BCHUNK_BITS_SHIFT)) // 128 on 64-bit\n#define MI_META_BLOCK_ALIGN MI_META_BLOCK_SIZE\n#define MI_META_BLOCKS_PER_PAGE (MI_META_PAGE_SIZE / MI_META_BLOCK_SIZE) // 512\n#define MI_META_MAX_SIZE (MI_BCHUNK_SIZE * MI_META_BLOCK_SIZE)\n\n#if MI_META_MAX_SIZE <= 4096\n#error \"max meta object size should be at least 4KiB\"\n#endif\n\ntypedef struct mi_meta_page_s {\n _Atomic(struct mi_meta_page_s*) next; // a linked list of meta-data pages (never released)\n mi_memid_t memid; // provenance of the meta-page memory itself\n mi_bbitmap_t blocks_free; // a small bitmap with 1 bit per block.\n} mi_meta_page_t;\n\nstatic mi_decl_cache_align _Atomic(mi_meta_page_t*) mi_meta_pages = MI_ATOMIC_VAR_INIT(NULL);\n\n\n#if MI_DEBUG > 1\nstatic mi_meta_page_t* mi_meta_page_of_ptr(void* p, size_t* block_idx) {\n mi_meta_page_t* mpage = (mi_meta_page_t*)((uint8_t*)_mi_align_down_ptr(p,MI_META_PAGE_ALIGN) + _mi_os_secure_guard_page_size());\n if (block_idx != NULL) {\n *block_idx = ((uint8_t*)p - (uint8_t*)mpage) / MI_META_BLOCK_SIZE;\n }\n return mpage;\n}\n#endif\n\nstatic mi_meta_page_t* mi_meta_page_next( mi_meta_page_t* mpage ) {\n return mi_atomic_load_ptr_acquire(mi_meta_page_t, &mpage->next);\n}\n\nstatic void* mi_meta_block_start( mi_meta_page_t* mpage, size_t block_idx ) {\n mi_assert_internal(_mi_is_aligned((uint8_t*)mpage - _mi_os_secure_guard_page_size(), MI_META_PAGE_ALIGN));\n mi_assert_internal(block_idx < MI_META_BLOCKS_PER_PAGE);\n void* p = ((uint8_t*)mpage - _mi_os_secure_guard_page_size() + (block_idx * MI_META_BLOCK_SIZE));\n mi_assert_internal(mpage == mi_meta_page_of_ptr(p,NULL));\n return p;\n}\n\n// allocate a fresh meta page and add it to the global list.\nstatic mi_meta_page_t* mi_meta_page_zalloc(void) {\n // allocate a fresh arena slice\n // note: careful with _mi_subproc as it may recurse into mi_tld and meta_page_zalloc again.. (same with _mi_os_numa_node()...)\n mi_memid_t memid;\n uint8_t* base = (uint8_t*)_mi_arenas_alloc_aligned(mi_heap_main(), MI_META_PAGE_SIZE, MI_META_PAGE_ALIGN, 0,\n true /* commit*/, (MI_SECURE==0) /* allow large? */,\n NULL /* req arena */, 0 /* thread_seq */, -1 /* numa node */, &memid);\n if (base == NULL) return NULL;\n mi_assert_internal(_mi_is_aligned(base,MI_META_PAGE_ALIGN));\n if (!memid.initially_zero) {\n _mi_memzero_aligned(base, MI_ARENA_SLICE_SIZE);\n }\n\n // guard pages\n #if MI_SECURE >= 1\n _mi_os_secure_guard_page_set_at(base, memid);\n _mi_os_secure_guard_page_set_before(base + MI_META_PAGE_SIZE, memid);\n #endif\n\n // initialize the page and free block bitmap\n mi_meta_page_t* mpage = (mi_meta_page_t*)(base + _mi_os_secure_guard_page_size());\n mpage->memid = memid;\n mi_bbitmap_init(&mpage->blocks_free, MI_META_BLOCKS_PER_PAGE, true /* already_zero */);\n const size_t mpage_size = offsetof(mi_meta_page_t,blocks_free) + mi_bbitmap_size(MI_META_BLOCKS_PER_PAGE, NULL);\n const size_t info_blocks = _mi_divide_up(mpage_size,MI_META_BLOCK_SIZE);\n const size_t guard_blocks = _mi_divide_up(_mi_os_secure_guard_page_size(), MI_META_BLOCK_SIZE);\n mi_assert_internal(info_blocks + 2*guard_blocks < MI_META_BLOCKS_PER_PAGE);\n mi_bbitmap_unsafe_setN(&mpage->blocks_free, info_blocks + guard_blocks, MI_META_BLOCKS_PER_PAGE - info_blocks - 2*guard_blocks);\n\n // push atomically in front of the meta page list\n // (note: there is no ABA issue since we never free meta-pages)\n mi_meta_page_t* old = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);\n do {\n mi_atomic_store_ptr_release(mi_meta_page_t, &mpage->next, old);\n } while(!mi_atomic_cas_ptr_weak_acq_rel(mi_meta_page_t,&mi_meta_pages,&old,mpage));\n return mpage;\n}\n\n\n// allocate meta-data\nmi_decl_noinline void* _mi_meta_zalloc( size_t size, mi_memid_t* pmemid )\n{\n mi_assert_internal(pmemid != NULL);\n size = _mi_align_up(size,MI_META_BLOCK_SIZE);\n if (size == 0 || size > MI_META_MAX_SIZE) return NULL;\n const size_t block_count = _mi_divide_up(size,MI_META_BLOCK_SIZE);\n mi_assert_internal(block_count > 0 && block_count < MI_BCHUNK_BITS);\n mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);\n mi_meta_page_t* mpage = mpage0;\n while (mpage != NULL) {\n size_t block_idx;\n if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {\n // found and claimed `block_count` blocks\n *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);\n return mi_meta_block_start(mpage,block_idx);\n }\n else {\n mpage = mi_meta_page_next(mpage);\n }\n }\n // failed to find space in existing pages\n if (mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages) != mpage0) {\n // the page list was updated by another thread in the meantime, retry\n return _mi_meta_zalloc(size,pmemid);\n }\n // otherwise, allocate a fresh metapage and try once more\n mpage = mi_meta_page_zalloc();\n if (mpage != NULL) {\n size_t block_idx;\n if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {\n // found and claimed `block_count` blocks\n *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);\n return mi_meta_block_start(mpage,block_idx);\n }\n }\n // if all this failed, allocate from the OS\n return _mi_os_alloc(size, pmemid);\n}\n\n// free meta-data\nmi_decl_noinline void _mi_meta_free(void* p, size_t size, mi_memid_t memid) {\n if (p==NULL) return;\n if (memid.memkind == MI_MEM_META) {\n mi_assert_internal(_mi_divide_up(size, MI_META_BLOCK_SIZE) == memid.mem.meta.block_count);\n const size_t block_count = memid.mem.meta.block_count;\n const size_t block_idx = memid.mem.meta.block_index;\n mi_meta_page_t* mpage = (mi_meta_page_t*)memid.mem.meta.meta_page;\n mi_assert_internal(mi_meta_page_of_ptr(p,NULL) == mpage);\n mi_assert_internal(block_idx + block_count <= MI_META_BLOCKS_PER_PAGE);\n mi_assert_internal(mi_bbitmap_is_clearN(&mpage->blocks_free, block_idx, block_count));\n // we zero on free (and on the initial page allocation) so we don't need a \"dirty\" map\n _mi_memzero_aligned(mi_meta_block_start(mpage, block_idx), block_count*MI_META_BLOCK_SIZE);\n mi_bbitmap_setN(&mpage->blocks_free, block_idx, block_count);\n }\n else {\n _mi_arenas_free(p,size,memid);\n }\n}\n\n// used for debug output\nbool _mi_meta_is_meta_page(void* p)\n{\n mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t, &mi_meta_pages);\n mi_meta_page_t* mpage = mpage0;\n while (mpage != NULL) {\n if ((void*)mpage == p) return true;\n mpage = mi_meta_page_next(mpage);\n }\n return false;\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/arena.c", "sha256": "a1d6900d2b9a17461aa2a72a7791dfb22f6167b86f7ac6f15f981d9735aa2adb", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* ----------------------------------------------------------------------------\n\"Arenas\" are fixed area's of OS memory from which we can allocate\nlarge blocks (>= MI_ARENA_MIN_BLOCK_SIZE, 64KiB).\nIn contrast to the rest of mimalloc, the arenas are shared between\nthreads and need to be accessed using atomic operations.\n\nArenas are also used to for huge OS page (1GiB) reservations or for reserving\nOS memory upfront which can be improve performance or is sometimes needed\non embedded devices. We can also employ this with WASI or `sbrk` systems\nto reserve large arenas upfront and be able to reuse the memory more effectively.\n\nThe arena allocation needs to be thread safe and we use an atomic bitmap to allocate.\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\"\n#include \"bitmap.h\"\n\n/* -----------------------------------------------------------\n Arena id's\n----------------------------------------------------------- */\n\nmi_arena_id_t _mi_arena_id_none(void) {\n return NULL;\n}\n\nmi_arena_t* _mi_arena_from_id(mi_arena_id_t id) {\n mi_arena_t* const arena = (mi_arena_t*)id;\n mi_assert_internal(arena==NULL || arena->parent==NULL); // id's should never point to sub-arena's\n return arena;\n}\n\nmi_arena_id_t mi_arena_id_from_arena(mi_arena_t* arena) {\n mi_assert_internal(arena==NULL || arena->parent==NULL);\n return (arena==NULL ? _mi_arena_id_none() : (mi_arena_id_t)arena);\n}\n\n\nstatic bool mi_arena_is_suitable(mi_arena_t* arena, mi_arena_t* req_arena) {\n if (arena == req_arena) return true; // they match\n if (arena == NULL) return false;\n if (req_arena == NULL && !arena->is_exclusive) return true; // or the arena is not exclusive, and we didn't request a specific one\n if (arena->parent != NULL && arena->parent == req_arena) return true; // sub-arena? (note that req_arena is never a sub arena)\n return false;\n}\n\nbool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena) {\n if (memid.memkind == MI_MEM_ARENA) {\n return mi_arena_is_suitable(memid.mem.arena.arena, request_arena);\n }\n else {\n return mi_arena_is_suitable(NULL, request_arena);\n }\n}\n\nsize_t mi_arenas_get_count(mi_subproc_t* subproc) {\n return mi_atomic_load_relaxed(&subproc->arena_count);\n}\n\nmi_arena_t* mi_arena_from_index(mi_subproc_t* subproc, size_t idx) {\n mi_assert_internal(idx < mi_arenas_get_count(subproc));\n return mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[idx]);\n}\n\nstatic size_t mi_arena_info_slices(mi_arena_t* arena) {\n return arena->info_slices;\n}\n\n#if MI_DEBUG > 1\nstatic bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page) {\n mi_assert(arena->arena_idx < MI_MAX_ARENAS);\n mi_arena_pages_t* arena_pages = heap->arena_pages[arena->arena_idx];\n return (page->memid.memkind == MI_MEM_ARENA &&\n page->memid.mem.arena.arena == arena &&\n arena_pages != NULL &&\n mi_bitmap_is_setN(arena_pages->pages, page->memid.mem.arena.slice_index, 1));\n}\n#endif\n\nsize_t mi_arena_min_alignment(void) {\n return MI_ARENA_SLICE_ALIGN;\n}\n\nsize_t mi_arena_min_size(void) {\n return MI_ARENA_MIN_SIZE;\n}\n\nstatic size_t mi_arena_max_object_size(void) {\n size_t max_size = mi_option_get_size(mi_option_arena_max_object_size);\n max_size = _mi_align_up(max_size, MI_ARENA_SLICE_SIZE);\n if (max_size <= MI_ARENA_MIN_OBJ_SIZE) {\n return MI_ARENA_MIN_OBJ_SIZE;\n }\n else if (max_size >= MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE) { // minus a bchunk to accommodate meta info\n return (MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE);\n }\n else {\n return max_size;\n }\n}\n\nmi_decl_nodiscard static bool mi_arena_commit(mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t already_committed) {\n if (arena != NULL && arena->commit_fun != NULL) {\n return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg);\n }\n else if (already_committed > 0) {\n return _mi_os_commit_ex(start, size, is_zero, already_committed);\n }\n else {\n return _mi_os_commit(start, size, is_zero);\n }\n}\n\n\n\n/* -----------------------------------------------------------\n Util\n----------------------------------------------------------- */\n\n\n// Size of an arena\nstatic size_t mi_arena_size(mi_arena_t* arena) {\n return mi_size_of_slices(arena->slice_count);\n}\n\n// Start of the arena memory area\nstatic uint8_t* mi_arena_start(mi_arena_t* arena) {\n return ((uint8_t*)arena);\n}\n\n// Start of a slice\nuint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) {\n mi_assert_internal(slice_index < arena->slice_count);\n return (mi_arena_start(arena) + mi_size_of_slices(slice_index));\n}\n\nmi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) {\n mi_assert_internal(slice_index < arena->slice_count);\n if (arena->pages_meta != NULL) {\n mi_page_t* const page = &arena->pages_meta[slice_index];\n #if MI_PAGE_META_ALIGNED_FREE_SMALL\n // pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0)\n if (page->block_size>0) return page;\n #else\n return page;\n #endif\n }\n // fall through (for MI_PAGE_META_ALIGNED_FREE_SMALL)\n return (mi_page_t*)mi_arena_slice_start(arena,slice_index);\n}\n\n// Arena area\nvoid* mi_arena_area(mi_arena_id_t arena_id, size_t* size) {\n if (size != NULL) *size = 0;\n mi_arena_t* arena = _mi_arena_from_id(arena_id);\n if (arena == NULL) return NULL;\n if (size != NULL) {\n mi_assert_internal(mi_size_of_slices(arena->slice_count) <= arena->total_size);\n *size = arena->total_size;\n }\n return mi_arena_start(arena);\n}\n\n\n// Create an arena memid\nstatic mi_memid_t mi_memid_create_arena(mi_arena_t* arena, size_t slice_index, size_t slice_count) {\n mi_assert_internal(slice_index < UINT32_MAX);\n mi_assert_internal(slice_count < UINT32_MAX);\n mi_assert_internal(slice_count > 0);\n mi_assert_internal(slice_index < arena->slice_count);\n mi_memid_t memid = _mi_memid_create(MI_MEM_ARENA);\n memid.mem.arena.arena = arena;\n memid.mem.arena.slice_index = (uint32_t)slice_index;\n memid.mem.arena.slice_count = (uint32_t)slice_count;\n return memid;\n}\n\n// get the arena and slice span\nstatic mi_arena_t* mi_arena_from_memid(mi_memid_t memid, size_t* slice_index, size_t* slice_count) {\n mi_assert_internal(memid.memkind == MI_MEM_ARENA);\n mi_arena_t* arena = memid.mem.arena.arena;\n if (slice_index!=NULL) { *slice_index = memid.mem.arena.slice_index; }\n if (slice_count!=NULL) { *slice_count = memid.mem.arena.slice_count; }\n return arena;\n}\n\nstatic size_t mi_page_full_size(mi_page_t* page) {\n if (page->memid.memkind == MI_MEM_ARENA) {\n return page->memid.mem.arena.slice_count * MI_ARENA_SLICE_SIZE;\n }\n else if (mi_memid_is_os(page->memid) || page->memid.memkind == MI_MEM_EXTERNAL) {\n mi_assert_internal((uint8_t*)page->memid.mem.os.base <= (uint8_t*)page);\n const ptrdiff_t presize = (uint8_t*)page - (uint8_t*)page->memid.mem.os.base;\n mi_assert_internal((ptrdiff_t)page->memid.mem.os.size >= presize);\n return (presize > (ptrdiff_t)page->memid.mem.os.size ? 0 : page->memid.mem.os.size - presize);\n }\n else {\n return 0;\n }\n}\n\n\n/* -----------------------------------------------------------\n Arena Allocation\n----------------------------------------------------------- */\n\nstatic mi_decl_noinline void* mi_arena_try_alloc_at(\n mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid)\n{\n size_t slice_index;\n if (!mi_bbitmap_try_find_and_clearN(arena->slices_free, tseq, slice_count, &slice_index)) return NULL;\n\n // claimed it!\n void* p = mi_arena_slice_start(arena, slice_index);\n *memid = mi_memid_create_arena(arena, slice_index, slice_count);\n memid->is_pinned = arena->memid.is_pinned;\n\n // set the dirty bits and track which slices become accessible\n size_t touched_slices = slice_count;\n if (arena->memid.initially_zero) {\n size_t already_dirty = 0;\n memid->initially_zero = mi_bitmap_setN(arena->slices_dirty, slice_index, slice_count, &already_dirty);\n mi_assert_internal(already_dirty <= touched_slices);\n touched_slices -= already_dirty;\n }\n\n // set commit state\n if (commit) {\n // commit requested, but the range may not be committed as a whole: ensure it is committed now\n const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count);\n if (already_committed < slice_count) {\n // not all committed, try to commit now\n bool commit_zero = false;\n if (!mi_arena_commit(arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) {\n // if the commit fails, release ownership, and return NULL;\n // note: this does not roll back dirty bits but that is ok.\n mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);\n return NULL;\n }\n if (commit_zero) {\n memid->initially_zero = true;\n }\n\n // set the commit bits\n mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, NULL);\n\n // committed\n #if MI_DEBUG > 1\n if (memid->initially_zero) {\n if (!mi_mem_is_zero(p, mi_size_of_slices(slice_count))) {\n _mi_error_message(EFAULT, \"internal error: arena allocation was not zero-initialized!\\n\");\n memid->initially_zero = false;\n }\n }\n #endif\n }\n else {\n // already fully committed.\n _mi_os_reuse(p, mi_size_of_slices(slice_count));\n // if the OS has overcommit, and this is the first time we access these pages, then\n // count the commit now (as at arena reserve we didn't count those commits as these are on-demand)\n if (_mi_os_has_overcommit() && touched_slices > 0 && !arena->memid.is_pinned /* huge pages, issue #1236 */) {\n mi_subproc_stat_increase( arena->subproc, committed, mi_size_of_slices(touched_slices));\n }\n }\n\n mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));\n memid->initially_committed = true;\n\n // tool support\n if (memid->initially_zero) {\n mi_track_mem_defined(p, slice_count * MI_ARENA_SLICE_SIZE);\n }\n else {\n mi_track_mem_undefined(p, slice_count * MI_ARENA_SLICE_SIZE);\n }\n }\n else {\n // no need to commit, but check if it is already fully committed\n memid->initially_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count);\n if (!memid->initially_committed) {\n // partly committed.. adjust stats\n size_t already_committed_count = 0;\n mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count);\n mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);\n mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed_count));\n }\n }\n\n mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));\n if (commit) { mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); }\n if (commit) { mi_assert_internal(memid->initially_committed); }\n mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));\n\n return p;\n}\n\n\nstatic int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id);\n\n// try to reserve a fresh arena space\nstatic bool mi_arena_reserve(mi_subproc_t* subproc, size_t req_size, bool allow_large, mi_arena_id_t* arena_id)\n{\n const size_t arena_count = mi_arenas_get_count(subproc);\n if (arena_count > (MI_MAX_ARENAS - 4)) return false;\n\n // calc reserve\n size_t arena_reserve = mi_option_get_size(mi_option_arena_reserve);\n if (arena_reserve == 0) return false;\n\n if (!_mi_os_has_virtual_reserve()) {\n arena_reserve = arena_reserve/4; // be conservative if virtual reserve is not supported (for WASM for example)\n }\n arena_reserve = _mi_align_up(arena_reserve, MI_ARENA_SLICE_SIZE);\n\n if (arena_count >= 1 && arena_count <= 128) {\n // scale up the arena sizes exponentially every 8 entries\n const size_t multiplier = (size_t)1 << _mi_clamp(arena_count/8, 0, 16);\n size_t reserve = 0;\n if (!mi_mul_overflow(multiplier, arena_reserve, &reserve)) {\n arena_reserve = reserve;\n }\n }\n\n // try to accommodate the requested size for huge allocations\n req_size = _mi_align_up(req_size + MI_ARENA_MAX_CHUNK_OBJ_SIZE, MI_ARENA_MAX_CHUNK_OBJ_SIZE); // over-reserve for meta-info\n if (arena_reserve < req_size) {\n arena_reserve = req_size;\n }\n\n // check arena bounds\n const size_t min_reserve = MI_ARENA_MIN_SIZE;\n const size_t max_reserve = MI_ARENA_MAX_SIZE; // 16 GiB\n if (arena_reserve < min_reserve) {\n arena_reserve = min_reserve;\n }\n else if (arena_reserve > max_reserve) {\n arena_reserve = max_reserve;\n }\n\n // should be able to at least handle the current allocation size\n if (arena_reserve < req_size) return false;\n\n // commit eagerly?\n bool arena_commit = false;\n const bool overcommit = _mi_os_has_overcommit();\n if (mi_option_get(mi_option_arena_eager_commit) == 2) { arena_commit = overcommit || mi_option_is_enabled(mi_option_allow_large_os_pages); }\n else if (mi_option_get(mi_option_arena_eager_commit) == 1) { arena_commit = true; }\n\n // on an OS with overcommit (Linux) we don't count the commit yet as it is on-demand. Once a slice\n // is actually allocated for the first time it will be counted.\n const bool adjust = (overcommit && arena_commit);\n if (adjust) { mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve); }\n // and try to reserve the arena\n int err = mi_reserve_os_memory_ex2(subproc, arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id);\n if (err != 0) {\n if (adjust) { mi_subproc_stat_adjust_increase( subproc, committed, arena_reserve); } // roll back\n // failed to allocate: try a smaller size arena as fallback?\n const size_t small_arena_reserve = 4 * MI_ARENA_MIN_SIZE; // 128 MiB (or 32 MiB on 32-bit)\n if (arena_reserve > small_arena_reserve && small_arena_reserve > req_size) {\n // try again\n if (adjust) { mi_subproc_stat_adjust_decrease(subproc, committed, small_arena_reserve); }\n err = mi_reserve_os_memory_ex2(subproc, small_arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id);\n if (err != 0 && adjust) { mi_subproc_stat_adjust_increase( subproc, committed, small_arena_reserve); } // roll back\n }\n }\n return (err==0);\n}\n\n\n\n\n/* -----------------------------------------------------------\n Arena iteration\n----------------------------------------------------------- */\n\nstatic inline bool mi_arena_is_suitable_ex(mi_arena_t* arena, mi_arena_t* req_arena, bool match_numa, int numa_node, bool allow_pinned) {\n if (!allow_pinned && arena->memid.is_pinned) return false;\n if (!mi_arena_is_suitable(arena, req_arena)) return false;\n if (req_arena == NULL) { // if not specific, check numa affinity\n const bool numa_suitable = (numa_node < 0 || arena->numa_node < 0 || arena->numa_node == numa_node);\n if (match_numa) { if (!numa_suitable) return false; }\n else { if (numa_suitable) return false; }\n }\n return true;\n}\n\n// determine the start of search; important to keep heaps and threads\n// into their own memory regions to reduce contention.\nstatic size_t mi_arena_start_idx(mi_heap_t* heap, size_t tseq, size_t arena_cycle) {\n const size_t hseq = heap->heap_seq;\n const size_t hcount = mi_atomic_load_relaxed(&heap->subproc->heap_count);\n if (arena_cycle <= 1) return 0;\n if (hseq==0 || hcount<=1 || arena_cycle > 0x8FF) return (tseq % arena_cycle); // common for single heap programs\n\n // spread heaps evenly among arena's, and then evenly for threads in their fraction\n size_t start;\n mi_assert_internal(arena_cycle <= 0x8FF); // prevent overflow on 32-bit\n const size_t frac = (arena_cycle * 256) / hcount; // fraction in the arena_cycle; at most: arena_cycle * 0x100\n if (frac==0) {\n // many heaps (> 256 per arena)\n start = (hseq % arena_cycle);\n }\n else {\n const size_t hspot = (hseq % hcount); \n start = (frac * hspot) / 256; // (arena_cycle * (hseq % hcount)) / hcount\n if (frac >= 512) { // at least 2 arena's per heap?\n start = start + (tseq % (frac/256));\n }\n }\n mi_assert_internal(start < arena_cycle);\n return start;\n}\n\n#define mi_forall_arenas(heap, req_arena, tseq, name_arena) { \\\n const size_t _arena_count = mi_arenas_get_count(heap->subproc); \\\n const size_t _arena_cycle = (_arena_count == 0 ? 0 : _arena_count - 1); /* first search the arenas below the last one */ \\\n /* always start searching in the arena's below the max */ \\\n const size_t _start = mi_arena_start_idx(heap,tseq,_arena_cycle); \\\n for (size_t _i = 0; _i < _arena_count; _i++) { \\\n mi_arena_t* name_arena; \\\n if (req_arena != NULL) { \\\n name_arena = req_arena; /* if there is a specific req_arena, only search that one */\\\n if (_i > 0) break; /* only once */ \\\n } \\\n else { \\\n size_t _idx; \\\n if (_i < _arena_cycle) { \\\n _idx = _i + _start; \\\n if (_idx >= _arena_cycle) { _idx -= _arena_cycle; } /* adjust so we rotate through the cycle */ \\\n } \\\n else { \\\n _idx = _i; /* remaining arena's after the cycle */ \\\n } \\\n name_arena = mi_arena_from_index(heap->subproc,_idx); \\\n } \\\n if (name_arena != NULL) \\\n {\n\n#define mi_forall_arenas_end() \\\n } \\\n } \\\n }\n\n#define mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, numa_node, allow_large, name_arena) \\\n mi_forall_arenas(heap, req_arena,tseq,name_arena) { \\\n if (mi_arena_is_suitable_ex(name_arena, req_arena, match_numa, numa_node, allow_large)) { \\\n\n#define mi_forall_suitable_arenas_end() \\\n }} \\\n mi_forall_arenas_end()\n\n/* -----------------------------------------------------------\n Arena allocation\n----------------------------------------------------------- */\n\n// allocate slices from the arenas\nstatic mi_decl_noinline void* mi_arenas_try_find_free(\n mi_heap_t* heap, size_t slice_count, size_t alignment,\n bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)\n{\n // mi_assert_internal(slice_count <= mi_slice_count_of_size(MI_ARENA_MAX_CHUNK_OBJ_SIZE));\n mi_assert(alignment <= MI_ARENA_SLICE_ALIGN);\n if (alignment > MI_ARENA_SLICE_ALIGN) return NULL;\n\n // search arena's\n mi_forall_suitable_arenas(heap, req_arena, tseq, true /* only numa matching */, numa_node, allow_large, arena)\n {\n void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid);\n if (p != NULL) return p;\n }\n mi_forall_suitable_arenas_end();\n if (numa_node < 0) return NULL;\n\n // search again but now regardless of preferred numa affinity\n mi_forall_suitable_arenas(heap, req_arena, tseq, false /* numa non-matching now */, numa_node, allow_large, arena)\n {\n void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid);\n if (p != NULL) return p;\n }\n mi_forall_suitable_arenas_end();\n return NULL;\n}\n\n// Allocate slices from the arena's -- potentially allocating a fresh arena\nstatic mi_decl_noinline void* mi_arenas_try_alloc(\n mi_heap_t* heap,\n size_t slice_count, size_t alignment,\n bool commit, bool allow_large,\n mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)\n{\n // mi_assert(slice_count <= MI_ARENA_MAX_CHUNK_OBJ_SLICES);\n mi_assert(alignment <= MI_ARENA_SLICE_ALIGN);\n void* p;\n\n // not too large?\n if (slice_count * MI_ARENA_SLICE_SIZE > MI_ARENA_MAX_SIZE) return NULL;\n\n // try to find free slices in the arena's\n p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid);\n if (p != NULL) return p;\n\n // did we need a specific arena?\n if (req_arena != NULL) return NULL;\n\n // don't create arena's while preloading (todo: or should we?)\n if (_mi_preloading()) return NULL;\n\n // don't create arena's if OS allocation is disallowed\n if (mi_option_is_enabled(mi_option_disallow_os_alloc)) return NULL;\n\n // otherwise, try to reserve a new arena -- but one thread at a time.. (todo: allow 2 or 4 to reduce contention?)\n mi_subproc_t* const subproc = heap->subproc;\n const size_t arena_count = mi_arenas_get_count(subproc);\n mi_lock(&subproc->arena_reserve_lock) {\n if (arena_count == mi_arenas_get_count(subproc)) {\n // we are the first to enter the lock, reserve a fresh arena\n mi_arena_id_t arena_id = _mi_arena_id_none();\n mi_arena_reserve(subproc, mi_size_of_slices(slice_count), allow_large, &arena_id);\n }\n else {\n // another thread already reserved a new arena\n }\n }\n // try once more to allocate in the new arena\n mi_assert_internal(req_arena == NULL);\n p = mi_arenas_try_find_free(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid);\n if (p != NULL) return p;\n\n return NULL;\n}\n\n// Allocate from the OS (if allowed)\nstatic void* mi_arena_os_alloc_aligned(\n size_t size, size_t alignment, size_t align_offset,\n bool commit, bool allow_large,\n mi_arena_id_t req_arena_id, mi_memid_t* memid)\n{\n // if we cannot use OS allocation, return NULL\n if (mi_option_is_enabled(mi_option_disallow_os_alloc) || req_arena_id != _mi_arena_id_none()) {\n errno = ENOMEM;\n return NULL;\n }\n\n if (align_offset > 0) {\n return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid);\n }\n else {\n return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);\n }\n}\n\n\n// Allocate large sized memory\nvoid* _mi_arenas_alloc_aligned( mi_heap_t* heap,\n size_t size, size_t alignment, size_t align_offset,\n bool commit, bool allow_large,\n mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)\n{\n mi_assert_internal(memid != NULL);\n mi_assert_internal(size > 0);\n\n // try to allocate in an arena if the alignment is small enough and the object is not too small (as for theap meta data)\n if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && // is arena allocation allowed?\n size >= MI_ARENA_MIN_OBJ_SIZE && size <= mi_arena_max_object_size() && // and not too small or too large\n alignment <= MI_ARENA_SLICE_ALIGN && align_offset == 0) // and good alignment\n {\n const size_t slice_count = mi_slice_count_of_size(size);\n void* p = mi_arenas_try_alloc(heap, slice_count, alignment, commit, allow_large, req_arena, tseq, numa_node, memid);\n if (p != NULL) return p;\n }\n\n // fall back to the OS\n void* p = mi_arena_os_alloc_aligned(size, alignment, align_offset, commit, allow_large, req_arena, memid);\n return p;\n}\n\nvoid* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid)\n{\n return _mi_arenas_alloc_aligned(heap, size, MI_ARENA_SLICE_SIZE, 0, commit, allow_large, req_arena, tseq, numa_node, memid);\n}\n\n\n\n/* -----------------------------------------------------------\n Arena page allocation\n----------------------------------------------------------- */\n\n// release ownership of a page. This may free the page if all blocks were concurrently\n// freed in the meantime. Returns true if the page was freed.\nstatic bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theap) {\n mi_assert_internal(mi_page_is_owned(page));\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id);\n mi_thread_free_t tf_new;\n mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);\n do {\n mi_assert_internal(mi_tf_is_owned(tf_old));\n while mi_unlikely(mi_tf_block(tf_old) != NULL) {\n _mi_page_free_collect(page, false); // update used\n if (mi_page_all_free(page)) { // it may become free just before unowning it\n _mi_arenas_page_unabandon(page, current_theap);\n _mi_arenas_page_free(page, current_theap);\n return true;\n }\n tf_old = mi_atomic_load_relaxed(&page->xthread_free);\n }\n mi_assert_internal(mi_tf_block(tf_old)==NULL);\n tf_new = mi_tf_create(NULL, false);\n } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new));\n return false;\n}\n\n\nstatic bool mi_arena_try_claim_abandoned(size_t slice_index, mi_arena_t* arena, bool* keep_abandoned) {\n // found an abandoned page of the right size\n mi_page_t* const page = mi_arena_page_at_slice(arena, slice_index);\n // can we claim ownership?\n if (!mi_page_claim_ownership(page)) {\n // there was a concurrent free that reclaims this page ..\n // we need to keep it in the abandoned map as the free will call `mi_arena_page_unabandon`,\n // and wait for readers (us!) to finish. This is why it is very important to set the abandoned\n // bit again (or otherwise the unabandon will never stop waiting).\n *keep_abandoned = true;\n return false;\n }\n else {\n // yes, we can reclaim it, keep the abandoned map entry clear\n *keep_abandoned = false;\n return true;\n }\n}\n\n// allocate initial arena_pages from the main heap\nstatic mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena);\n\nstatic mi_arena_pages_t* mi_heap_arena_pages(mi_heap_t* heap, mi_arena_t* arena) {\n mi_assert_internal(arena!=NULL);\n mi_assert_internal(heap!=NULL);\n mi_assert(arena->arena_idx < MI_MAX_ARENAS);\n return mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]);\n}\n\nstatic mi_arena_t* mi_page_arena_pages(mi_page_t* page, size_t* slice_index, size_t* slice_count, mi_arena_pages_t** parena_pages) {\n // todo: maybe store the arena* directly in the page?\n mi_assert_internal(mi_page_is_owned(page));\n mi_arena_t* const arena = mi_arena_from_memid(page->memid, slice_index, slice_count);\n mi_assert_internal(arena != NULL);\n if (parena_pages != NULL) {\n mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(mi_page_heap(page), arena);\n mi_assert_internal(arena_pages != NULL);\n mi_assert_internal(slice_index==NULL || mi_bitmap_is_set(arena_pages->pages, *slice_index));\n *parena_pages = arena_pages;\n }\n return arena;\n}\n\nstatic mi_arena_pages_t* mi_heap_ensure_arena_pages(mi_heap_t* heap, mi_arena_t* arena) {\n mi_assert_internal(arena!=NULL);\n mi_assert_internal(heap!=NULL);\n mi_assert(arena->arena_idx < MI_MAX_ARENAS);\n mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);\n if (arena_pages==NULL) {\n mi_lock(&heap->arena_pages_lock) {\n arena_pages = mi_atomic_load_ptr_acquire(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx]);\n if (arena_pages == NULL) { // still NULL?\n if (_mi_is_heap_main(heap)) {\n // the page info for the main heap is always allocated as part of an arena\n arena_pages = &arena->pages_main;\n }\n else {\n // always allocate the arena pages info from the main heap\n // todo: allocate into the current arena?\n arena_pages = mi_arena_pages_alloc(arena);\n }\n mi_atomic_store_ptr_release(mi_arena_pages_t, &heap->arena_pages[arena->arena_idx], arena_pages);\n }\n }\n }\n if (_mi_is_heap_main(heap)) { mi_assert(arena_pages != NULL); } // can never fail\n return arena_pages;\n}\n\nstatic mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t slice_count, size_t block_size)\n{\n mi_heap_t* const heap = _mi_theap_heap(theap);\n const size_t tseq = theap->tld->thread_seq;\n mi_arena_t* const req_arena = heap->exclusive_arena;\n\n MI_UNUSED(slice_count);\n const size_t bin = _mi_bin(block_size);\n mi_assert_internal(bin < MI_BIN_COUNT);\n\n // any abandoned in our size class?\n mi_assert_internal(heap != NULL);\n if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) {\n return NULL;\n }\n\n // search arena's\n const bool allow_large = true;\n const int any_numa = -1;\n const bool match_numa = true;\n mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, any_numa, allow_large, arena)\n {\n mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena);\n if (arena_pages != NULL) {\n size_t slice_index;\n mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin];\n\n if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena)) {\n // found an abandoned page of the right size\n // and claimed ownership.\n mi_page_t* page = mi_arena_page_at_slice(arena, slice_index);\n mi_assert_internal(mi_page_is_owned(page));\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_assert_internal(mi_heap_has_page(heap, arena, page));\n mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]);\n mi_theap_stat_decrease(theap, pages_abandoned, 1);\n mi_theap_stat_counter_increase(theap, pages_reclaim_on_alloc, 1);\n\n _mi_page_free_collect(page, false); // update `used` count\n mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));\n mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));\n mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);\n mi_assert_internal(mi_page_block_size(page) == block_size);\n mi_assert_internal(!mi_page_is_full(page));\n return page;\n }\n }\n }\n mi_forall_suitable_arenas_end();\n return NULL;\n}\n\nstatic uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid) {\n MI_UNUSED_RELEASE(block_size);\n const bool allow_large = (MI_SECURE < 5); // 5 = guard page at end of each arena page\n const size_t page_alignment = MI_ARENA_SLICE_ALIGN;\n\n mi_heap_t* const heap = _mi_theap_heap(theap);\n mi_tld_t* const tld = theap->tld;\n mi_arena_t* const req_arena = heap->exclusive_arena;\n const int numa_node = (heap->numa_node >= 0 ? heap->numa_node : tld->numa_node);\n\n // try to allocate from free space in arena's\n uint8_t* start = NULL;\n *memid = _mi_memid_none();\n const size_t alloc_size = mi_size_of_slices(slice_count);\n if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && // allowed to allocate from arena's?\n !os_align && // not large alignment\n slice_count <= mi_arena_max_object_size()/MI_ARENA_SLICE_SIZE) // and not too large\n {\n start = (uint8_t*)mi_arenas_try_alloc(heap, slice_count, page_alignment, commit, allow_large, req_arena, tld->thread_seq, numa_node, memid);\n if (start != NULL) {\n mi_arena_pages_t* const arena_pages = mi_heap_ensure_arena_pages(heap, memid->mem.arena.arena);\n if (arena_pages==NULL) {\n _mi_arenas_free(start, mi_size_of_slices(slice_count), *memid); // roll back\n start = NULL;\n }\n else {\n // note: the following assert should hold if we could check it atomically, but in a concurrent setting we may already allocate in slice_count\n // mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages, memid->mem.arena.slice_index, memid->mem.arena.slice_count));\n mi_assert_internal(mi_bitmap_is_clear(arena_pages->pages, memid->mem.arena.slice_index));\n mi_bitmap_set(arena_pages->pages, memid->mem.arena.slice_index);\n }\n }\n }\n\n // otherwise fall back to the OS\n if (start == NULL) {\n if (os_align) {\n // note: slice_count already includes the page\n mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment));\n start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid);\n }\n else {\n start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid);\n }\n }\n\n if (start == NULL) return NULL;\n mi_assert_internal(_mi_is_aligned(start, MI_PAGE_ALIGN));\n mi_assert_internal(!os_align || _mi_is_aligned(start + page_alignment, block_alignment));\n return start;\n}\n\nstatic size_t mi_page_block_start(size_t block_size, bool os_align)\n{\n #if MI_GUARDED\n // in a guarded build, we align pages with blocks a multiple of an OS page size, to the OS page size\n // this ensures that all blocks in such pages are OS page size aligned (which is needed for the guard pages)\n const size_t os_page_size = _mi_os_page_size();\n mi_assert_internal(MI_PAGE_ALIGN >= os_page_size);\n if (!os_align && block_size % os_page_size == 0 && block_size > os_page_size /* at least 2 or more */ ) {\n return _mi_align_up(mi_page_info_size(), os_page_size);\n }\n else\n #endif\n if (os_align) {\n return MI_PAGE_ALIGN;\n }\n else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {\n // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB)\n return _mi_align_up(mi_page_info_size(), block_size);\n }\n else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) {\n // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB)\n return _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2);\n }\n else {\n // otherwise start after the info\n return mi_page_info_size();\n }\n}\n\n// Allocate a fresh page\nstatic mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit)\n{\n const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);\n const size_t alloc_size = mi_size_of_slices(slice_count);\n mi_memid_t memid = _mi_memid_none();\n uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,block_size,block_alignment,os_align,commit,&memid);\n if (!slice_start) return NULL;\n\n // guard page at the end of mimalloc page?\n #if MI_SECURE>=5\n mi_assert(alloc_size > _mi_os_secure_guard_page_size());\n const size_t page_noguard_size = alloc_size - _mi_os_secure_guard_page_size();\n if (memid.initially_committed) {\n _mi_os_secure_guard_page_set_at(slice_start + page_noguard_size, memid);\n }\n #else\n const size_t page_noguard_size = alloc_size;\n #endif\n\n // allocate the page meta info\n mi_page_t* page = NULL;\n bool page_meta_is_separate = false;\n size_t block_start = 0;\n\n // allocate page meta info at the arena start?\n if (memid.memkind == MI_MEM_ARENA) {\n mi_arena_t* const arena = memid.mem.arena.arena;\n if (arena->pages_meta != NULL) {\n mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0);\n mi_page_t* const page_meta = &arena->pages_meta[memid.mem.arena.slice_index];\n mi_assert_internal(page_meta->block_size == 0);\n #if MI_PAGE_META_ALIGNED_FREE_SMALL\n // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice,\n // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`)\n if (block_size > MI_SMALL_SIZE_MAX)\n #endif\n {\n page = page_meta;\n page_meta_is_separate = true;\n block_start = 0;\n #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET)\n #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */\n #endif\n if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) {\n block_start += block_size;\n }\n mi_assert_internal(page->block_size == 0);\n _mi_memzero_aligned(page, sizeof(*page));\n }\n }\n }\n if (page == NULL) {\n // put page meta info in front of the slice\n page = (mi_page_t*)slice_start;\n block_start = mi_page_block_start(block_size, os_align);\n }\n\n // commit first block?\n size_t commit_size = 0;\n if (!memid.initially_committed) {\n commit_size = _mi_align_up(block_start + block_size, MI_PAGE_MIN_COMMIT_SIZE);\n if (commit_size > page_noguard_size) { commit_size = page_noguard_size; }\n bool is_zero = false;\n if mi_unlikely(!mi_arena_commit( mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) {\n _mi_arenas_free(slice_start, alloc_size, memid);\n return NULL;\n }\n }\n if (!memid.initially_zero && !page_meta_is_separate) {\n _mi_memzero_aligned(page, sizeof(*page));\n }\n\n // claimed free slices: initialize the page partly\n if (!memid.initially_zero && memid.initially_committed) {\n mi_track_mem_undefined(slice_start, slice_count * MI_ARENA_SLICE_SIZE);\n }\n else if (memid.initially_committed) {\n mi_track_mem_defined(slice_start, slice_count * MI_ARENA_SLICE_SIZE);\n }\n #if MI_DEBUG > 1\n if (memid.initially_zero && memid.initially_committed) {\n if (!mi_mem_is_zero(slice_start, page_noguard_size)) {\n _mi_error_message(EFAULT, \"internal error: page memory was not zero initialized.\\n\");\n memid.initially_zero = false;\n if (block_start > 0) { _mi_memzero_aligned(page, sizeof(*page)); }\n }\n }\n #endif\n const size_t reserved = (os_align ? 1 : (page_noguard_size - block_start) / block_size);\n mi_assert_internal(reserved > 0 && reserved <= UINT16_MAX);\n\n // initialize\n page->reserved = (uint16_t)reserved;\n page->page_start = slice_start + block_start;\n page->block_size = block_size;\n page->slice_committed = commit_size;\n page->memid = memid;\n page->free_is_zero = memid.initially_zero;\n mi_assert_internal(page->free==NULL);\n mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page));\n mi_assert_internal(mi_page_slice_start(page) == slice_start);\n\n // and own it\n mi_page_claim_ownership(page);\n\n // register in the page map\n if mi_unlikely(!_mi_page_map_register(page)) {\n _mi_arenas_free( slice_start, alloc_size, memid );\n return NULL;\n }\n\n // stats\n mi_theap_stat_increase(theap, pages, 1);\n mi_theap_stat_increase(theap, page_bins[_mi_page_stats_bin(page)], 1);\n\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page),MI_PAGE_ALIGN));\n mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);\n mi_assert_internal(mi_page_block_size(page) == block_size);\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_assert_internal(mi_page_is_owned(page));\n\n return page;\n}\n\n// Allocate a regular small/medium/large page.\nstatic mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_count, size_t block_size)\n{\n // 1. look for an abandoned page\n mi_page_t* page = mi_arenas_page_try_find_abandoned(theap, slice_count, block_size);\n if (page != NULL) {\n return page; // return as abandoned\n }\n\n // 2. find a free block, potentially allocating a new arena\n const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand);\n const bool commit = (slice_count <= mi_slice_count_of_size(MI_PAGE_MIN_COMMIT_SIZE) || // always commit small pages\n (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0));\n page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit);\n if (page == NULL) return NULL;\n\n mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count);\n if (!_mi_page_init(theap, page)) {\n _mi_arenas_free( page, mi_page_full_size(page), page->memid);\n return NULL;\n }\n\n return page;\n}\n\n// Allocate a page containing one block (very large, or with large alignment)\nstatic mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment)\n{\n const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);\n const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size());\n #if MI_SECURE < 2\n const size_t slice_count = mi_slice_count_of_size(info_size + block_size);\n #else\n const size_t slice_count = mi_slice_count_of_size(_mi_align_up(info_size + block_size, _mi_os_secure_guard_page_size()) + _mi_os_secure_guard_page_size());\n #endif\n\n mi_page_t* page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, block_alignment, true /* commit singletons always */);\n if (page == NULL) return NULL;\n\n mi_assert(page->reserved == 1);\n if (!_mi_page_init(theap, page)) {\n _mi_arenas_free( page, mi_page_full_size(page), page->memid);\n return NULL;\n }\n\n return page;\n}\n\n\nmi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) {\n mi_page_t* page;\n if mi_unlikely(block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {\n mi_assert_internal(_mi_is_power_of_two(block_alignment));\n page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment);\n }\n else if (block_size <= MI_SMALL_MAX_OBJ_SIZE) {\n page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_SMALL_PAGE_SIZE), block_size);\n }\n else if (block_size <= MI_MEDIUM_MAX_OBJ_SIZE) {\n page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_MEDIUM_PAGE_SIZE), block_size);\n }\n #if MI_ENABLE_LARGE_PAGES\n else if (block_size <= MI_LARGE_MAX_OBJ_SIZE) {\n page = mi_arenas_page_regular_alloc(theap, mi_slice_count_of_size(MI_LARGE_PAGE_SIZE), block_size);\n }\n #endif\n else {\n page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment);\n }\n if mi_unlikely(page == NULL) {\n return NULL;\n }\n // mi_assert_internal(page == NULL || _mi_page_segment(page)->subproc == tld->subproc);\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);\n mi_assert_internal(block_alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN || _mi_is_aligned(mi_page_start(page), block_alignment));\n\n return page;\n}\n\nvoid _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);\n mi_assert_internal(mi_page_is_owned(page));\n mi_assert_internal(mi_page_all_free(page));\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_assert_internal(page->next==NULL && page->prev==NULL);\n mi_assert_internal(current_theapx == NULL || _mi_thread_id()==current_theapx->tld->thread_id);\n\n if (current_theapx != NULL) {\n mi_theap_stat_decrease(current_theapx, page_bins[_mi_page_stats_bin(page)], 1);\n mi_theap_stat_decrease(current_theapx, pages, 1);\n }\n else {\n mi_heap_t* const heap = mi_page_heap(page);\n mi_heap_stat_decrease(heap, page_bins[_mi_page_stats_bin(page)], 1);\n mi_heap_stat_decrease(heap, pages, 1);\n }\n\n #if MI_DEBUG>1\n if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {\n size_t bin = _mi_bin(mi_page_block_size(page));\n size_t slice_index;\n size_t slice_count;\n mi_arena_pages_t* arena_pages = NULL;\n mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);\n mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));\n mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));\n mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1));\n mi_assert_internal(mi_bitmap_is_setN(arena_pages->pages, slice_index, 1));\n // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may\n // be (temporarily) not true if the free happens while trying to reclaim\n // see `mi_arena_try_claim_abandoned`\n }\n #endif\n\n // recommit guard page at the end?\n // we must do this since we may later allocate large spans over this page and cannot have a guard page in between\n #if MI_SECURE >= 5\n if (!page->memid.is_pinned) {\n _mi_os_secure_guard_page_reset_before(mi_page_slice_start(page) + mi_page_full_size(page), page->memid);\n }\n #endif\n\n // unregister page\n _mi_page_map_unregister(page);\n if (page->memid.memkind == MI_MEM_ARENA) {\n mi_arena_pages_t* arena_pages;\n size_t slice_index;\n size_t slice_count; MI_UNUSED(slice_count);\n mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);\n mi_assert_internal(arena_pages!=NULL);\n mi_bitmap_clear(arena_pages->pages, slice_index);\n if (page->slice_committed > 0) {\n // if committed on-demand, set the commit bits to account commit properly\n mi_assert_internal(mi_page_full_size(page) >= page->slice_committed);\n const size_t total_slices = page->slice_committed / MI_ARENA_SLICE_SIZE; // conservative\n //mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, slice_index, total_slices));\n mi_assert_internal(slice_count >= total_slices);\n if (total_slices > 0) {\n mi_bitmap_setN(arena->slices_committed, slice_index, total_slices, NULL);\n }\n // any left over?\n const size_t extra = page->slice_committed % MI_ARENA_SLICE_SIZE;\n if (extra > 0) {\n // pretend it was decommitted already\n mi_subproc_stat_decrease(arena->subproc, committed, extra);\n }\n }\n else {\n mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));\n }\n }\n if (mi_page_meta_is_separated(page)) { page->block_size = 0; } // for assertion checking\n _mi_arenas_free( mi_page_slice_start(page), mi_page_full_size(page), page->memid);\n}\n\n/* -----------------------------------------------------------\n Arena abandon\n----------------------------------------------------------- */\n\nvoid _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) {\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);\n mi_assert_internal(mi_page_is_owned(page));\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_assert_internal(!mi_page_all_free(page));\n mi_assert_internal(page->next==NULL && page->prev == NULL);\n mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id);\n // mi_assert_internal(current_theap == _mi_page_associated_theap(page));\n\n mi_heap_t* heap = mi_page_heap(page); mi_assert_internal(heap==_mi_theap_heap(current_theap));\n if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {\n // make available for allocations\n size_t bin = _mi_bin(mi_page_block_size(page));\n size_t slice_index;\n size_t slice_count;\n mi_arena_pages_t* arena_pages = NULL;\n mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena);\n\n mi_assert_internal(!mi_page_is_singleton(page));\n mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));\n mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));\n mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));\n\n mi_page_set_abandoned_mapped(page);\n const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index);\n MI_UNUSED(was_clear); mi_assert_internal(was_clear);\n mi_atomic_increment_relaxed(&heap->abandoned_count[bin]);\n mi_theap_stat_increase(current_theap, pages_abandoned, 1);\n }\n else {\n // page is full (or a singleton), or the page is OS/externally allocated\n // leave as is; it will be reclaimed when an object is free'd in the page\n // but for non-arena pages, add to the subproc list so these can be visited\n if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {\n mi_lock(&heap->os_abandoned_pages_lock) {\n // push in front\n page->prev = NULL;\n page->next = heap->os_abandoned_pages;\n if (page->next != NULL) { page->next->prev = page; }\n heap->os_abandoned_pages = page;\n }\n }\n mi_theap_stat_increase(current_theap, pages_abandoned, 1);\n }\n mi_abandoned_page_unown(page, current_theap);\n}\n\n\n// this is called from `free.c:mi_free_try_collect_mt` only.\nbool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) {\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);\n mi_assert_internal(mi_page_is_owned(page));\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_assert_internal(!mi_page_is_abandoned_mapped(page));\n mi_assert_internal(!mi_page_is_full(page));\n mi_assert_internal(!mi_page_all_free(page));\n mi_assert_internal(!mi_page_is_singleton(page));\n if (mi_page_is_full(page) || mi_page_is_abandoned_mapped(page) || page->memid.memkind != MI_MEM_ARENA) {\n return false;\n }\n else {\n // do not use _mi_heap_theap as we may call this during shutdown of threads and don't want to reinitialize the theap\n mi_theap_t* const theap = _mi_page_associated_theap_peek(page);\n if (theap == NULL) {\n return false;\n }\n else {\n mi_theap_stat_counter_increase(theap, pages_reabandon_full, 1);\n mi_theap_stat_adjust_decrease(theap, pages_abandoned, 1); // adjust as we are not abandoning fresh\n _mi_arenas_page_abandon(page, theap);\n return true;\n }\n }\n}\n\n// called from `mi_free` if trying to unabandon an abandoned page\nvoid _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) {\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);\n mi_assert_internal(mi_page_is_owned(page));\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_assert_internal(current_theapx==NULL || _mi_thread_id()==current_theapx->tld->thread_id);\n\n mi_heap_t* const heap = mi_page_heap(page);\n if (mi_page_is_abandoned_mapped(page)) {\n mi_assert_internal(page->memid.memkind==MI_MEM_ARENA);\n // remove from the abandoned map\n size_t bin = _mi_bin(mi_page_block_size(page));\n size_t slice_index;\n size_t slice_count;\n mi_arena_pages_t* arena_pages;\n mi_arena_t* arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena);\n\n mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));\n mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));\n\n // this busy waits until a concurrent reader (from alloc_abandoned) is done\n mi_bitmap_clear_once_set(arena_pages->pages_abandoned[bin], slice_index);\n mi_page_clear_abandoned_mapped(page);\n mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]);\n }\n else {\n // page is full (or a singleton), page is OS allocated\n // if not an arena page, remove from the subproc os pages list\n if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {\n mi_lock(&heap->os_abandoned_pages_lock) {\n if (page->prev != NULL) { page->prev->next = page->next; }\n if (page->next != NULL) { page->next->prev = page->prev; }\n if (heap->os_abandoned_pages == page) { heap->os_abandoned_pages = page->next; }\n page->next = NULL;\n page->prev = NULL;\n }\n }\n }\n if (current_theapx!=NULL) {\n mi_theap_stat_decrease(current_theapx, pages_abandoned, 1);\n }\n else {\n mi_heap_stat_decrease(heap, pages_abandoned, 1);\n }\n}\n\n\n/* -----------------------------------------------------------\n Arena free\n----------------------------------------------------------- */\nstatic void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slices);\nstatic void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq);\n\nvoid _mi_arenas_free(void* p, size_t size, mi_memid_t memid) {\n if (p==NULL) return;\n if (size==0) return;\n\n // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.)\n mi_track_mem_undefined(p, size);\n\n if (mi_memkind_is_os(memid.memkind)) {\n // was a direct OS allocation, pass through\n _mi_os_free(p, size, memid);\n }\n else if (memid.memkind == MI_MEM_ARENA) {\n // allocated in an arena\n size_t slice_count;\n size_t slice_index;\n mi_arena_t* arena = mi_arena_from_memid(memid, &slice_index, &slice_count);\n mi_assert_internal((size%MI_ARENA_SLICE_SIZE)==0);\n mi_assert_internal((slice_count*MI_ARENA_SLICE_SIZE)==size);\n mi_assert_internal(mi_arena_slice_start(arena,slice_index) <= (uint8_t*)p);\n mi_assert_internal(mi_arena_slice_start(arena,slice_index) + mi_size_of_slices(slice_count) > (uint8_t*)p);\n // checks\n if (arena == NULL) {\n _mi_error_message(EINVAL, \"trying to free from an invalid arena: %p, size %zu, memid: 0x%zx\\n\", p, size, memid);\n return;\n }\n mi_assert_internal(slice_index < arena->slice_count);\n mi_assert_internal(slice_index >= mi_arena_info_slices(arena));\n if (slice_index < mi_arena_info_slices(arena) || slice_index >= arena->slice_count) {\n _mi_error_message(EINVAL, \"trying to free from an invalid arena block: %p, size %zu, memid: 0x%zx\\n\", p, size, memid);\n return;\n }\n\n // potentially decommit\n if (!arena->memid.is_pinned /* && !arena->memid.initially_committed */) { // todo: allow decommit even if initially committed?\n // (delay) purge the page\n mi_arena_schedule_purge(arena, slice_index, slice_count);\n }\n\n // and make it available to others again\n bool all_inuse = mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);\n if (!all_inuse) {\n _mi_error_message(EAGAIN, \"trying to free an already freed arena block: %p, size %zu\\n\", mi_arena_slice_start(arena,slice_index), mi_size_of_slices(slice_count));\n return;\n };\n }\n else if (memid.memkind == MI_MEM_META) {\n _mi_meta_free(p, size, memid);\n }\n else {\n // arena was none, external, or static; nothing to do\n mi_assert_internal(mi_memid_needs_no_free(memid));\n }\n\n // try to purge expired decommits\n // mi_arenas_try_purge(false, false, NULL);\n}\n\n// Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired\nvoid _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld) {\n mi_arenas_try_purge(force_purge, visit_all, tld->subproc, tld->thread_seq);\n}\n\n\n// Is a pointer contained in the given arena area?\nstatic bool mi_arena_strictly_contains(mi_arena_t* arena, const void* p) {\n return (arena != NULL &&\n mi_arena_start(arena) <= (const uint8_t*)p &&\n mi_arena_start(arena) + mi_size_of_slices(arena->slice_count) >(const uint8_t*)p);\n}\n\n// Is a pointer inside any of our arenas?\nstatic bool mi_arenas_contain_ex(const void* p, mi_arena_t* parent) {\n mi_subproc_t* subproc = _mi_subproc();\n const size_t max_arena = mi_arenas_get_count(subproc);\n for (size_t i = 0; i < max_arena; i++) {\n mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);\n if (arena != NULL) {\n if (parent==NULL || arena==parent || arena->parent==parent) {\n if (mi_arena_strictly_contains(arena, p)) {\n return true;\n }\n }\n }\n }\n return false;\n}\n\n// Is a pointer inside any of our arenas?\nbool _mi_arenas_contain(const void* p) {\n return mi_arenas_contain_ex(p, NULL);\n}\n\n// Is a pointer contained in the given arena area?\nbool mi_arena_contains(mi_arena_id_t arena_id, const void* p) {\n mi_arena_t* arena = _mi_arena_from_id(arena_id);\n if (arena==NULL) return false;\n else if (mi_arena_strictly_contains(arena, p)) return true;\n else return mi_arenas_contain_ex(p, arena); // maybe a subarena?\n}\n\n\n/* -----------------------------------------------------------\n Remove an arena.\n----------------------------------------------------------- */\n\n// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`\n// for dynamic libraries that are unloaded and need to release all their allocated memory.\nstatic void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) {\n mi_assert_internal(subproc != NULL);\n const size_t arena_count = mi_arenas_get_count(subproc);\n for (size_t i = 0; i < arena_count; i++) {\n mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);\n if (arena != NULL) {\n // mi_lock_done(&arena->abandoned_visit_lock);\n mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL);\n if (mi_memkind_is_os(arena->memid.memkind)) {\n _mi_os_free_ex(mi_arena_start(arena), mi_arena_size(arena), true, arena->memid, subproc); // pass `subproc` to avoid accessing the theap pointer (in `_mi_subproc()`)\n }\n }\n }\n // try to lower the max arena.\n size_t expected = arena_count;\n mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, (size_t)0);\n}\n\n\n// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`\n// for dynamic libraries that are unloaded and need to release all their allocated memory.\nvoid _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc) {\n mi_arenas_unsafe_destroy(subproc);\n // mi_arenas_try_purge(true /* force purge */, true /* visit all*/, subproc, 0 /* thread seq */); // purge non-owned arenas\n}\n\n\n/* -----------------------------------------------------------\n Add an arena.\n----------------------------------------------------------- */\n\nstatic bool mi_arenas_add(mi_subproc_t* subproc, mi_arena_t* arena, mi_arena_id_t* arena_id)\n{\n mi_assert_internal(arena != NULL);\n mi_assert_internal(arena->slice_count > 0);\n if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }\n\n // try to find a NULL entry\n mi_arena_t* expected;\n size_t count = mi_arenas_get_count(subproc);\n for( size_t i = 0; i < count; i++) {\n if (mi_arena_from_index(subproc,i) == NULL) {\n arena->arena_idx = i;\n expected = NULL;\n if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[i], &expected, arena)) {\n // success\n if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); }\n return true;\n }\n }\n }\n\n // otherwise, try to allocate a fresh slot\n while(countarena_count, &count, count+1)) {\n arena->arena_idx = count;\n expected = NULL;\n if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[count], &expected, arena)) {\n mi_subproc_stat_counter_increase(arena->subproc, arena_count, 1);\n if (arena_id != NULL) { *arena_id = mi_arena_id_from_arena(arena); }\n return true;\n }\n }\n }\n\n // failed\n arena->arena_idx = 0;\n arena->subproc = NULL;\n return false;\n}\n\nstatic size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) {\n if (slice_count == 0) slice_count = MI_BCHUNK_BITS;\n mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);\n const size_t base_size = _mi_align_up(sizeof(mi_arena_pages_t), MI_BCHUNK_SIZE);\n const size_t bitmaps_count = 1 + MI_ARENA_BIN_COUNT; // pages, and abandoned\n const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL);\n const size_t size = base_size + bitmaps_size;\n if (bitmap_base != NULL) *bitmap_base = base_size;\n return size;\n}\n\nstatic size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) {\n if (slice_count == 0) slice_count = MI_BCHUNK_BITS;\n mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);\n const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE);\n const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned\n const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free\n #if MI_PAGE_META_IS_SEPARATED\n const size_t pages_size = slice_count * sizeof(mi_page_t);\n #else\n const size_t pages_size = 0;\n #endif\n const size_t size = base_size + bitmaps_size + pages_size;\n\n const size_t os_page_size = _mi_os_page_size();\n const size_t info_size = _mi_align_up(size, os_page_size) + _mi_os_secure_guard_page_size();\n const size_t info_slices = mi_slice_count_of_size(info_size);\n\n if (bitmap_base != NULL) *bitmap_base = base_size;\n return info_slices;\n}\n\nstatic mi_bitmap_t* mi_arena_bitmap_init(size_t slice_count, uint8_t** base) {\n mi_bitmap_t* bitmap = (mi_bitmap_t*)(*base);\n *base = (*base) + mi_bitmap_init(bitmap, slice_count, true /* already zero */);\n return bitmap;\n}\n\nstatic mi_bbitmap_t* mi_arena_bbitmap_init(size_t slice_count, uint8_t** base) {\n mi_bbitmap_t* bbitmap = (mi_bbitmap_t*)(*base);\n *base = (*base) + mi_bbitmap_init(bbitmap, slice_count, true /* already zero */);\n return bbitmap;\n}\n\nstatic mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) {\n const size_t slice_count = arena->slice_count;\n size_t bitmap_base = 0;\n const size_t size = mi_arena_pages_size(slice_count, &bitmap_base);\n mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(mi_heap_main(), size, MI_BCHUNK_SIZE);\n if (arena_pages==NULL) return NULL;\n uint8_t* base = (uint8_t*)arena_pages + bitmap_base;\n mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE));\n arena_pages->pages = mi_arena_bitmap_init(slice_count, &base);\n for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {\n arena_pages->pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);\n }\n return arena_pages;\n}\n\nstatic mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,\n size_t slice_count, mi_arena_t* parent, size_t total_size,\n int numa_node, bool exclusive,\n mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id)\n{\n mi_assert_internal(_mi_is_aligned(start,MI_ARENA_SLICE_ALIGN));\n mi_assert_internal(mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE);\n\n if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now\n // note: this should never happen if called from `mi_manage_os_memory` (as that allocates sub-arenas when needed)\n _mi_warning_message(\"cannot use OS memory since it is too large (size %zu MiB, maximum is %zu MiB)\", mi_size_of_slices(slice_count)/MI_MiB, mi_size_of_slices(MI_BITMAP_MAX_BIT_COUNT)/MI_MiB);\n return NULL;\n }\n\n size_t bitmap_base;\n const size_t info_slices = mi_arena_info_slices_needed(slice_count, &bitmap_base);\n if (slice_count < info_slices+1) {\n _mi_warning_message(\"cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)\", mi_size_of_slices(slice_count)/MI_KiB, mi_size_of_slices(info_slices+1)/MI_KiB);\n return NULL;\n }\n else if (info_slices >= MI_ARENA_MAX_CHUNK_OBJ_SLICES) {\n _mi_warning_message(\"cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)\", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES);\n return NULL;\n }\n\n mi_arena_t* arena = (mi_arena_t*)start;\n\n // commit & zero if needed\n if (!memid.initially_committed) {\n size_t commit_size = mi_size_of_slices(info_slices);\n // leave a guard OS page decommitted at the end?\n if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); }\n bool ok = false;\n if (commit_fun != NULL) {\n ok = (*commit_fun)(true /* commit */, arena, commit_size, NULL, commit_fun_arg);\n }\n else {\n ok = _mi_os_commit(arena, commit_size, NULL);\n }\n if (!ok) {\n _mi_warning_message(\"unable to commit meta-data for OS memory\");\n return NULL;\n }\n }\n else if (!memid.is_pinned) {\n // if MI_SECURE, set a guard page at the end of the arena info\n // todo: this does not respect the commit_fun as the memid is of external memory\n _mi_os_secure_guard_page_set_before((uint8_t*)arena + mi_size_of_slices(info_slices), memid);\n }\n if (!memid.initially_zero) {\n _mi_memzero(arena, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size());\n }\n\n // init\n arena->subproc = subproc;\n arena->memid = memid;\n arena->is_exclusive = exclusive;\n arena->slice_count = slice_count;\n arena->info_slices = info_slices;\n if (numa_node<0 && mi_option_is_enabled(mi_option_arena_is_numa_local)) {\n arena->numa_node = _mi_os_numa_node();\n }\n else {\n arena->numa_node = numa_node;\n }\n arena->purge_expire = 0;\n arena->commit_fun = commit_fun;\n arena->commit_fun_arg = commit_fun_arg;\n arena->parent = parent;\n arena->total_size = total_size;\n\n // init bitmaps\n uint8_t* base = mi_arena_start(arena) + bitmap_base;\n arena->slices_free = mi_arena_bbitmap_init(slice_count, &base);\n arena->slices_committed = mi_arena_bitmap_init(slice_count, &base);\n arena->slices_dirty = mi_arena_bitmap_init(slice_count, &base);\n arena->slices_purge = mi_arena_bitmap_init(slice_count, &base);\n arena->pages_main.pages = mi_arena_bitmap_init(slice_count, &base);\n for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {\n arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);\n }\n #if MI_PAGE_META_IS_SEPARATED\n arena->pages_meta = (mi_page_t*)base;\n base += (slice_count * sizeof(mi_page_t));\n #else\n arena->pages_meta = NULL;\n #endif\n mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena)));\n\n // reserve our meta info (and reserve slices outside the memory area)\n mi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices);\n if (memid.initially_committed) {\n mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count);\n }\n if (!memid.initially_zero) {\n mi_bitmap_unsafe_setN(arena->slices_dirty, 0, arena->slice_count);\n }\n\n if (!mi_arenas_add(subproc, arena, arena_id)) { return NULL; }\n return arena;\n}\n\nstatic bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t size, int numa_node, bool exclusive,\n mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept\n{\n // checks\n mi_assert(_mi_is_aligned(start, MI_ARENA_SLICE_SIZE));\n mi_assert(start!=NULL);\n if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }\n if (start==NULL) return false;\n if (!_mi_is_aligned(start, MI_ARENA_SLICE_SIZE)) {\n // we can align the start since the memid tracks the real base of the memory.\n void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_SLICE_SIZE);\n const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start;\n if (diff >= size || (size - diff) < MI_ARENA_SLICE_SIZE) {\n _mi_warning_message(\"after alignment, the size of the arena becomes too small (memory at %p with size %zu)\\n\", start, size);\n return false;\n }\n start = aligned_start;\n size = size - diff;\n }\n\n // allocate enough arena's to span the full memory area\n // the first arena is the owner, the rest are \"sub-arena\" (with `parent` pointing to the first one)\n size_t total_slice_count = _mi_align_down(size / MI_ARENA_SLICE_SIZE, MI_BCHUNK_BITS);\n size_t total_size = mi_size_of_slices(total_slice_count);\n if (total_size < MI_ARENA_MIN_SIZE) {\n _mi_warning_message(\"cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)\", size/MI_KiB, MI_ARENA_MIN_SIZE/MI_KiB);\n return false;\n }\n\n mi_arena_t* parent = NULL;\n do {\n // counting down on the total_slice_count\n size_t slice_count = total_slice_count;\n if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now (with 64KiB slices)\n slice_count = MI_BITMAP_MAX_BIT_COUNT;\n }\n\n // initialize\n mi_arena_t* arena = mi_arena_initialize( subproc, start, slice_count, parent,\n (parent==NULL ? total_size : 0), numa_node, exclusive,\n memid, commit_fun, commit_fun_arg,\n (parent==NULL ? arena_id : NULL));\n if (arena==NULL) {\n // failed to initialize due to failing commit or too many arena's\n if (parent==NULL) {\n return false;\n }\n else {\n // partial success, but failed to use the full area..\n // todo: roll-back in this case? that requires a lock on the arena's array though\n mi_assert(mi_size_of_slices(total_slice_count) <= parent->total_size);\n parent->total_size -= mi_size_of_slices(total_slice_count);\n return true;\n }\n }\n\n // success\n if (parent==NULL) {\n parent = arena;\n memid.memkind = MI_MEM_NONE;\n }\n mi_assert(slice_count <= total_slice_count);\n total_slice_count -= slice_count;\n start = (uint8_t*)start + mi_size_of_slices(slice_count);\n }\n while (total_slice_count > 0);\n\n return true;\n}\n\nbool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {\n mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL);\n memid.mem.os.base = start;\n memid.mem.os.size = size;\n memid.initially_committed = is_committed;\n memid.initially_zero = is_zero;\n memid.is_pinned = is_pinned;\n return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, NULL, NULL, arena_id);\n}\n\nbool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept\n{\n mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL);\n memid.mem.os.base = start;\n memid.mem.os.size = size;\n memid.initially_committed = is_committed;\n memid.initially_zero = is_zero;\n memid.is_pinned = is_pinned;\n return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, commit_fun, commit_fun_arg, arena_id);\n}\n\n\n// Reserve a range of regular OS memory\nstatic int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) {\n if (arena_id != NULL) *arena_id = _mi_arena_id_none();\n size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice\n mi_memid_t memid;\n void* start = _mi_os_alloc_aligned(size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid);\n if (start == NULL) return ENOMEM;\n if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) {\n _mi_os_free_ex(start, size, commit, memid, NULL);\n _mi_verbose_message(\"failed to reserve %zu KiB memory\\n\", _mi_divide_up(size, 1024));\n return ENOMEM;\n }\n _mi_verbose_message(\"reserved %zu KiB memory%s\\n\", _mi_divide_up(size, 1024), memid.is_pinned ? \" (in large os pages)\" : \"\");\n // mi_debug_show_arenas(true, true, false);\n\n return 0;\n}\n\n// Reserve a range of regular OS memory\nint mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {\n return mi_reserve_os_memory_ex2(_mi_subproc(), size, commit, allow_large, exclusive, arena_id);\n}\n\n// Manage a range of regular OS memory\nbool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept {\n return mi_manage_os_memory_ex(start, size, is_committed, is_large, is_zero, numa_node, false /* exclusive? */, NULL);\n}\n\n// Reserve a range of regular OS memory\nint mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept {\n return mi_reserve_os_memory_ex(size, commit, allow_large, false, NULL);\n}\n\n\n/* -----------------------------------------------------------\n Debugging\n----------------------------------------------------------- */\n\n// Return idx of the slice past the last used slice\nstatic size_t mi_arena_used_slices(mi_arena_t* arena) {\n size_t idx;\n if (mi_bbitmap_bsr_inv(arena->slices_free, &idx)) {\n return (idx + 1);\n }\n else {\n return mi_arena_info_slices(arena);\n }\n}\n\nstatic size_t mi_debug_show_bfield(mi_bfield_t field, char* buf, size_t* k) {\n size_t bit_set_count = 0;\n for (int bit = 0; bit < MI_BFIELD_BITS; bit++) {\n bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0);\n if (is_set) bit_set_count++;\n buf[*k] = (is_set ? 'x' : '.');\n *k = *k + 1;\n }\n return bit_set_count;\n}\n\ntypedef enum mi_ansi_color_e {\n MI_BLACK = 30,\n MI_MAROON,\n MI_DARKGREEN,\n MI_ORANGE,\n MI_NAVY,\n MI_PURPLE,\n MI_TEAL,\n MI_GRAY,\n MI_DARKGRAY = 90,\n MI_RED,\n MI_GREEN,\n MI_YELLOW,\n MI_BLUE,\n MI_MAGENTA,\n MI_CYAN,\n MI_WHITE\n} mi_ansi_color_t;\n\nstatic void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) {\n *k += _mi_snprintf(buf + *k, 32, \"\\x1B[%dm\", (int)color);\n}\n\nstatic int mi_page_commit_usage(mi_page_t* page) {\n // if (mi_page_size(page) <= MI_PAGE_MIN_COMMIT_SIZE) return 100;\n const size_t committed_size = mi_page_committed(page);\n const size_t used_size = page->used * mi_page_block_size(page);\n return (int)(used_size * 100 / committed_size);\n}\n\nstatic size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, size_t slice_index, long* pbit_of_page, mi_ansi_color_t* pcolor_of_page ) {\n size_t bit_set_count = 0;\n long bit_of_page = *pbit_of_page;\n mi_ansi_color_t color = *pcolor_of_page;\n mi_ansi_color_t prev_color = MI_GRAY;\n for (int bit = 0; bit < MI_BFIELD_BITS; bit++, bit_of_page--) {\n // bool is_set = ((((mi_bfield_t)1 << bit) & field) != 0);\n void* start = mi_arena_slice_start(arena, slice_index + bit);\n mi_page_t* page = _mi_safe_ptr_page(start);\n char c = ' ';\n if (page!=NULL && start==mi_page_slice_start(page)) {\n mi_assert_internal(bit_of_page <= 0);\n bit_set_count++;\n c = 'p';\n color = MI_GRAY;\n if (mi_page_is_singleton(page)) { c = 's'; }\n else if (mi_page_is_full(page)) { c = 'f'; }\n if (!mi_page_is_abandoned(page)) { c = _mi_toupper(c); }\n int commit_usage = mi_page_commit_usage(page);\n if (commit_usage < 25) { color = MI_MAROON; }\n else if (commit_usage < 50) { color = MI_ORANGE; }\n else if (commit_usage < 75) { color = MI_TEAL; }\n else color = MI_DARKGREEN;\n bit_of_page = (long)page->memid.mem.arena.slice_count;\n }\n else {\n c = '?';\n if (bit_of_page > 0) { c = '-'; }\n else if (_mi_meta_is_meta_page(start)) { c = 'm'; color = MI_GRAY; }\n else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; }\n // else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; }\n else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) {\n if (mi_bitmap_is_set(arena->slices_purge, slice_index + bit)) { c = '~'; color = MI_ORANGE; }\n else if (mi_bitmap_is_set(arena->slices_committed, slice_index + bit)) { c = '_'; color = MI_GRAY; }\n else { c = '.'; color = MI_GRAY; }\n }\n if (bit==MI_BFIELD_BITS-1 && bit_of_page > 1) { c = '>'; }\n }\n if (color != prev_color) {\n mi_debug_color(buf, k, color);\n prev_color = color;\n }\n buf[*k] = c; *k += 1;\n }\n mi_debug_color(buf, k, MI_GRAY);\n *pbit_of_page = bit_of_page;\n *pcolor_of_page = color;\n return bit_set_count;\n}\n\nstatic size_t mi_debug_show_chunks(const char* header1, const char* header2, const char* header3,\n size_t slice_count, size_t chunk_count,\n mi_bchunk_t* chunks, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow)\n{\n _mi_raw_message(\"\\x1B[37m%s%s%s (use/commit: \\x1B[31m0 - 25%%\\x1B[33m - 50%%\\x1B[36m - 75%%\\x1B[32m - 100%%\\x1B[0m)\\n\", header1, header2, header3);\n const size_t fields_per_line = (narrow ? 2 : 4);\n const size_t used_slice_count = mi_arena_used_slices(arena);\n size_t bit_count = 0;\n size_t bit_set_count = 0;\n long bit_of_page = 0;\n mi_ansi_color_t color_of_page = MI_GRAY;\n for (size_t i = 0; i < chunk_count && bit_count < slice_count; i++) {\n char buf[5*MI_BCHUNK_BITS + 64]; _mi_memzero(buf, sizeof(buf));\n if (bit_count > used_slice_count && i+2 < chunk_count) {\n const size_t diff = chunk_count - 1 - i;\n bit_count += diff*MI_BCHUNK_BITS;\n _mi_raw_message(\" |\\n\");\n i = chunk_count-1;\n }\n\n size_t k = 0;\n\n if (i<10) { buf[k++] = ('0' + (char)i); buf[k++] = ' '; buf[k++] = ' '; }\n else if (i<100) { buf[k++] = ('0' + (char)(i/10)); buf[k++] = ('0' + (char)(i%10)); buf[k++] = ' '; }\n else if (i<1000) { buf[k++] = ('0' + (char)(i/100)); buf[k++] = ('0' + (char)((i%100)/10)); buf[k++] = ('0' + (char)(i%10)); }\n\n char chunk_kind = ' ';\n if (chunk_bins != NULL) {\n switch (mi_bbitmap_debug_get_bin(chunk_bins,i)) {\n case MI_CBIN_SMALL: chunk_kind = 'S'; break;\n case MI_CBIN_MEDIUM: chunk_kind = 'M'; break;\n case MI_CBIN_LARGE: chunk_kind = 'L'; break;\n case MI_CBIN_HUGE: chunk_kind = 'H'; break;\n case MI_CBIN_OTHER: chunk_kind = 'X'; break;\n default: chunk_kind = ' '; break; // suppress warning\n // case MI_CBIN_NONE: chunk_kind = 'N'; break;\n }\n }\n buf[k++] = chunk_kind;\n buf[k++] = ' ';\n\n for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {\n if (j > 0 && (j % fields_per_line) == 0) {\n // buf[k++] = '\\n'; _mi_memset(buf+k,' ',7); k += 7;\n _mi_raw_message(\" %s\\n\\x1B[37m\", buf);\n _mi_memzero(buf, sizeof(buf));\n _mi_memset(buf, ' ', 5); k = 5;\n }\n if (bit_count < slice_count) {\n mi_bfield_t bfield = 0;\n if (chunks!=NULL) {\n bfield = chunks[i].bfields[j];\n }\n if (invert) bfield = ~bfield;\n size_t xcount = (chunks==NULL ? mi_debug_show_page_bfield(buf, &k, arena, bit_count, &bit_of_page, &color_of_page)\n : mi_debug_show_bfield(bfield, buf, &k));\n if (invert) xcount = MI_BFIELD_BITS - xcount;\n bit_set_count += xcount;\n buf[k++] = ' ';\n }\n else {\n _mi_memset(buf + k, 'o', MI_BFIELD_BITS);\n k += MI_BFIELD_BITS;\n }\n bit_count += MI_BFIELD_BITS;\n }\n _mi_raw_message(\" %s\\n\\x1B[37m\", buf);\n }\n _mi_raw_message(\"\\x1B[0m total pages: %zu\\n\", bit_set_count);\n return bit_set_count;\n}\n\n//static size_t mi_debug_show_bitmap_binned(const char* header1, const char* header2, const char* header3, size_t slice_count,\n// mi_bitmap_t* bitmap, mi_bchunkmap_t* chunk_bins, bool invert, mi_arena_t* arena, bool narrow) {\n// return mi_debug_show_chunks(header1, header2, header3, slice_count, mi_bitmap_chunk_count(bitmap), &bitmap->chunks[0], chunk_bins, invert, arena, narrow);\n//}\n\nstatic void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narrow) mi_attr_noexcept {\n mi_subproc_t* subproc = heap->subproc;\n size_t max_arenas = mi_arenas_get_count(subproc);\n //size_t free_total = 0;\n //size_t slice_total = 0;\n //size_t abandoned_total = 0;\n size_t page_total = 0;\n for (size_t i = 0; i < max_arenas; i++) {\n mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);\n if (arena == NULL) break;\n mi_assert(arena->subproc == subproc);\n // slice_total += arena->slice_count;\n _mi_raw_message(\"%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %p, numa: %i\\n\",\n (arena->parent==NULL ? \"\" : \"(sub)\"), i, arena, arena->slice_count, (size_t)(mi_size_of_slices(arena->slice_count)/MI_MiB),\n (arena->memid.is_pinned ? \", pinned\" : \"\"), (arena->is_exclusive ? \", exclusive\" : \"\"),\n arena->subproc, arena->numa_node);\n //if (show_inuse) {\n // free_total += mi_debug_show_bbitmap(\"in-use slices\", arena->slice_count, arena->slices_free, true, NULL);\n //}\n //if (show_committed) {\n // mi_debug_show_bitmap(\"committed slices\", arena->slice_count, arena->slices_committed, false, NULL);\n //}\n // todo: abandoned slices\n //if (show_purge) {\n // purge_total += mi_debug_show_bitmap(\"purgeable slices\", arena->slice_count, arena->slices_purge, false, NULL);\n //}\n if (show_pages) {\n // mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);\n // if (arena_pages != NULL)\n {\n const char* header1 = \"chunks (p:page, f:full, s:singleton, P,F,S:not abandoned, i:arena-info, m:meta-data, ~:free-purgable, _:free-committed, .:free-reserved)\";\n const char* header2 = (narrow ? \"\\n \" : \" \");\n const char* header3 = \"(chunk bin: S:small, M : medium, L : large, X : other)\";\n page_total += mi_debug_show_chunks(header1, header2, header3, arena->slice_count,\n mi_bbitmap_chunk_count(arena->slices_free), NULL,\n arena->slices_free->chunkmap_bins, false, arena, narrow);\n }\n }\n }\n // if (show_inuse) _mi_raw_message(\"total inuse slices : %zu\\n\", slice_total - free_total);\n // if (show_abandoned) _mi_raw_message(\"total abandoned slices: %zu\\n\", abandoned_total);\n if (show_pages) _mi_raw_message(\"total pages in arenas: %zu\\n\", page_total);\n}\n\nvoid mi_debug_show_arenas(void) mi_attr_noexcept {\n mi_debug_show_arenas_ex(mi_heap_main(), true /* show pages */, true /* narrow? */);\n}\n\nvoid mi_arenas_print(void) mi_attr_noexcept {\n mi_debug_show_arenas();\n}\n\n\n/* -----------------------------------------------------------\n Reserve a huge page arena.\n----------------------------------------------------------- */\n// reserve at a specific numa node\nint mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {\n if (arena_id != NULL) *arena_id = NULL;\n if (pages==0) return 0;\n if (numa_node < -1) numa_node = -1;\n if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count();\n size_t hsize = 0;\n size_t pages_reserved = 0;\n mi_memid_t memid;\n void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid);\n if (p==NULL || pages_reserved==0) {\n _mi_warning_message(\"failed to reserve %zu GiB huge pages\\n\", pages);\n return ENOMEM;\n }\n _mi_verbose_message(\"numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\\n\", numa_node, pages_reserved, pages);\n\n if (!mi_manage_os_memory_ex2(_mi_subproc(), p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) {\n _mi_os_free(p, hsize, memid);\n return ENOMEM;\n }\n return 0;\n}\n\nint mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept {\n return mi_reserve_huge_os_pages_at_ex(pages, numa_node, timeout_msecs, false, NULL);\n}\n\n// reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected)\nint mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept {\n if (pages == 0) return 0;\n\n // pages per numa node\n int numa_count = (numa_nodes > 0 && numa_nodes <= INT_MAX ? (int)numa_nodes : _mi_os_numa_node_count());\n if (numa_count <= 0) { numa_count = 1; }\n const size_t pages_per = pages / numa_count;\n const size_t pages_mod = pages % numa_count;\n const size_t timeout_per = (timeout_msecs==0 ? 0 : (timeout_msecs / numa_count) + 50);\n\n // reserve evenly among numa nodes\n for (int numa_node = 0; numa_node < numa_count && pages > 0; numa_node++) {\n size_t node_pages = pages_per; // can be 0\n if ((size_t)numa_node < pages_mod) { node_pages++; }\n int err = mi_reserve_huge_os_pages_at(node_pages, numa_node, timeout_per);\n if (err) return err;\n if (pages < node_pages) {\n pages = 0;\n }\n else {\n pages -= node_pages;\n }\n }\n\n return 0;\n}\n\nint mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept {\n MI_UNUSED(max_secs);\n _mi_warning_message(\"mi_reserve_huge_os_pages is deprecated: use mi_reserve_huge_os_pages_interleave/at instead\\n\");\n if (pages_reserved != NULL) *pages_reserved = 0;\n int err = mi_reserve_huge_os_pages_interleave(pages, 0, (size_t)(max_secs * 1000.0));\n if (err==0 && pages_reserved!=NULL) *pages_reserved = pages;\n return err;\n}\n\n\n\n\n\n/* -----------------------------------------------------------\n Arena purge\n----------------------------------------------------------- */\n\nstatic long mi_arena_purge_delay(void) {\n // <0 = no purging allowed, 0=immediate purging, >0=milli-second delay\n return (mi_option_get(mi_option_purge_delay) * mi_option_get(mi_option_arena_purge_mult));\n}\n\n// reset or decommit in an arena and update the commit bitmap\n// assumes we own the area (i.e. slices_free is claimed by us)\n// returns if the memory is no longer committed (versus reset which keeps the commit)\nstatic bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) {\n mi_assert_internal(!arena->memid.is_pinned);\n mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));\n\n const size_t size = mi_size_of_slices(slice_count);\n void* const p = mi_arena_slice_start(arena, slice_index);\n //const bool all_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count);\n size_t already_committed;\n mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed); // pretend all committed.. (as we lack a clearN call that counts the already set bits..)\n const bool all_committed = (already_committed == slice_count);\n const bool needs_recommit = _mi_os_purge_ex(p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg);\n\n if (needs_recommit) {\n // no longer committed\n mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);\n // we just counted in the purge to decommit all, but the some part was not committed so adjust that here\n // mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(slice_count - already_committed));\n }\n else if (!all_committed) {\n // we cannot assume any of these are committed any longer (even with reset since we did setN and may have marked uncommitted slices as committed)\n mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);\n // we adjust the commit count as parts will be re-committed\n // mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed));\n }\n\n return needs_recommit;\n}\n\n\n// Schedule a purge. This is usually delayed to avoid repeated decommit/commit calls.\n// Note: assumes we (still) own the area as we may purge immediately\nstatic void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slice_count) {\n const long delay = mi_arena_purge_delay();\n if (arena->memid.is_pinned || delay < 0 || _mi_preloading()) return; // is purging allowed at all?\n\n mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));\n if (delay == 0) {\n // purge directly\n mi_arena_purge(arena, slice_index, slice_count);\n }\n else {\n // schedule purge\n const mi_msecs_t expire = _mi_clock_now() + delay;\n mi_msecs_t expire0 = 0;\n if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) {\n // expiration was not yet set\n // maybe set the global arenas expire as well (if it wasn't set already)\n mi_assert_internal(expire0==0);\n mi_atomic_casi64_strong_acq_rel(&arena->subproc->purge_expire, &expire0, expire);\n }\n else {\n // already an expiration was set\n }\n mi_bitmap_setN(arena->slices_purge, slice_index, slice_count, NULL);\n }\n}\n\ntypedef struct mi_purge_visit_info_s {\n mi_msecs_t now;\n mi_msecs_t delay;\n bool all_purged;\n bool any_purged;\n} mi_purge_visit_info_t;\n\nstatic bool mi_arena_try_purge_range(mi_arena_t* arena, size_t slice_index, size_t slice_count) {\n mi_assert(slice_count < MI_BCHUNK_BITS);\n if (mi_bbitmap_try_clearNC(arena->slices_free, slice_index, slice_count)) {\n // purge\n bool decommitted = mi_arena_purge(arena, slice_index, slice_count); MI_UNUSED(decommitted);\n mi_assert_internal(!decommitted || mi_bitmap_is_clearN(arena->slices_committed, slice_index, slice_count));\n // and reset the free range\n mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);\n return true;\n }\n else {\n // was allocated again already\n return false;\n }\n}\n\nstatic bool mi_arena_try_purge_visitor(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) {\n mi_purge_visit_info_t* vinfo = (mi_purge_visit_info_t*)arg;\n // try to purge: first claim the free blocks\n if (mi_arena_try_purge_range(arena, slice_index, slice_count)) {\n vinfo->any_purged = true;\n vinfo->all_purged = true;\n }\n else if (slice_count > 1)\n {\n // failed to claim the full range, try per slice instead\n for (size_t i = 0; i < slice_count; i++) {\n const bool purged = mi_arena_try_purge_range(arena, slice_index + i, 1);\n vinfo->any_purged = vinfo->any_purged || purged;\n vinfo->all_purged = vinfo->all_purged && purged;\n }\n }\n // don't clear the purge bits as that is done atomically be the _bitmap_forall_set_ranges\n // mi_bitmap_clearN(arena->slices_purge, slice_index, slice_count);\n return true; // continue\n}\n\n// returns\n// -1 = nothing was purged\n// 0 = nothing was purged yet because have not yet reached the expire time\n// 1 = some pages in the arena were purged\nstatic int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)\n{\n // check pre-conditions\n if (arena->memid.is_pinned) return -1;\n\n // expired yet?\n mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire);\n if (!force) {\n if (expire==0) return -1;\n if (expire > now) return 0;\n }\n\n // reset expire\n mi_atomic_storei64_release(&arena->purge_expire, (mi_msecs_t)0);\n mi_subproc_stat_counter_increase(arena->subproc, arena_purges, 1);\n\n // go through all purge info's (with max MI_BFIELD_BITS ranges at a time)\n // this also clears those ranges atomically (so any newly freed blocks will get purged next\n // time around)\n mi_purge_visit_info_t vinfo = { now, mi_arena_purge_delay(), true /*all?*/, false /*any?*/};\n\n // we purge by at least `minslices` to not fragment transparent huge pages for example\n const size_t minslices = mi_slice_count_of_size(_mi_os_minimal_purge_size());\n _mi_bitmap_forall_setc_rangesn(arena->slices_purge, minslices, &mi_arena_try_purge_visitor, arena, &vinfo);\n\n return (vinfo.any_purged ? 1 : -1);\n}\n\n\nstatic void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq)\n{\n // try purge can be called often so try to only run when needed\n const long delay = mi_arena_purge_delay();\n if (_mi_preloading() || delay <= 0) return; // nothing will be scheduled\n\n // check if any arena needs purging?\n const mi_msecs_t now = _mi_clock_now();\n const mi_msecs_t arenas_expire = mi_atomic_loadi64_acquire(&subproc->purge_expire);\n if (!visit_all && !force && (arenas_expire == 0 || arenas_expire > now)) return;\n\n const size_t max_arena = mi_arenas_get_count(subproc);\n if (max_arena == 0) return;\n\n // allow only one thread to purge at a time (todo: allow concurrent purging?)\n static mi_atomic_guard_t purge_guard;\n mi_atomic_guard(&purge_guard)\n {\n // increase global expire: at most one purge per delay cycle\n if (arenas_expire > now) { mi_atomic_storei64_release(&subproc->purge_expire, now + (delay/10)); }\n const size_t arena_start = tseq % max_arena;\n size_t max_purge_count = (visit_all ? max_arena : (max_arena/4)+1);\n bool all_visited = true;\n bool any_purged = false;\n for (size_t _i = 0; _i < max_arena; _i++) {\n size_t i = _i + arena_start;\n if (i >= max_arena) { i -= max_arena; }\n mi_arena_t* arena = mi_arena_from_index(subproc,i);\n if (arena != NULL) {\n const int purged = mi_arena_try_purge(arena, now, force);\n if (purged >= 0) { // purged, or arena expire is not yet reached\n any_purged = true;\n if (purged >= 1) { // purged\n if (max_purge_count <= 1) {\n all_visited = false;\n break;\n }\n max_purge_count--;\n }\n }\n }\n }\n if (all_visited && !any_purged) {\n mi_atomic_storei64_release(&subproc->purge_expire, (mi_msecs_t)0);\n }\n }\n}\n\n\n/* -----------------------------------------------------------\n Visit all pages and blocks in a heap\n----------------------------------------------------------- */\n\ntypedef struct mi_heap_visit_info_s {\n mi_heap_t* heap;\n mi_block_visit_fun* visitor;\n void* arg;\n bool visit_blocks;\n} mi_heap_visit_info_t;\n\nstatic bool mi_heap_visit_page(mi_page_t* page, mi_heap_visit_info_t* vinfo) {\n mi_heap_area_t area;\n _mi_heap_area_init(&area, page);\n mi_assert_internal(vinfo->heap == mi_page_heap(page));\n if (!vinfo->visitor(vinfo->heap, &area, NULL, area.block_size, vinfo->arg)) {\n return false;\n }\n if (vinfo->visit_blocks) {\n return _mi_theap_area_visit_blocks(&area, page, vinfo->visitor, vinfo->arg);\n }\n else {\n return true;\n }\n}\n\nstatic bool mi_heap_visit_page_at(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) {\n MI_UNUSED(slice_count);\n mi_heap_visit_info_t* vinfo = (mi_heap_visit_info_t*)arg;\n mi_page_t* page = mi_arena_page_at_slice(arena, slice_index);\n return mi_heap_visit_page(page, vinfo);\n}\n\nbool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {\n mi_assert(visitor!=NULL);\n if (visitor==NULL) return false;\n if (heap==NULL) { heap = mi_heap_main(); }\n // visit all pages in a heap\n // we don't have to claim because we assume we are the only thread running (with this heap).\n // (but we could atomically claim as well by first doing abandoned_reclaim and afterwards reabandoning).\n mi_heap_visit_info_t visit_info = { heap, visitor, arg, visit_blocks };\n bool ok = true;\n mi_forall_arenas(heap, NULL, 0, arena) {\n mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);\n if (ok && arena_pages != NULL) {\n if (abandoned_only) {\n for (size_t bin = 0; ok && bin < MI_BIN_COUNT; bin++) {\n // todo: if we had a single abandoned page map as well, this can be faster.\n if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) {\n ok = _mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info);\n }\n }\n }\n else {\n ok = _mi_bitmap_forall_set(arena_pages->pages, &mi_heap_visit_page_at, arena, &visit_info);\n }\n }\n }\n mi_forall_arenas_end();\n if (!ok) return false;\n\n // visit abandoned pages in OS allocated memory\n // (technically we don't need the initial lock as we assume we are the only thread running in this subproc)\n mi_page_t* page = NULL;\n mi_lock(&heap->os_abandoned_pages_lock) {\n page = heap->os_abandoned_pages;\n }\n while (ok && page != NULL) {\n mi_page_t* next = page->next; // read upfront in case the visitor frees the page\n ok = mi_heap_visit_page(page, &visit_info);\n page = next;\n }\n\n return ok;\n}\n\nbool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {\n return _mi_heap_visit_blocks(heap, false, visit_blocks, visitor, arg);\n}\n\nbool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {\n return _mi_heap_visit_blocks(heap, true, visit_blocks, visitor, arg);\n}\n\n\ntypedef struct mi_heap_delete_visit_info_s {\n mi_heap_t* heap_target;\n mi_theap_t* theap_target;\n mi_theap_t* theap;\n} mi_heap_delete_visit_info_t;\n\nstatic bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) {\n MI_UNUSED(block); MI_UNUSED(block_size); MI_UNUSED(heap);\n mi_heap_delete_visit_info_t* info = (mi_heap_delete_visit_info_t*)arg;\n mi_heap_t* heap_target = info->heap_target;\n mi_theap_t* const theap = NULL; // info->theap; mi_assert_internal(_mi_theap_heap(theap) == heap);\n mi_page_t* const page = (mi_page_t*)area->reserved1;\n\n mi_page_claim_ownership(page); // claim ownership\n if (mi_page_is_abandoned(page)) {\n _mi_arenas_page_unabandon(page,theap);\n }\n else {\n page->next = page->prev = NULL; // yikes.. better not to try to access this from a thread later on..\n mi_page_set_theap(page,NULL); // set threadid to abandoned\n }\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_assert_internal(mi_page_is_owned(page));\n\n if (page->used==0) {\n // free the page\n _mi_arenas_page_free(page, theap);\n }\n else if (heap_target==NULL) {\n // destroy the page\n page->used=0; // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case\n _mi_arenas_page_free(page, theap);\n }\n else {\n // move the page to `heap_target` as an abandoned page\n // first remove it from the current heap\n const size_t sbin = _mi_page_stats_bin(page);\n size_t slice_index;\n size_t slice_count;\n mi_arena_pages_t* arena_pages = NULL;\n mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);\n mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index));\n mi_bitmap_clear(arena_pages->pages, slice_index);\n if (theap != NULL) {\n mi_theap_stat_decrease(theap, page_bins[sbin], 1);\n mi_theap_stat_decrease(theap, pages, 1);\n }\n else {\n mi_heap_stat_decrease((mi_heap_t*)heap, page_bins[_mi_page_stats_bin(page)], 1);\n mi_heap_stat_decrease((mi_heap_t*)heap, pages, 1);\n }\n mi_theap_t* theap_target = info->theap_target;\n\n // and then add it to the new target heap\n mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);\n if mi_unlikely(arena_pages_target==NULL) {\n // if we cannot allocate this, we move it to the main heap instead (which does not require allocation)\n heap_target = mi_heap_main();\n theap_target = mi_heap_theap(heap_target);\n arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);\n mi_assert_internal(arena_pages_target!=NULL);\n }\n mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index));\n mi_bitmap_set(arena_pages_target->pages, slice_index);\n page->heap = heap_target;\n mi_theap_stat_increase(theap_target, page_bins[sbin], 1);\n mi_theap_stat_increase(theap_target, pages, 1);\n\n // and abandon in the new heap\n _mi_arenas_page_abandon(page,theap_target);\n }\n return true;\n}\n\nstatic void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) {\n mi_theap_t* const theap_target = (heap_target != NULL ? _mi_heap_theap(heap_target) : NULL);\n // mi_theap_t* const theap = _mi_heap_theap(heap);\n mi_heap_delete_visit_info_t info = { heap_target, theap_target, NULL };\n _mi_heap_visit_blocks(heap, false, false, &mi_heap_delete_page, &info);\n #if MI_DEBUG>1\n // no more arena pages?\n for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {\n mi_arena_pages_t* const arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);\n if (arena_pages!=NULL) {\n mi_assert_internal(mi_bitmap_is_all_clear(arena_pages->pages));\n }\n }\n // nor os abandoned pages?\n mi_lock(&heap->os_abandoned_pages_lock) {\n\n mi_assert_internal(heap->os_abandoned_pages == NULL);\n }\n // nor arena abandoned pages?\n for (size_t i = 0; i < MI_BIN_COUNT; i++) {\n mi_assert_internal(mi_atomic_load_relaxed(&heap->abandoned_count[i])==0);\n }\n #endif\n}\n\nvoid _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to) {\n if (_mi_is_heap_main(heap_from)) return;\n if (heap_to==NULL) { heap_to = mi_heap_main(); }\n mi_heap_delete_pages(heap_from, heap_to);\n}\n\nvoid _mi_heap_destroy_pages(mi_heap_t* heap_from) {\n if (_mi_is_heap_main(heap_from)) return;\n mi_heap_delete_pages(heap_from, NULL);\n}\n\n/* -----------------------------------------------------------\n Unloading and reloading an arena.\n----------------------------------------------------------- */\n/*\nstatic bool mi_arena_page_register(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) {\n MI_UNUSED(arg); MI_UNUSED(slice_count);\n mi_assert_internal(slice_count == 1);\n mi_page_t* page = mi_arena_page_at_slice(arena, slice_index);\n mi_assert_internal(mi_bitmap_is_setN(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index, 1));\n if (!_mi_page_map_register(page)) return false; // break\n mi_assert_internal(_mi_ptr_page(page)==page);\n return true;\n}\n\nmi_decl_nodiscard static bool mi_arena_pages_reregister(mi_arena_t* arena) {\n return _mi_bitmap_forall_set(arena->pages, &mi_arena_page_register, arena, NULL);\n}\n\nmi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* full_size) {\n mi_arena_t* arena = _mi_arena_from_id(arena_id);\n if (arena==NULL) {\n return false;\n }\n else if (!arena->is_exclusive) {\n _mi_warning_message(\"cannot unload a non-exclusive arena (id %zu at %p)\\n\", arena_id, arena);\n return false;\n }\n else if (arena->memid.memkind != MI_MEM_EXTERNAL) {\n _mi_warning_message(\"can only unload managed arena's for external memory (id %zu at %p)\\n\", arena_id, arena);\n return false;\n }\n\n // find accessed size\n const size_t asize = mi_size_of_slices(mi_arena_used_slices(arena));\n if (base != NULL) { *base = (void*)arena; }\n if (full_size != NULL) { *full_size = arena->memid.mem.os.size; }\n if (accessed_size != NULL) { *accessed_size = asize; }\n\n // adjust abandoned page count\n mi_subproc_t* const subproc = arena->subproc;\n for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {\n const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]);\n if (count > 0) { mi_atomic_decrement_acq_rel(&subproc->abandoned_count[bin]); }\n }\n\n // unregister the pages\n _mi_page_map_unregister_range(arena, asize);\n\n // set arena entry to NULL\n const size_t count = mi_arenas_get_count(subproc);\n for(size_t i = 0; i < count; i++) {\n if (mi_arena_from_index(subproc, i) == arena) {\n mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL);\n if (i + 1 == count) { // try adjust the count?\n size_t expected = count;\n mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, count-1);\n }\n break;\n }\n }\n return true;\n}\n\nmi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) {\n // assume the memory area is already containing the arena\n if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }\n if (start == NULL || size == 0) return false;\n mi_arena_t* arena = (mi_arena_t*)start;\n mi_memid_t memid = arena->memid;\n if (memid.memkind != MI_MEM_EXTERNAL) {\n _mi_warning_message(\"can only reload arena's from external memory (%p)\\n\", arena);\n return false;\n }\n if (memid.mem.os.base != start) {\n _mi_warning_message(\"the reloaded arena base address differs from the external memory (arena: %p, external: %p)\\n\", arena, start);\n return false;\n }\n if (memid.mem.os.size != size) {\n _mi_warning_message(\"the reloaded arena size differs from the external memory (arena size: %zu, external size: %zu)\\n\", arena->memid.mem.os.size, size);\n return false;\n }\n if (!arena->is_exclusive) {\n _mi_warning_message(\"the reloaded arena is not exclusive\\n\");\n return false;\n }\n\n // re-initialize\n arena->is_exclusive = true;\n arena->commit_fun = commit_fun;\n arena->commit_fun_arg = commit_fun_arg;\n arena->subproc = _mi_subproc();\n if (!mi_arenas_add(arena->subproc, arena, arena_id)) {\n return false;\n }\n if (!mi_arena_pages_reregister(arena)) {\n // todo: clear arena entry in the subproc?\n return false;\n }\n\n // adjust abandoned page count\n for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {\n const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]);\n if (count > 0) { mi_atomic_decrement_acq_rel(&arena->subproc->abandoned_count[bin]); }\n }\n\n return true;\n}\n\n*/\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/bitmap.c", "sha256": "de16033e029488e9ade4152ea1274e1a5ecbdc2c8a8d8ee50b745fd037ee6f6c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2024 Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* ----------------------------------------------------------------------------\nConcurrent bitmap that can set/reset sequences of bits atomically\n---------------------------------------------------------------------------- */\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/bits.h\"\n#include \"mimalloc/prim.h\" // _mi_prim_thread_yield\n#include \"bitmap.h\"\n\n#ifndef MI_OPT_SIMD\n#define MI_OPT_SIMD 0\n#endif\n\n/* --------------------------------------------------------------------------------\n bfields\n-------------------------------------------------------------------------------- */\n\nstatic inline size_t mi_bfield_ctz(mi_bfield_t x) {\n return mi_ctz(x);\n}\n\nstatic inline size_t mi_bfield_clz(mi_bfield_t x) {\n return mi_clz(x);\n}\n\nstatic inline size_t mi_bfield_popcount(mi_bfield_t x) {\n return mi_popcount(x);\n}\n\nstatic inline mi_bfield_t mi_bfield_clear_least_bit(mi_bfield_t x) {\n return (x & (x-1));\n}\n\n// find the least significant bit that is set (i.e. count trailing zero's)\n// return false if `x==0` (with `*idx` undefined) and true otherwise,\n// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).\nstatic inline bool mi_bfield_find_least_bit(mi_bfield_t x, size_t* idx) {\n return mi_bsf(x,idx);\n}\n\n// find the most significant bit that is set.\n// return false if `x==0` (with `*idx` undefined) and true otherwise,\n// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).\nstatic inline bool mi_bfield_find_highest_bit(mi_bfield_t x, size_t* idx) {\n return mi_bsr(x, idx);\n}\n\n\n\n// find each set bit in a bit field `x` and clear it, until it becomes zero.\nstatic inline bool mi_bfield_foreach_bit(mi_bfield_t* x, size_t* idx) {\n const bool found = mi_bfield_find_least_bit(*x, idx);\n *x = mi_bfield_clear_least_bit(*x);\n return found;\n}\n\nstatic inline mi_bfield_t mi_bfield_zero(void) {\n return 0;\n}\n\nstatic inline mi_bfield_t mi_bfield_one(void) {\n return 1;\n}\n\nstatic inline mi_bfield_t mi_bfield_all_set(void) {\n return ~((mi_bfield_t)0);\n}\n\n// mask of `bit_count` bits set shifted to the left by `shiftl`\nstatic inline mi_bfield_t mi_bfield_mask(size_t bit_count, size_t shiftl) {\n mi_assert_internal(bit_count > 0);\n mi_assert_internal(bit_count + shiftl <= MI_BFIELD_BITS);\n mi_assert_internal(shiftl < MI_BFIELD_BITS);\n const mi_bfield_t mask0 = (bit_count < MI_BFIELD_BITS ? (mi_bfield_one() << bit_count)-1 : mi_bfield_all_set());\n return (mask0 << shiftl);\n}\n\n\n// ------- mi_bfield_atomic_set ---------------------------------------\n// the `_set` functions return also the count of bits that were already set (for commit statistics)\n// the `_clear` functions return also whether the new bfield is all clear or not (for the chunk_map)\n\n// Set a bit atomically. Returns `true` if the bit transitioned from 0 to 1\nstatic inline bool mi_bfield_atomic_set(_Atomic(mi_bfield_t)*b, size_t idx) {\n mi_assert_internal(idx < MI_BFIELD_BITS);\n const mi_bfield_t mask = mi_bfield_mask(1, idx);;\n const mi_bfield_t old = mi_atomic_or_acq_rel(b, mask);\n return ((old&mask) == 0);\n}\n\n// Clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0.\n// `all_clear` is set if the new bfield is zero.\nstatic inline bool mi_bfield_atomic_clear(_Atomic(mi_bfield_t)*b, size_t idx, bool* all_clear) {\n mi_assert_internal(idx < MI_BFIELD_BITS);\n const mi_bfield_t mask = mi_bfield_mask(1, idx);;\n mi_bfield_t old = mi_atomic_and_acq_rel(b, ~mask);\n if (all_clear != NULL) { *all_clear = ((old&~mask)==0); }\n return ((old&mask) == mask);\n}\n\n// Clear a bit but only when/once it is set. This is used by concurrent free's while\n// the page is abandoned and mapped. This can incure a busy wait :-( but it should\n// happen almost never (and is accounted for in the stats)\nstatic inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_t idx) {\n mi_assert_internal(idx < MI_BFIELD_BITS);\n const mi_bfield_t mask = mi_bfield_mask(1, idx);;\n mi_bfield_t old = mi_atomic_load_relaxed(b);\n do {\n if mi_unlikely((old&mask) == 0) {\n old = mi_atomic_load_acquire(b);\n if ((old&mask)==0) {\n mi_subproc_stat_counter_increase(_mi_subproc(), pages_unabandon_busy_wait, 1);\n }\n while ((old&mask)==0) { // busy wait\n _mi_prim_thread_yield(); \n old = mi_atomic_load_acquire(b);\n }\n }\n } while (!mi_atomic_cas_weak_acq_rel(b,&old, (old&~mask)));\n mi_assert_internal((old&mask)==mask); // we should only clear when it was set\n}\n\n// Set a mask set of bits atomically, and return true of the mask bits transitioned from all 0's to 1's.\n// `already_set` contains the count of bits that were already set (used when committing ranges to account\n// statistics correctly).\nstatic inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_set) {\n mi_assert_internal(mask != 0);\n mi_bfield_t old = mi_atomic_load_relaxed(b);\n while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)) {}; // try to atomically set the mask bits until success\n if (already_set!=NULL) { *already_set = mi_bfield_popcount(old&mask); }\n return ((old&mask) == 0);\n}\n\n// Clear a mask set of bits atomically, and return true of the mask bits transitioned from all 1's to 0's\n// `all_clear` is set to `true` if the new bfield became zero.\nstatic inline bool mi_bfield_atomic_clear_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, bool* all_clear) {\n mi_assert_internal(mask != 0);\n mi_bfield_t old = mi_atomic_load_relaxed(b);\n while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)) {}; // try to atomically clear the mask bits until success\n if (all_clear != NULL) { *all_clear = ((old&~mask)==0); }\n return ((old&mask) == mask);\n}\n\nstatic inline bool mi_bfield_atomic_setX(_Atomic(mi_bfield_t)*b, size_t* already_set) {\n const mi_bfield_t old = mi_atomic_exchange_release(b, mi_bfield_all_set());\n if (already_set!=NULL) { *already_set = mi_bfield_popcount(old); }\n return (old==0);\n}\n\n// static inline bool mi_bfield_atomic_clearX(_Atomic(mi_bfield_t)*b, bool* all_clear) {\n// const mi_bfield_t old = mi_atomic_exchange_release(b, mi_bfield_zero());\n// if (all_clear!=NULL) { *all_clear = true; }\n// return (~old==0);\n// }\n\n// ------- mi_bfield_atomic_try_clear ---------------------------------------\n\n\n// Tries to clear a mask atomically, and returns true if the mask bits atomically transitioned from mask to 0\n// and false otherwise (leaving the bit field as is).\n// `all_clear` is set to `true` if the new bfield became zero.\nstatic inline bool mi_bfield_atomic_try_clear_mask_of(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t expect, bool* all_clear) {\n mi_assert_internal(mask != 0);\n // try to atomically clear the mask bits\n do {\n if ((expect & mask) != mask) { // are all bits still set?\n if (all_clear != NULL) { *all_clear = (expect == 0); }\n return false;\n }\n } while (!mi_atomic_cas_weak_acq_rel(b, &expect, expect & ~mask));\n if (all_clear != NULL) { *all_clear = ((expect & ~mask) == 0); }\n return true;\n}\n\nstatic inline bool mi_bfield_atomic_try_clear_mask(_Atomic(mi_bfield_t)* b, mi_bfield_t mask, bool* all_clear) {\n mi_assert_internal(mask != 0);\n const mi_bfield_t expect = mi_atomic_load_relaxed(b);\n return mi_bfield_atomic_try_clear_mask_of(b, mask, expect, all_clear);\n}\n\n// Tries to clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0\n// and `false` otherwise leaving the bfield `b` as-is.\n// `all_clear` is set to true if the new bfield became zero (and false otherwise)\nmi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear(_Atomic(mi_bfield_t)* b, size_t idx, bool* all_clear) {\n mi_assert_internal(idx < MI_BFIELD_BITS);\n const mi_bfield_t mask = mi_bfield_one()<bfields[i], idx);\n if (already_set != NULL) { *already_set = (was_clear ? 0 : 1); }\n return was_clear;\n}\n\n// Set `0 < n <= MI_BFIELD_BITS`, and return true of the mask bits transitioned from all 0's to 1's.\n// `already_set` contains the count of bits that were already set (used when committing ranges to account\n// statistics correctly).\n// Can cross over two bfields.\nstatic inline bool mi_bchunk_setNX(mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* already_set) {\n mi_assert_internal(cidx < MI_BCHUNK_BITS);\n mi_assert_internal(n > 0 && n <= MI_BFIELD_BITS);\n const size_t i = cidx / MI_BFIELD_BITS;\n const size_t idx = cidx % MI_BFIELD_BITS;\n if mi_likely(idx + n <= MI_BFIELD_BITS) {\n // within one field\n return mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(n,idx), already_set);\n }\n else {\n // spanning two fields\n const size_t m = MI_BFIELD_BITS - idx; // bits to clear in the first field\n mi_assert_internal(m < n);\n mi_assert_internal(i < MI_BCHUNK_FIELDS - 1);\n mi_assert_internal(idx + m <= MI_BFIELD_BITS);\n size_t already_set1;\n const bool all_set1 = mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &already_set1);\n mi_assert_internal(n - m > 0);\n mi_assert_internal(n - m < MI_BFIELD_BITS);\n size_t already_set2;\n const bool all_set2 = mi_bfield_atomic_set_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &already_set2);\n if (already_set != NULL) { *already_set = already_set1 + already_set2; }\n return (all_set1 && all_set2);\n }\n}\n\n// Set a sequence of `n` bits within a chunk.\n// Returns true if all bits transitioned from 0 to 1 (or 1 to 0).\nmi_decl_noinline static bool mi_bchunk_xsetNC(mi_xset_t set, mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* palready_set, bool* pmaybe_all_clear) {\n mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);\n mi_assert_internal(n>0);\n bool all_transition = true;\n bool maybe_all_clear = true;\n size_t total_already_set = 0;\n size_t idx = cidx % MI_BFIELD_BITS;\n size_t field = cidx / MI_BFIELD_BITS;\n while (n > 0) {\n size_t m = MI_BFIELD_BITS - idx; // m is the bits to xset in this field\n if (m > n) { m = n; }\n mi_assert_internal(idx + m <= MI_BFIELD_BITS);\n mi_assert_internal(field < MI_BCHUNK_FIELDS);\n const mi_bfield_t mask = mi_bfield_mask(m, idx);\n size_t already_set = 0;\n bool all_clear = false;\n const bool transition = (set ? mi_bfield_atomic_set_mask(&chunk->bfields[field], mask, &already_set)\n : mi_bfield_atomic_clear_mask(&chunk->bfields[field], mask, &all_clear));\n mi_assert_internal((transition && already_set == 0) || (!transition && already_set > 0));\n all_transition = all_transition && transition;\n total_already_set += already_set;\n maybe_all_clear = maybe_all_clear && all_clear;\n // next field\n field++;\n idx = 0;\n mi_assert_internal(m <= n);\n n -= m;\n }\n if (palready_set!=NULL) { *palready_set = total_already_set; }\n if (pmaybe_all_clear!=NULL) { *pmaybe_all_clear = maybe_all_clear; }\n return all_transition;\n}\n\nstatic inline bool mi_bchunk_setN(mi_bchunk_t* chunk, size_t cidx, size_t n, size_t* already_set) {\n mi_assert_internal(n>0 && n <= MI_BCHUNK_BITS);\n if (n==1) return mi_bchunk_set(chunk, cidx, already_set);\n // if (n==8 && (cidx%8) == 0) return mi_bchunk_set8(chunk, cidx, already_set);\n // if (n==MI_BFIELD_BITS) return mi_bchunk_setX(chunk, cidx, already_set);\n if (n<=MI_BFIELD_BITS) return mi_bchunk_setNX(chunk, cidx, n, already_set);\n return mi_bchunk_xsetNC(MI_BIT_SET, chunk, cidx, n, already_set, NULL);\n}\n\n// ------- mi_bchunk_clear ---------------------------------------\n\nstatic inline bool mi_bchunk_clear(mi_bchunk_t* chunk, size_t cidx, bool* all_clear) {\n mi_assert_internal(cidx < MI_BCHUNK_BITS);\n const size_t i = cidx / MI_BFIELD_BITS;\n const size_t idx = cidx % MI_BFIELD_BITS;\n return mi_bfield_atomic_clear(&chunk->bfields[i], idx, all_clear);\n}\n\nstatic inline bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) {\n mi_assert_internal(n>0 && n <= MI_BCHUNK_BITS);\n if (n==1) return mi_bchunk_clear(chunk, cidx, maybe_all_clear);\n // if (n==8) return mi_bchunk_clear8(chunk, cidx, maybe_all_clear);\n // if (n==MI_BFIELD_BITS) return mi_bchunk_clearX(chunk, cidx, maybe_all_clear);\n // TODO: implement mi_bchunk_xsetNX instead of setNX\n return mi_bchunk_xsetNC(MI_BIT_CLEAR, chunk, cidx, n, NULL, maybe_all_clear);\n}\n\n// Check if a sequence of `n` bits within a chunk are all set/cleared.\n// This can cross bfield's\nmi_decl_noinline static size_t mi_bchunk_popcountNC(mi_bchunk_t* chunk, size_t field_idx, size_t idx, size_t n) {\n mi_assert_internal((field_idx*MI_BFIELD_BITS) + idx + n <= MI_BCHUNK_BITS);\n size_t count = 0;\n while (n > 0) {\n size_t m = MI_BFIELD_BITS - idx; // m is the bits to xset in this field\n if (m > n) { m = n; }\n mi_assert_internal(idx + m <= MI_BFIELD_BITS);\n mi_assert_internal(field_idx < MI_BCHUNK_FIELDS);\n const size_t mask = mi_bfield_mask(m, idx);\n count += mi_bfield_atomic_popcount_mask(&chunk->bfields[field_idx], mask);\n // next field\n field_idx++;\n idx = 0;\n n -= m;\n }\n return count;\n}\n\n// Count set bits a sequence of `n` bits.\nstatic inline size_t mi_bchunk_popcountN(mi_bchunk_t* chunk, size_t cidx, size_t n) {\n mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);\n mi_assert_internal(n>0);\n if (n==0) return 0;\n const size_t i = cidx / MI_BFIELD_BITS;\n const size_t idx = cidx % MI_BFIELD_BITS;\n if (n==1) { return (mi_bfield_atomic_is_set(&chunk->bfields[i], idx) ? 1 : 0); }\n if (idx + n <= MI_BFIELD_BITS) { return mi_bfield_atomic_popcount_mask(&chunk->bfields[i], mi_bfield_mask(n, idx)); }\n return mi_bchunk_popcountNC(chunk, i, idx, n);\n}\n\n\n// ------- mi_bchunk_is_xset ---------------------------------------\n\n// Check if a sequence of `n` bits within a chunk are all set/cleared.\n// This can cross bfield's\nmi_decl_noinline static bool mi_bchunk_is_xsetNC(mi_xset_t set, const mi_bchunk_t* chunk, size_t field_idx, size_t idx, size_t n) {\n mi_assert_internal((field_idx*MI_BFIELD_BITS) + idx + n <= MI_BCHUNK_BITS);\n while (n > 0) {\n size_t m = MI_BFIELD_BITS - idx; // m is the bits to xset in this field\n if (m > n) { m = n; }\n mi_assert_internal(idx + m <= MI_BFIELD_BITS);\n mi_assert_internal(field_idx < MI_BCHUNK_FIELDS);\n const size_t mask = mi_bfield_mask(m, idx);\n if (!mi_bfield_atomic_is_xset_mask(set, &chunk->bfields[field_idx], mask)) {\n return false;\n }\n // next field\n field_idx++;\n idx = 0;\n n -= m;\n }\n return true;\n}\n\n// Check if a sequence of `n` bits within a chunk are all set/cleared.\nstatic inline bool mi_bchunk_is_xsetN(mi_xset_t set, const mi_bchunk_t* chunk, size_t cidx, size_t n) {\n mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);\n mi_assert_internal(n>0);\n if (n==0) return true;\n const size_t i = cidx / MI_BFIELD_BITS;\n const size_t idx = cidx % MI_BFIELD_BITS;\n if (n==1) { return mi_bfield_atomic_is_xset(set, &chunk->bfields[i], idx); }\n if (idx + n <= MI_BFIELD_BITS) { return mi_bfield_atomic_is_xset_mask(set, &chunk->bfields[i], mi_bfield_mask(n, idx)); }\n return mi_bchunk_is_xsetNC(set, chunk, i, idx, n);\n}\n\n\n// ------- mi_bchunk_try_clear ---------------------------------------\n\n// Clear `0 < n <= MI_BITFIELD_BITS`. Can cross over a bfield boundary.\nstatic inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) {\n mi_assert_internal(cidx < MI_BCHUNK_BITS);\n mi_assert_internal(n <= MI_BFIELD_BITS);\n const size_t i = cidx / MI_BFIELD_BITS;\n const size_t idx = cidx % MI_BFIELD_BITS;\n if mi_likely(idx + n <= MI_BFIELD_BITS) {\n // within one field\n return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear);\n }\n else {\n // spanning two fields (todo: use double-word atomic ops?)\n const size_t m = MI_BFIELD_BITS - idx; // bits to clear in the first field\n mi_assert_internal(m < n);\n mi_assert_internal(i < MI_BCHUNK_FIELDS - 1);\n bool field1_is_clear;\n if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear)) return false;\n // try the second field as well\n mi_assert_internal(n - m > 0);\n mi_assert_internal(n - m < MI_BFIELD_BITS);\n bool field2_is_clear;\n if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear)) {\n // we failed to clear the second field, restore the first one\n mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), NULL);\n return false;\n }\n if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = field1_is_clear && field2_is_clear; }\n return true;\n }\n}\n\n// Clear a full aligned bfield.\n// static inline bool mi_bchunk_try_clearX(mi_bchunk_t* chunk, size_t cidx, bool* pmaybe_all_clear) {\n// mi_assert_internal(cidx < MI_BCHUNK_BITS);\n// mi_assert_internal((cidx%MI_BFIELD_BITS) == 0);\n// const size_t i = cidx / MI_BFIELD_BITS;\n// return mi_bfield_atomic_try_clearX(&chunk->bfields[i], pmaybe_all_clear);\n// }\n\n// Try to atomically clear a sequence of `n` bits within a chunk.\n// Returns true if all bits transitioned from 1 to 0,\n// and false otherwise leaving all bit fields as is.\n// Note: this is the complex one as we need to unwind partial atomic operations if we fail halfway..\n// `maybe_all_clear` is set to `true` if all the bfields involved become zero.\nmi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) {\n mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);\n mi_assert_internal(n>0);\n if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = true; }\n if (n==0) return true;\n\n // first field\n const size_t start_idx = cidx % MI_BFIELD_BITS;\n const size_t start_field = cidx / MI_BFIELD_BITS;\n size_t field = start_field;\n size_t m = MI_BFIELD_BITS - start_idx; // m are the bits to clear in this field\n if (m > n) { m = n; }\n mi_assert_internal(start_idx + m <= MI_BFIELD_BITS);\n mi_assert_internal(start_field < MI_BCHUNK_FIELDS);\n const mi_bfield_t mask_start = mi_bfield_mask(m, start_idx);\n bool maybe_all_clear;\n if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear)) return false;\n\n // done?\n mi_assert_internal(m <= n);\n n -= m;\n\n // continue with mid fields and last field: if these fail we need to recover by unsetting previous fields\n // mid fields?\n while (n >= MI_BFIELD_BITS) {\n field++;\n mi_assert_internal(field < MI_BCHUNK_FIELDS);\n bool field_is_clear;\n if (!mi_bfield_atomic_try_clearX(&chunk->bfields[field], &field_is_clear)) goto restore;\n maybe_all_clear = maybe_all_clear && field_is_clear;\n n -= MI_BFIELD_BITS;\n }\n\n // last field?\n if (n > 0) {\n mi_assert_internal(n < MI_BFIELD_BITS);\n field++;\n mi_assert_internal(field < MI_BCHUNK_FIELDS);\n const mi_bfield_t mask_end = mi_bfield_mask(n, 0);\n bool field_is_clear;\n if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear)) goto restore;\n maybe_all_clear = maybe_all_clear && field_is_clear;\n }\n\n if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = maybe_all_clear; }\n return true;\n\nrestore:\n // `field` is the index of the field that failed to set atomically; we need to restore all previous fields\n mi_assert_internal(field > start_field);\n while( field > start_field) {\n field--;\n if (field == start_field) {\n mi_bfield_atomic_set_mask(&chunk->bfields[field], mask_start, NULL);\n }\n else {\n mi_bfield_atomic_setX(&chunk->bfields[field], NULL); // mid-field: set all bits again\n }\n }\n return false;\n}\n\n\nstatic inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) {\n mi_assert_internal(n>0);\n // if (n==MI_BFIELD_BITS) return mi_bchunk_try_clearX(chunk, cidx, maybe_all_clear);\n if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear);\n return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear);\n}\n\n\n// ------- mi_bchunk_try_find_and_clear ---------------------------------------\n\n#if MI_OPT_SIMD && defined(__AVX2__)\nmi_decl_maybe_unused static inline __m256i mi_mm256_zero(void) {\n return _mm256_setzero_si256();\n}\nmi_decl_maybe_unused static inline __m256i mi_mm256_ones(void) {\n return _mm256_set1_epi64x(~0);\n}\nmi_decl_maybe_unused static inline bool mi_mm256_is_ones(__m256i vec) {\n return _mm256_testc_si256(vec, _mm256_cmpeq_epi32(vec, vec));\n}\nmi_decl_maybe_unused static inline bool mi_mm256_is_zero( __m256i vec) {\n return _mm256_testz_si256(vec,vec);\n}\n#endif\n\nstatic inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) {\n mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);\n // note: this must be acquire (and not relaxed), or otherwise the AVX code below can loop forever\n // as the compiler won't reload the registers vec1 and vec2 from memory again.\n const mi_bfield_t b = mi_atomic_load_acquire(&chunk->bfields[chunk_idx]);\n size_t idx;\n if (mi_bfield_find_least_bit(b, &idx)) { // find the least bit\n if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], mi_bfield_mask(1,idx), b, NULL)) { // clear it atomically\n *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;\n mi_assert_internal(*pidx < MI_BCHUNK_BITS);\n return true;\n }\n }\n return false;\n}\n\n// Find least 1-bit in a chunk and try to clear it atomically\n// set `*pidx` to the bit index (0 <= *pidx < MI_BCHUNK_BITS) on success.\n// This is used to find free slices and abandoned pages and should be efficient.\n// todo: try neon version\nstatic inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx) {\n #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)\n for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries\n const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);\n const __m256i vcmp = _mm256_cmpeq_epi64(vec, mi_mm256_zero()); // (elem64 == 0 ? 0xFF : 0)\n const uint32_t mask = ~_mm256_movemask_epi8(vcmp); // mask of most significant bit of each byte (so each 8 bits are all set or clear)\n // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared)\n if (mask==0) return false;\n mi_assert_internal((_tzcnt_u32(mask)%8) == 0); // tzcnt == 0, 8, 16, or 24\n const size_t chunk_idx = _tzcnt_u32(mask) / 8;\n if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true;\n // try again\n // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded\n // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206)\n #if defined(__GNUC__)\n __asm __volatile (\"\" : : \"g\"(chunk) : \"memory\");\n #endif\n }\n #elif MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)\n for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries\n size_t chunk_idx = 0;\n #if 0\n // one vector at a time\n __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);\n if (mi_mm256_is_zero(vec)) {\n chunk_idx += 4;\n vec = _mm256_load_si256(((const __m256i*)chunk->bfields) + 1);\n }\n const __m256i vcmp = _mm256_cmpeq_epi64(vec, mi_mm256_zero()); // (elem64 == 0 ? 0xFF : 0)\n const uint32_t mask = ~_mm256_movemask_epi8(vcmp); // mask of most significant bit of each byte (so each 8 bits are all set or clear)\n // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared)\n if (mask==0) return false;\n mi_assert_internal((_tzcnt_u32(mask)%8) == 0); // tzcnt == 0, 8, 16, or 24\n chunk_idx += _tzcnt_u32(mask) / 8;\n #else\n // a cache line is 64b so we can just as well load all at the same time\n const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);\n const __m256i vec2 = _mm256_load_si256(((const __m256i*)chunk->bfields)+1);\n const __m256i cmpv = mi_mm256_zero();\n const __m256i vcmp1 = _mm256_cmpeq_epi64(vec1, cmpv); // (elem64 == 0 ? 0xFF : 0)\n const __m256i vcmp2 = _mm256_cmpeq_epi64(vec2, cmpv); // (elem64 == 0 ? 0xFF : 0)\n const uint32_t mask1 = ~_mm256_movemask_epi8(vcmp1); // mask of most significant bit of each byte (so each 8 bits are all set or clear)\n const uint32_t mask2 = ~_mm256_movemask_epi8(vcmp2); // mask of most significant bit of each byte (so each 8 bits are all set or clear)\n const uint64_t mask = ((uint64_t)mask2 << 32) | mask1;\n // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared)\n if (mask==0) return false;\n mi_assert_internal((_tzcnt_u64(mask)%8) == 0); // tzcnt == 0, 8, 16, 24 , ..\n chunk_idx = mi_ctz(mask) / 8;\n #endif\n if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true;\n // try again\n // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded\n // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206)\n #if defined(__GNUC__)\n __asm __volatile (\"\" : : \"g\"(chunk) : \"memory\");\n #endif\n }\n #elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64\n for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries\n // a cache line is 64b so we can just as well load all at the same time (?)\n const uint64x2_t vzero1_lo = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields)); // 2x64 bit is_zero\n const uint64x2_t vzero1_hi = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 2)); // 2x64 bit is_zero\n const uint64x2_t vzero2_lo = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 4)); // 2x64 bit is_zero\n const uint64x2_t vzero2_hi = vceqzq_u64(vld1q_u64((uint64_t*)chunk->bfields + 6)); // 2x64 bit is_zero\n const uint32x4_t vzero1 = vuzp1q_u32(vreinterpretq_u32_u64(vzero1_lo),vreinterpretq_u32_u64(vzero1_hi)); // unzip even elements: narrow to 4x32 bit is_zero ()\n const uint32x4_t vzero2 = vuzp1q_u32(vreinterpretq_u32_u64(vzero2_lo),vreinterpretq_u32_u64(vzero2_hi)); // unzip even elements: narrow to 4x32 bit is_zero ()\n const uint32x4_t vzero1x = vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(vzero1), 24)); // shift-right 2x32bit elem by 24: lo 16 bits contain the 2 lo bytes\n const uint32x4_t vzero2x = vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(vzero2), 24));\n const uint16x8_t vzero12 = vreinterpretq_u16_u32(vuzp1q_u32(vzero1x,vzero2x)); // unzip even 32-bit elements into one vector\n const uint8x8_t vzero = vmovn_u16(vzero12); // narrow the bottom 16-bits\n const uint64_t mask = ~vget_lane_u64(vreinterpret_u64_u8(vzero), 0); // 1 byte for each bfield (0xFF => bfield has a bit set)\n if (mask==0) return false;\n mi_assert_internal((mi_ctz(mask)%8) == 0); // tzcnt == 0, 8, 16, 24 , ..\n const size_t chunk_idx = mi_ctz(mask) / 8;\n if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true;\n // try again\n // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded\n // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206)\n #if defined(__GNUC__)\n __asm __volatile (\"\" : : \"g\"(chunk) : \"memory\");\n #endif\n }\n #else\n for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {\n if (mi_bchunk_try_find_and_clear_at(chunk, i, pidx)) return true;\n }\n #endif\n return false; \n}\n\nstatic inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx) {\n mi_assert_internal(n==1); MI_UNUSED(n);\n return mi_bchunk_try_find_and_clear(chunk, pidx);\n}\n\nmi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) {\n const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[chunk_idx]);\n // has_set8 has low bit in each byte set if the byte in x == 0xFF\n const mi_bfield_t has_set8 =\n ((~b - MI_BFIELD_LO_BIT8) & // high bit set if byte in x is 0xFF or < 0x7F\n (b & MI_BFIELD_HI_BIT8)) // high bit set if byte in x is >= 0x80\n >> 7; // shift high bit to low bit\n size_t idx;\n if (mi_bfield_find_least_bit(has_set8, &idx)) { // find least 1-bit\n mi_assert_internal(idx <= (MI_BFIELD_BITS - 8));\n mi_assert_internal((idx%8)==0);\n if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], (mi_bfield_t)0xFF << idx, b, NULL)) { // unset the byte atomically\n *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;\n mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);\n return true;\n }\n }\n return false;\n}\n\n// find least aligned byte in a chunk with all bits set, and try unset it atomically\n// set `*pidx` to its bit index (0 <= *pidx < MI_BCHUNK_BITS) on success.\n// Used to find medium size pages in the free blocks.\n// todo: try neon version\nstatic mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx) {\n #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)\n while (true) {\n // since a cache-line is 64b, load all at once\n const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);\n const __m256i vec2 = _mm256_load_si256((const __m256i*)chunk->bfields+1);\n const __m256i cmpv = mi_mm256_ones();\n const __m256i vcmp1 = _mm256_cmpeq_epi8(vec1, cmpv); // (byte == ~0 ? 0xFF : 0)\n const __m256i vcmp2 = _mm256_cmpeq_epi8(vec2, cmpv); // (byte == ~0 ? 0xFF : 0)\n const uint32_t mask1 = _mm256_movemask_epi8(vcmp1); // mask of most significant bit of each byte\n const uint32_t mask2 = _mm256_movemask_epi8(vcmp2); // mask of most significant bit of each byte\n const uint64_t mask = ((uint64_t)mask2 << 32) | mask1;\n // mask is inverted, so each bit is 0xFF iff the corresponding byte has a bit set (and thus can be cleared)\n if (mask==0) return false;\n const size_t bidx = _tzcnt_u64(mask); // byte-idx of the byte in the chunk\n const size_t chunk_idx = bidx / 8;\n const size_t idx = (bidx % 8)*8;\n mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);\n if mi_likely(mi_bfield_atomic_try_clear8(&chunk->bfields[chunk_idx], idx, NULL)) { // clear it atomically\n *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;\n mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);\n return true;\n }\n // try again\n // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded }\n }\n #else\n for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {\n if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx)) return true;\n }\n return false;\n #endif\n}\n\nstatic inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx) {\n mi_assert_internal(n==8); MI_UNUSED(n);\n return mi_bchunk_try_find_and_clear8(chunk, pidx);\n}\n\n\n// find a sequence of `n` bits in a chunk with `0 < n <= MI_BFIELD_BITS` with all bits set,\n// and try to clear them atomically.\n// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success.\n// will cross bfield boundaries.\nmi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx) {\n if (n == 0 || n > MI_BFIELD_BITS) return false;\n const mi_bfield_t mask = mi_bfield_mask(n, 0);\n // for all fields in the chunk\n for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {\n mi_bfield_t b0 = mi_atomic_load_relaxed(&chunk->bfields[i]);\n mi_bfield_t b = b0;\n size_t idx;\n\n // is there a range inside the field?\n while (mi_bfield_find_least_bit(b, &idx)) { // find least 1-bit\n if (idx + n > MI_BFIELD_BITS) break; // too short: maybe cross over, or continue with the next field\n\n const size_t bmask = mask<>idx == mask);\n if ((b&bmask) == bmask) { // found a match with all bits set, try clearing atomically\n if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[i], bmask, b0, NULL)) {\n *pidx = (i*MI_BFIELD_BITS) + idx;\n mi_assert_internal(*pidx < MI_BCHUNK_BITS);\n mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS);\n return true;\n }\n else {\n // if we failed to atomically commit, reload b and try again from the start\n b = b0 = mi_atomic_load_acquire(&chunk->bfields[i]);\n }\n }\n else {\n // advance by clearing the least run of ones, for example, with n>=4, idx=2:\n // b = 1111 1101 1010 1100\n // .. + (1< 0) {\n const size_t pre = mi_bfield_ctz(~mi_atomic_load_relaxed(&chunk->bfields[i+1]));\n if (post + pre >= n) {\n // it fits -- try to claim it atomically\n const size_t cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - post);\n if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL)) {\n // we cleared all atomically\n *pidx = cidx;\n mi_assert_internal(*pidx < MI_BCHUNK_BITS);\n mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS);\n return true;\n }\n }\n }\n }\n }\n return false;\n}\n\n// find a sequence of `n` bits in a chunk with `n <= MI_BCHUNK_BITS` with all bits set,\n// and try to clear them atomically.\n// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success.\n// This can cross bfield boundaries.\nstatic mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx) {\n if (n == 0 || n > MI_BCHUNK_BITS) return false; // cannot be more than a chunk\n\n // we first scan ahead to see if there is a range of `n` set bits, and only then try to clear atomically\n mi_assert_internal(n>0);\n const size_t skip_count = (n-1)/MI_BFIELD_BITS;\n size_t cidx;\n for (size_t i = 0; i < MI_BCHUNK_FIELDS - skip_count; i++)\n {\n size_t m = n; // bits to go\n\n // first field\n mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);\n size_t ones = mi_bfield_clz(~b);\n\n cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - ones); // start index\n if (ones >= m) {\n // we found enough bits already!\n m = 0;\n }\n else if (ones > 0) {\n // keep scanning further fields until we have enough bits\n m -= ones;\n size_t j = 1; // field count from i\n while (i+j < MI_BCHUNK_FIELDS) {\n mi_assert_internal(m > 0);\n b = mi_atomic_load_relaxed(&chunk->bfields[i+j]);\n ones = mi_bfield_ctz(~b);\n if (ones >= m) {\n // we found enough bits\n m = 0;\n break;\n }\n else if (ones == MI_BFIELD_BITS) {\n // not enough yet, proceed to the next field\n j++;\n m -= MI_BFIELD_BITS;\n }\n else {\n // the range was not enough, start from scratch\n i = i + j - 1; // no need to re-scan previous fields, except the last one (with clz this time)\n mi_assert_internal(m>0);\n break;\n }\n }\n }\n\n // did we find a range?\n if (m==0) {\n if (mi_bchunk_try_clearN(chunk, cidx, n, NULL)) {\n // we cleared all atomically\n *pidx = cidx;\n mi_assert_internal(*pidx < MI_BCHUNK_BITS);\n mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS);\n return true;\n }\n // note: if we fail for a small `n` on the first field, we don't rescan that field (as `i` is incremented)\n }\n // otherwise continue searching\n }\n return false;\n}\n\n\n\n// ------- mi_bchunk_clear_once_set ---------------------------------------\n\nstatic inline void mi_bchunk_clear_once_set(mi_bchunk_t* chunk, size_t cidx) {\n mi_assert_internal(cidx < MI_BCHUNK_BITS);\n const size_t i = cidx / MI_BFIELD_BITS;\n const size_t idx = cidx % MI_BFIELD_BITS;\n mi_bfield_atomic_clear_once_set(&chunk->bfields[i], idx);\n}\n\n\n// ------- mi_bitmap_all_are_clear ---------------------------------------\n\n\n// are all bits in a bitmap chunk clear?\nstatic inline bool mi_bchunk_all_are_clear_relaxed(mi_bchunk_t* chunk) {\n #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)\n const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);\n return mi_mm256_is_zero(vec);\n #elif MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)\n // a 64b cache-line contains the entire chunk anyway so load both at once\n const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);\n const __m256i vec2 = _mm256_load_si256(((const __m256i*)chunk->bfields)+1);\n return (mi_mm256_is_zero(_mm256_or_si256(vec1,vec2)));\n #elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64\n const uint64x2_t v0 = vld1q_u64((uint64_t*)chunk->bfields);\n const uint64x2_t v1 = vld1q_u64((uint64_t*)chunk->bfields + 2);\n const uint64x2_t v2 = vld1q_u64((uint64_t*)chunk->bfields + 4);\n const uint64x2_t v3 = vld1q_u64((uint64_t*)chunk->bfields + 6);\n const uint64x2_t v = vorrq_u64(vorrq_u64(v0,v1),vorrq_u64(v2,v3));\n return (vmaxvq_u32(vreinterpretq_u32_u64(v)) == 0);\n #else\n for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {\n if (mi_atomic_load_relaxed(&chunk->bfields[i]) != 0) return false;\n }\n return true;\n #endif\n}\n\n// are all bits in a bitmap chunk set?\nstatic inline bool mi_bchunk_all_are_set_relaxed(mi_bchunk_t* chunk) {\n#if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)\n const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);\n return mi_mm256_is_ones(vec);\n#elif MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)\n // a 64b cache-line contains the entire chunk anyway so load both at once\n const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);\n const __m256i vec2 = _mm256_load_si256(((const __m256i*)chunk->bfields)+1);\n return (mi_mm256_is_ones(_mm256_and_si256(vec1, vec2)));\n#elif MI_OPT_SIMD && (MI_BCHUNK_BITS==512) && MI_ARCH_ARM64\n const uint64x2_t v0 = vld1q_u64((uint64_t*)chunk->bfields);\n const uint64x2_t v1 = vld1q_u64((uint64_t*)chunk->bfields + 2);\n const uint64x2_t v2 = vld1q_u64((uint64_t*)chunk->bfields + 4);\n const uint64x2_t v3 = vld1q_u64((uint64_t*)chunk->bfields + 6);\n const uint64x2_t v = vandq_u64(vandq_u64(v0,v1),vandq_u64(v2,v3));\n return (vminvq_u32(vreinterpretq_u32_u64(v)) == 0xFFFFFFFFUL);\n#else\n for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {\n if (~mi_atomic_load_relaxed(&chunk->bfields[i]) != 0) return false;\n }\n return true;\n#endif\n}\n\n\nstatic bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) {\n for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) {\n i--;\n mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);\n size_t idx;\n if (mi_bsr(b, &idx)) {\n *pidx = (i*MI_BFIELD_BITS) + idx;\n return true;\n }\n }\n return false;\n}\n\nstatic bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) {\n for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) {\n i--;\n mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);\n size_t idx;\n if (mi_bsr(~b, &idx)) {\n *pidx = (i*MI_BFIELD_BITS) + idx;\n return true;\n }\n }\n return false;\n}\n\nstatic size_t mi_bchunk_popcount(mi_bchunk_t* chunk) {\n size_t popcount = 0;\n for (size_t i = 0; i < MI_BCHUNK_FIELDS; i++) {\n const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);\n popcount += mi_bfield_popcount(b);\n }\n return popcount;\n}\n\n\n/* --------------------------------------------------------------------------------\n bitmap chunkmap\n-------------------------------------------------------------------------------- */\n\nstatic void mi_bitmap_chunkmap_set(mi_bitmap_t* bitmap, size_t chunk_idx) {\n mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap));\n mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL);\n}\n\nstatic bool mi_bitmap_chunkmap_try_clear(mi_bitmap_t* bitmap, size_t chunk_idx) {\n mi_assert(chunk_idx < mi_bitmap_chunk_count(bitmap));\n // check if the corresponding chunk is all clear\n if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) return false;\n // clear the chunkmap bit\n mi_bchunk_clear(&bitmap->chunkmap, chunk_idx, NULL);\n // .. but a concurrent set may have happened in between our all-clear test and the clearing of the\n // bit in the mask. We check again to catch this situation.\n if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) {\n mi_bchunk_set(&bitmap->chunkmap, chunk_idx, NULL);\n return false;\n }\n return true;\n}\n\n\n/* --------------------------------------------------------------------------------\n bitmap\n-------------------------------------------------------------------------------- */\n\nsize_t mi_bitmap_size(size_t bit_count, size_t* pchunk_count) {\n mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0);\n bit_count = _mi_align_up(bit_count, MI_BCHUNK_BITS);\n mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT);\n mi_assert_internal(bit_count > 0);\n const size_t chunk_count = bit_count / MI_BCHUNK_BITS;\n mi_assert_internal(chunk_count >= 1);\n const size_t size = offsetof(mi_bitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE);\n mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 );\n if (pchunk_count != NULL) { *pchunk_count = chunk_count; }\n return size;\n}\n\n\n// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true\n// returns the size of the bitmap\nsize_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero) {\n size_t chunk_count;\n const size_t size = mi_bitmap_size(bit_count, &chunk_count);\n if (!already_zero) {\n _mi_memzero_aligned(bitmap, size);\n }\n mi_atomic_store_release(&bitmap->chunk_count, chunk_count);\n mi_assert_internal(mi_atomic_load_relaxed(&bitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT);\n return size;\n}\n\n\n// Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread.\nstatic void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, size_t idx, size_t n) {\n mi_assert_internal(n>0);\n\n // start chunk and index\n size_t chunk_idx = idx / MI_BCHUNK_BITS;\n const size_t cidx = idx % MI_BCHUNK_BITS;\n const size_t ccount = _mi_divide_up(n, MI_BCHUNK_BITS);\n\n // first update the chunkmap\n mi_bchunk_setN(cmap, chunk_idx, ccount, NULL);\n\n // first chunk\n size_t m = MI_BCHUNK_BITS - cidx;\n if (m > n) { m = n; }\n mi_bchunk_setN(&chunks[chunk_idx], cidx, m, NULL);\n\n // n can be large so use memset for efficiency for all in-between chunks\n chunk_idx++;\n n -= m;\n const size_t mid_chunks = n / MI_BCHUNK_BITS;\n if (mid_chunks > 0) {\n _mi_memset(&chunks[chunk_idx], ~0, mid_chunks * MI_BCHUNK_SIZE);\n chunk_idx += mid_chunks;\n n -= (mid_chunks * MI_BCHUNK_BITS);\n }\n\n // last chunk\n if (n > 0) {\n mi_assert_internal(n < MI_BCHUNK_BITS);\n mi_bchunk_setN(&chunks[chunk_idx], 0, n, NULL);\n }\n}\n\n// Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread.\nvoid mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) {\n mi_assert_internal(n>0);\n mi_assert_internal(idx + n <= mi_bitmap_max_bits(bitmap));\n mi_bchunks_unsafe_setN(&bitmap->chunks[0], &bitmap->chunkmap, idx, n);\n}\n\n\n\n\n// ------- mi_bitmap_xset ---------------------------------------\n\n// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's).\nbool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* palready_set) {\n mi_assert_internal(n>0);\n const size_t maxbits = mi_bitmap_max_bits(bitmap);\n mi_assert_internal(idx + n <= maxbits);\n if (idx+n > maxbits) { // paranoia\n if (idx >= maxbits) return false;\n n = maxbits - idx;\n }\n\n // iterate through the chunks\n size_t chunk_idx = idx / MI_BCHUNK_BITS;\n size_t cidx = idx % MI_BCHUNK_BITS;\n bool were_allclear = true;\n size_t already_set = 0;\n while (n > 0) {\n const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);\n size_t _already_set = 0;\n were_allclear = mi_bchunk_setN(&bitmap->chunks[chunk_idx], cidx, m, &_already_set) && were_allclear;\n already_set += _already_set;\n mi_bitmap_chunkmap_set(bitmap, chunk_idx); // set afterwards\n mi_assert_internal(m <= n);\n n -= m;\n cidx = 0;\n chunk_idx++;\n }\n if (palready_set != NULL) { *palready_set = already_set; }\n return were_allclear;\n}\n\n// Clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 1's to 0's.\nbool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) {\n mi_assert_internal(n>0);\n const size_t maxbits = mi_bitmap_max_bits(bitmap);\n mi_assert_internal(idx + n <= maxbits);\n if (idx+n > maxbits) { // paranoia\n if (idx >= maxbits) return false;\n n = maxbits - idx;\n }\n\n // iterate through the chunks\n size_t chunk_idx = idx / MI_BCHUNK_BITS;\n size_t cidx = idx % MI_BCHUNK_BITS;\n bool were_allset = true;\n while (n > 0) {\n const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);\n bool maybe_all_clear = false;\n were_allset = mi_bchunk_clearN(&bitmap->chunks[chunk_idx], cidx, m, &maybe_all_clear) && were_allset;\n if (maybe_all_clear) { mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); }\n mi_assert_internal(m <= n);\n n -= m;\n cidx = 0;\n chunk_idx++;\n }\n return were_allset;\n}\n\n// Count bits set in a range of `n` bits.\nsize_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n) {\n mi_assert_internal(n>0);\n const size_t maxbits = mi_bitmap_max_bits(bitmap);\n mi_assert_internal(idx + n <= maxbits);\n if (idx+n > maxbits) { // paranoia\n if (idx >= maxbits) return 0;\n n = maxbits - idx;\n }\n\n // iterate through the chunks\n size_t chunk_idx = idx / MI_BCHUNK_BITS;\n size_t cidx = idx % MI_BCHUNK_BITS;\n size_t popcount = 0;\n while (n > 0) {\n const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);\n popcount += mi_bchunk_popcountN(&bitmap->chunks[chunk_idx], cidx, m);\n mi_assert_internal(m <= n);\n n -= m;\n cidx = 0;\n chunk_idx++;\n }\n return popcount;\n}\n\n\n// Set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0)\nbool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx) {\n return mi_bitmap_setN(bitmap, idx, 1, NULL);\n}\n\nbool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx) {\n return mi_bitmap_clearN(bitmap, idx, 1);\n}\n\n\n\n// ------- mi_bitmap_is_xset ---------------------------------------\n\n// Is a sequence of n bits already all set/cleared?\nbool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n) {\n mi_assert_internal(n>0);\n const size_t maxbits = mi_bitmap_max_bits(bitmap);\n mi_assert_internal(idx + n <= maxbits);\n if (idx+n > maxbits) { // paranoia\n if (idx >= maxbits) return false;\n n = maxbits - idx;\n }\n\n // iterate through the chunks\n size_t chunk_idx = idx / MI_BCHUNK_BITS;\n size_t cidx = idx % MI_BCHUNK_BITS;\n bool xset = true;\n while (n > 0 && xset) {\n const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);\n xset = mi_bchunk_is_xsetN(set, &bitmap->chunks[chunk_idx], cidx, m) && xset;\n mi_assert_internal(m <= n);\n n -= m;\n cidx = 0;\n chunk_idx++;\n }\n return xset;\n}\n\nbool mi_bitmap_is_all_clear(mi_bitmap_t* bitmap) {\n return mi_bitmap_is_xsetN(MI_BIT_CLEAR, bitmap, 0, mi_bitmap_max_bits(bitmap));\n}\n\n/* --------------------------------------------------------------------------------\n Iterate through a bfield\n-------------------------------------------------------------------------------- */\n\n// Cycle iteration through a bitfield. This is used to space out threads\n// so there is less chance of contention. When searching for a free page we\n// like to first search only the accessed part (so we reuse better). This\n// high point is called the `cycle`.\n//\n// We then iterate through the bitfield as:\n// first: [start, cycle>\n// then : [0, start>\n// then : [cycle, MI_BFIELD_BITS>\n//\n// The start is determined usually as `tseq % cycle` to have each thread\n// start at a different spot.\n// - We use `popcount` to improve branch prediction (maybe not needed? can we simplify?)\n// - The `cycle_mask` is the part `[start, cycle>`.\n#define mi_bfield_iterate(bfield,start,cycle,name_idx,SUF) { \\\n mi_assert_internal(start <= cycle); \\\n mi_assert_internal(start < MI_BFIELD_BITS); \\\n mi_assert_internal(cycle <= MI_BFIELD_BITS); \\\n const mi_bfield_t _cycle_mask##SUF = mi_bfield_mask(cycle - start, start); \\\n size_t _bcount##SUF = mi_bfield_popcount(bfield); \\\n mi_bfield_t _b##SUF = bfield & _cycle_mask##SUF; /* process [start, cycle> first*/\\\n while(_bcount##SUF > 0) { \\\n _bcount##SUF--;\\\n if (_b##SUF==0) { _b##SUF = bfield & ~_cycle_mask##SUF; } /* process [0,start> + [cycle, MI_BFIELD_BITS> next */ \\\n /* size_t name_idx; */ \\\n const bool _found##SUF = mi_bfield_find_least_bit(_b##SUF,&name_idx); \\\n _b##SUF = mi_bfield_clear_least_bit(_b##SUF); /* clear early so `continue` works */ \\\n mi_assert_internal(_found##SUF); MI_UNUSED(_found##SUF); \\\n { \\\n\n#define mi_bfield_iterate_end(SUF) \\\n } \\\n } \\\n}\n\n\n#define mi_bfield_cycle_iterate(bfield,tseq,cycle,name_idx,SUF) { \\\n const size_t _start##SUF = (uint32_t)(tseq) % (uint32_t)(cycle); /* or: 0 to always search from the start? */\\\n mi_bfield_iterate(bfield,_start##SUF,cycle,name_idx,SUF)\n\n#define mi_bfield_cycle_iterate_end(SUF) \\\n mi_bfield_iterate_end(SUF); \\\n}\n\n\n/* --------------------------------------------------------------------------------\n mi_bitmap_find\n (used to find free pages)\n-------------------------------------------------------------------------------- */\n\ntypedef bool (mi_bitmap_visit_fun_t)(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* idx, void* arg1, void* arg2);\n\n// Go through the bitmap and for every sequence of `n` set bits, call the visitor function.\n// If it returns `true` stop the search.\nstatic inline bool mi_bitmap_find(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx, mi_bitmap_visit_fun_t* on_find, void* arg1, void* arg2)\n{\n const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);\n for (size_t i = 0; i < chunkmap_max; i++) {\n // and for each chunkmap entry we iterate over its bits to find the chunks\n const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);\n size_t hi;\n if (mi_bfield_find_highest_bit(cmap_entry, &hi)) {\n size_t eidx = 0;\n mi_bfield_cycle_iterate(cmap_entry, tseq%8, hi+1, eidx, Y) // reduce the tseq to 8 bins to reduce using extra memory (see `mstress`)\n {\n mi_assert_internal(eidx <= MI_BFIELD_BITS);\n const size_t chunk_idx = i*MI_BFIELD_BITS + eidx;\n mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap));\n if ((*on_find)(bitmap, chunk_idx, n, pidx, arg1, arg2)) {\n return true;\n }\n }\n mi_bfield_cycle_iterate_end(Y);\n }\n }\n return false;\n}\n\n\n/* --------------------------------------------------------------------------------\n Bitmap: try_find_and_claim -- used to allocate abandoned pages\n note: the compiler will fully inline the indirect function call\n-------------------------------------------------------------------------------- */\n\ntypedef struct mi_claim_fun_data_s {\n mi_arena_t* arena; \n} mi_claim_fun_data_t;\n\nstatic bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* pidx, void* arg1, void* arg2)\n{\n mi_assert_internal(n==1); MI_UNUSED(n);\n mi_claim_fun_t* claim_fun = (mi_claim_fun_t*)arg1;\n mi_claim_fun_data_t* claim_data = (mi_claim_fun_data_t*)arg2;\n size_t cidx;\n if mi_likely(mi_bchunk_try_find_and_clear(&bitmap->chunks[chunk_idx], &cidx)) {\n const size_t slice_index = (chunk_idx * MI_BCHUNK_BITS) + cidx;\n mi_assert_internal(slice_index < mi_bitmap_max_bits(bitmap));\n bool keep_set = true;\n if ((*claim_fun)(slice_index, claim_data->arena, &keep_set)) {\n // success!\n mi_assert_internal(!keep_set);\n *pidx = slice_index;\n return true;\n }\n else {\n // failed to claim it, set abandoned mapping again (unless the page was freed)\n if (keep_set) {\n const bool wasclear = mi_bchunk_set(&bitmap->chunks[chunk_idx], cidx, NULL);\n mi_assert_internal(wasclear); MI_UNUSED(wasclear);\n }\n }\n }\n else {\n // we may find that all are cleared only on a second iteration but that is ok as\n // the chunkmap is a conservative approximation.\n mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx);\n }\n return false;\n}\n\n// Find a set bit in the bitmap and try to atomically clear it and claim it.\n// (Used to find pages in the pages_abandoned bitmaps.)\nmi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx,\n mi_claim_fun_t* claim, mi_arena_t* arena )\n{\n mi_claim_fun_data_t claim_data = { arena };\n return mi_bitmap_find(bitmap, tseq, 1, pidx, &mi_bitmap_try_find_and_claim_visit, (void*)claim, &claim_data);\n}\n\n\nbool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx) {\n const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);\n for (size_t i = chunkmap_max; i > 0; ) {\n i--;\n mi_bfield_t cmap = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);\n size_t cmap_idx;\n if (mi_bsr(cmap,&cmap_idx)) {\n // highest chunk\n const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;\n size_t cidx;\n if (mi_bchunk_bsr(&bitmap->chunks[chunk_idx], &cidx)) {\n *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;\n return true;\n }\n }\n }\n return false;\n}\n\n// Return count of all set bits in a bitmap.\nsize_t mi_bitmap_popcount(mi_bitmap_t* bitmap) {\n // for all chunkmap entries\n size_t popcount = 0;\n const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);\n for (size_t i = 0; i < chunkmap_max; i++) {\n mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);\n size_t cmap_idx;\n // for each chunk (corresponding to a set bit in a chunkmap entry)\n while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {\n const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;\n // count bits in a chunk\n popcount += mi_bchunk_popcount(&bitmap->chunks[chunk_idx]);\n }\n }\n return popcount;\n}\n\n\n\n// Clear a bit once it is set.\nvoid mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx) {\n mi_assert_internal(idx < mi_bitmap_max_bits(bitmap));\n const size_t chunk_idx = idx / MI_BCHUNK_BITS;\n const size_t cidx = idx % MI_BCHUNK_BITS;\n mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap));\n mi_bchunk_clear_once_set(&bitmap->chunks[chunk_idx], cidx);\n}\n\n\n// Visit all set bits in a bitmap.\n// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore?\nbool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) {\n // for all chunkmap entries\n const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);\n for(size_t i = 0; i < chunkmap_max; i++) {\n mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);\n size_t cmap_idx;\n // for each chunk (corresponding to a set bit in a chunkmap entry)\n while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {\n const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;\n // for each chunk field\n mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx];\n for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {\n const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);\n mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[j]);\n size_t bidx;\n while (mi_bfield_foreach_bit(&b, &bidx)) {\n const size_t idx = base_idx + bidx;\n if (!visit(idx, 1, arena, arg)) return false;\n }\n }\n }\n }\n return true;\n}\n\n// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible.\n// Also clear those ranges atomically.\n// Used by purging to purge larger ranges when possible\n// todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore?\nbool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) {\n // for all chunkmap entries\n const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);\n for (size_t i = 0; i < chunkmap_max; i++) {\n mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);\n size_t cmap_idx;\n // for each chunk (corresponding to a set bit in a chunkmap entry)\n while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {\n const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;\n // for each chunk field\n mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx];\n for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {\n const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);\n mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0);\n #if MI_DEBUG > 1\n const size_t bpopcount = mi_popcount(b);\n size_t rngcount = 0;\n #endif\n size_t bidx;\n while (mi_bfield_find_least_bit(b, &bidx)) {\n size_t rng = mi_ctz(~(b>>bidx)); // all the set bits from bidx\n #if MI_DEBUG > 1\n rngcount += rng;\n #endif\n const size_t idx = base_idx + bidx;\n mi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS);\n mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS);\n mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap));\n if (!visit(idx, rng, arena, arg)) {\n // break early: reset the non-visited bits\n if (b!=0) {\n mi_atomic_or_relaxed(&chunk->bfields[j], b);\n }\n return false;\n }\n // clear rng bits in b\n b = b & ~mi_bfield_mask(rng, bidx);\n }\n mi_assert_internal(rngcount == bpopcount);\n }\n }\n }\n return true;\n}\n\n// Visit all set bits in a bitmap but try to return ranges (within bfields) if possible,\n// but only in chunks of at least `rngslices` slices (that are also aligned at `rngslices`)\n// and clear those ranges atomically.\n// However, the `rngslices` are capped at `MI_BFIELD_BITS` at most.\n// Used by purging to purge larger ranges when possible. With transparent huge pages we only\n// want to purge whole huge pages (2 MiB) at a time which is what the `rngslices` parameter achieves.\nbool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) \n{\n // use the generic routine for `rngslices<=1` (as that one finds longest ranges at a time)\n if (rngslices<=1) {\n return _mi_bitmap_forall_setc_ranges(bitmap, visit, arena, arg);\n }\n // mi_assert_internal(rngslices <= MI_BFIELD_BITS); \n if (rngslices > MI_BFIELD_BITS) { rngslices = MI_BFIELD_BITS; } // cap at MI_BFIELD_BITS at most\n\n // for all chunkmap entries\n const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS);\n for (size_t i = 0; i < chunkmap_max; i++) {\n mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);\n size_t cmap_idx;\n // for each chunk (corresponding to a set bit in a chunkmap entry)\n while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) {\n const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;\n // for each chunk field\n mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx];\n for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {\n const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);\n mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0); // atomic clear\n mi_bfield_t skipped = 0; // but track which bits we skip so we can restore them\n for(size_t shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignment\n const mi_bfield_t rngmask = mi_bfield_mask(rngslices, shift);\n if ((b & rngmask) == rngmask) {\n const size_t idx = base_idx + shift;\n if (!visit(idx, rngslices, arena, arg)) {\n // break early: restore non-visited entries\n mi_bfield_t notyet_visited = 0;\n if (shift + rngslices < MI_BFIELD_BITS) {\n notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices)));\n }\n mi_assert_internal((notyet_visited & skipped) == 0);\n if ((notyet_visited | skipped) != 0) {\n mi_atomic_or_relaxed(&chunk->bfields[j], notyet_visited | skipped);\n }\n return false;\n }\n }\n else {\n skipped = skipped | (b & rngmask);\n } \n } \n \n if (skipped != 0) {\n mi_atomic_or_relaxed(&chunk->bfields[j], skipped);\n }\n }\n }\n }\n return true;\n}\n\n\n/* --------------------------------------------------------------------------------\n binned bitmap's\n-------------------------------------------------------------------------------- */\n\n\nsize_t mi_bbitmap_size(size_t bit_count, size_t* pchunk_count) {\n // mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0);\n bit_count = _mi_align_up(bit_count, MI_BCHUNK_BITS);\n mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT);\n mi_assert_internal(bit_count > 0);\n const size_t chunk_count = bit_count / MI_BCHUNK_BITS;\n mi_assert_internal(chunk_count >= 1);\n const size_t size = offsetof(mi_bbitmap_t,chunks) + (chunk_count * MI_BCHUNK_SIZE);\n mi_assert_internal( (size%MI_BCHUNK_SIZE) == 0 );\n if (pchunk_count != NULL) { *pchunk_count = chunk_count; }\n return size;\n}\n\n// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true\n// returns the size of the bitmap\nsize_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) {\n size_t chunk_count;\n const size_t size = mi_bbitmap_size(bit_count, &chunk_count);\n if (!already_zero) {\n _mi_memzero_aligned(bbitmap, size);\n }\n mi_atomic_store_release(&bbitmap->chunk_count, chunk_count);\n mi_assert_internal(mi_atomic_load_relaxed(&bbitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT);\n return size;\n}\n\nvoid mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {\n mi_assert_internal(n>0);\n mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap));\n mi_bchunks_unsafe_setN(&bbitmap->chunks[0], &bbitmap->chunkmap, idx, n);\n}\n\nbool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) {\n const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap);\n const size_t chunkmap_max = _mi_divide_up(chunk_count, MI_BFIELD_BITS);\n size_t skip_at_top = chunk_count % MI_BFIELD_BITS;\n for (size_t i = chunkmap_max; i > 0; ) {\n i--;\n mi_bfield_t cmap = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[i]);\n size_t cmap_idx;\n // don't consider top 0 bits; set those to 1 here\n if (skip_at_top > 0) {\n const size_t mask_top = (~mi_bfield_zero()) << (MI_BFIELD_BITS - skip_at_top);\n skip_at_top = 0; // only for the first iteration\n cmap |= mask_top;\n }\n if (mi_bsr(~cmap, &cmap_idx)) {\n // highest chunk\n const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;\n size_t cidx;\n if (mi_bchunk_bsr_inv(&bbitmap->chunks[chunk_idx], &cidx)) {\n *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;\n return true;\n }\n }\n }\n return false;\n}\n\n\n/* --------------------------------------------------------------------------------\n binned bitmap used to track free slices\n-------------------------------------------------------------------------------- */\n\n// Assign a specific size bin to a chunk\nstatic void mi_bbitmap_set_chunk_bin(mi_bbitmap_t* bbitmap, size_t chunk_idx, mi_chunkbin_t bin) {\n mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap));\n for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) {\n if (ibin == bin) {\n const bool was_clear = mi_bchunk_set(& bbitmap->chunkmap_bins[ibin], chunk_idx, NULL);\n if (was_clear) { mi_os_stat_increase(chunk_bins[ibin],1); }\n }\n else {\n const bool was_set = mi_bchunk_clear(&bbitmap->chunkmap_bins[ibin], chunk_idx, NULL);\n if (was_set) { mi_os_stat_decrease(chunk_bins[ibin],1); }\n }\n }\n}\n\nmi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunkmap_t* chunkmap_bins, size_t chunk_idx) {\n for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) {\n if (mi_bchunk_is_xsetN(MI_BIT_SET, &chunkmap_bins[ibin], chunk_idx, 1)) {\n return ibin;\n }\n }\n return MI_CBIN_NONE;\n}\n\n// Track the index of the highest chunk that is accessed.\nstatic void mi_bbitmap_chunkmap_set_max(mi_bbitmap_t* bbitmap, size_t chunk_idx) {\n size_t oldmax = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed);\n if mi_unlikely(chunk_idx > oldmax) {\n mi_atomic_cas_strong_relaxed(&bbitmap->chunk_max_accessed, &oldmax, chunk_idx);\n }\n}\n\n// Set a bit in the chunkmap\nstatic void mi_bbitmap_chunkmap_set(mi_bbitmap_t* bbitmap, size_t chunk_idx, bool check_all_set) {\n mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap));\n if (check_all_set) {\n if (mi_bchunk_all_are_set_relaxed(&bbitmap->chunks[chunk_idx])) {\n // all slices are free in this chunk: return back to the NONE bin\n mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, MI_CBIN_NONE);\n }\n }\n mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL);\n mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx);\n}\n\nstatic bool mi_bbitmap_chunkmap_try_clear(mi_bbitmap_t* bbitmap, size_t chunk_idx) {\n mi_assert(chunk_idx < mi_bbitmap_chunk_count(bbitmap));\n // check if the corresponding chunk is all clear\n if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) return false;\n // clear the chunkmap bit\n mi_bchunk_clear(&bbitmap->chunkmap, chunk_idx, NULL);\n // .. but a concurrent set may have happened in between our all-clear test and the clearing of the\n // bit in the mask. We check again to catch this situation. (note: mi_bchunk_clear must be acq-rel)\n if (!mi_bchunk_all_are_clear_relaxed(&bbitmap->chunks[chunk_idx])) {\n mi_bchunk_set(&bbitmap->chunkmap, chunk_idx, NULL);\n return false;\n }\n mi_bbitmap_chunkmap_set_max(bbitmap, chunk_idx);\n return true;\n}\n\n\n/* --------------------------------------------------------------------------------\n mi_bbitmap_setN, try_clearN, and is_xsetN\n (used to find free pages)\n-------------------------------------------------------------------------------- */\n\n// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's).\nbool mi_bbitmap_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {\n mi_assert_internal(n>0);\n const size_t maxbits = mi_bbitmap_max_bits(bbitmap);\n mi_assert_internal(idx + n <= maxbits);\n if (idx+n > maxbits) { // paranoia\n if (idx >= maxbits) return false;\n n = maxbits - idx;\n }\n\n // iterate through the chunks\n size_t chunk_idx = idx / MI_BCHUNK_BITS;\n size_t cidx = idx % MI_BCHUNK_BITS;\n bool were_allclear = true;\n while (n > 0) {\n const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);\n were_allclear = mi_bchunk_setN(&bbitmap->chunks[chunk_idx], cidx, m, NULL) && were_allclear;\n mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, true); // set afterwards\n mi_assert_internal(m <= n);\n n -= m;\n cidx = 0;\n chunk_idx++;\n }\n return were_allclear;\n}\n\n// ------- mi_bbitmap_try_clearNC ---------------------------------------\n\n// Try to clear `n` bits at `idx` where `n <= MI_BCHUNK_BITS`.\nbool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {\n mi_assert_internal(n>0);\n mi_assert_internal(n<=MI_BCHUNK_BITS);\n mi_assert_internal(idx + n <= mi_bbitmap_max_bits(bbitmap));\n\n const size_t chunk_idx = idx / MI_BCHUNK_BITS;\n const size_t cidx = idx % MI_BCHUNK_BITS;\n mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); // don't cross chunks (for now)\n mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap));\n if (cidx + n > MI_BCHUNK_BITS) return false;\n bool maybe_all_clear = false;\n const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear);\n if (cleared && maybe_all_clear) { mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); }\n // note: we don't set the size class for an explicit try_clearN (only used by purging)\n return cleared;\n}\n\n\n\n// ------- mi_bbitmap_is_xset ---------------------------------------\n\n// Is a sequence of n bits already all set/cleared?\nbool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_t n) {\n mi_assert_internal(n>0);\n const size_t maxbits = mi_bbitmap_max_bits(bbitmap);\n mi_assert_internal(idx + n <= maxbits);\n if (idx+n > maxbits) { // paranoia\n if (idx >= maxbits) return false;\n n = maxbits - idx;\n }\n\n // iterate through the chunks\n size_t chunk_idx = idx / MI_BCHUNK_BITS;\n size_t cidx = idx % MI_BCHUNK_BITS;\n bool xset = true;\n while (n > 0 && xset) {\n const size_t m = (cidx + n > MI_BCHUNK_BITS ? MI_BCHUNK_BITS - cidx : n);\n xset = mi_bchunk_is_xsetN(set, &bbitmap->chunks[chunk_idx], cidx, m) && xset;\n mi_assert_internal(m <= n);\n n -= m;\n cidx = 0;\n chunk_idx++;\n }\n return xset;\n}\n\n\n\n\n/* --------------------------------------------------------------------------------\n mi_bbitmap_find\n (used to find free pages)\n-------------------------------------------------------------------------------- */\n\ntypedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx);\n\n// Go through the bbitmap and for every sequence of `n` set bits, call the visitor function.\n// If it returns `true` stop the search.\n//\n// This is used for finding free blocks and it is important to be efficient (with 2-level bitscan)\n// but also reduce fragmentation (through size bins).\nstatic inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find)\n{\n // we space out threads to reduce contention\n const size_t cmap_max_count = _mi_divide_up(mi_bbitmap_chunk_count(bbitmap),MI_BFIELD_BITS);\n const size_t chunk_acc = mi_atomic_load_relaxed(&bbitmap->chunk_max_accessed);\n const size_t cmap_acc = chunk_acc / MI_BFIELD_BITS;\n const size_t cmap_acc_bits = 1 + (chunk_acc % MI_BFIELD_BITS);\n\n // create a mask over the chunkmap entries to iterate over them efficiently\n mi_assert_internal(MI_BFIELD_BITS >= MI_BCHUNK_FIELDS);\n const mi_bfield_t cmap_mask = mi_bfield_mask(cmap_max_count,0);\n const size_t cmap_cycle = cmap_acc+1;\n const mi_chunkbin_t bbin = mi_chunkbin_of(n);\n // visit each cmap entry\n size_t cmap_idx = 0;\n mi_bfield_cycle_iterate(cmap_mask, tseq, cmap_cycle, cmap_idx, X)\n {\n // and for each chunkmap entry we iterate over its bits to find the chunks\n const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[cmap_idx]);\n const size_t cmap_entry_cycle = (cmap_idx != cmap_acc ? MI_BFIELD_BITS : cmap_acc_bits);\n if (cmap_entry == 0) {\n continue;\n }\n\n // get size bin masks\n mi_bfield_t cmap_bins[MI_CBIN_COUNT] = { 0 };\n cmap_bins[MI_CBIN_NONE] = cmap_entry;\n for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) {\n const mi_bfield_t cmap_bin = mi_atomic_load_relaxed(&bbitmap->chunkmap_bins[ibin].bfields[cmap_idx]);\n cmap_bins[ibin] = cmap_bin & cmap_entry;\n cmap_bins[MI_CBIN_NONE] &= ~cmap_bin; // clear bits that are in an assigned size bin\n }\n\n // consider only chunks for a particular size bin at a time\n // this picks the best bin only within a cmap entry (~ 1GiB address space), but avoids multiple\n // iterations through all entries.\n mi_assert_internal(bbin < MI_CBIN_NONE);\n for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin <= MI_CBIN_NONE;\n // skip from bbin to NONE (so, say, a SMALL will never be placed in a OTHER, MEDIUM, or LARGE chunk to reduce fragmentation)\n ibin = (ibin == bbin ? MI_CBIN_NONE : mi_chunkbin_inc(ibin)))\n {\n mi_assert_internal(ibin < MI_CBIN_COUNT);\n const mi_bfield_t cmap_bin = cmap_bins[ibin];\n size_t eidx = 0;\n mi_bfield_cycle_iterate(cmap_bin, tseq, cmap_entry_cycle, eidx, Y)\n {\n // assertion doesn't quite hold as the max_accessed may be out-of-date\n // mi_assert_internal(cmap_entry_cycle > eidx || ibin == MI_CBIN_NONE);\n\n // get the chunk\n const size_t chunk_idx = cmap_idx*MI_BFIELD_BITS + eidx;\n mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx];\n\n size_t cidx;\n if ((*on_find)(chunk, n, &cidx)) {\n if (cidx==0 && ibin == MI_CBIN_NONE) { // only the first block determines the size bin\n // this chunk is now reserved for the `bbin` size class\n mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, bbin);\n }\n *pidx = (chunk_idx * MI_BCHUNK_BITS) + cidx;\n mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap));\n return true;\n }\n else {\n // todo: should _on_find_ return a boolean if there is a chance all are clear to avoid calling `try_clear?`\n // we may find that all are cleared only on a second iteration but that is ok as the chunkmap is a conservative approximation.\n mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx);\n }\n }\n mi_bfield_cycle_iterate_end(Y);\n }\n }\n mi_bfield_cycle_iterate_end(X);\n return false;\n}\n\n/* --------------------------------------------------------------------------------\n mi_bbitmap_try_find_and_clear -- used to find free pages\n note: the compiler will fully inline the indirect function calls\n-------------------------------------------------------------------------------- */\n\nbool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) {\n return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 1, pidx, &mi_bchunk_try_find_and_clear_1);\n}\n\nbool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) {\n return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, 8, pidx, &mi_bchunk_try_find_and_clear_8);\n}\n\n// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx) {\n// return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, MI_BFIELD_BITS, pidx, &mi_bchunk_try_find_and_clear_X);\n// }\n\nbool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {\n mi_assert_internal(n<=MI_BFIELD_BITS);\n return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNX);\n}\n\nbool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {\n mi_assert_internal(n<=MI_BCHUNK_BITS);\n return mi_bbitmap_try_find_and_clear_generic(bbitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNC);\n}\n\n\n/* --------------------------------------------------------------------------------\n mi_bbitmap_try_find_and_clear for huge objects spanning multiple chunks\n-------------------------------------------------------------------------------- */\n\n// Try to atomically clear `n` bits starting at `chunk_idx` where `n` can span over multiple chunks\nstatic bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_t n) {\n mi_assert_internal((chunk_idx * MI_BCHUNK_BITS) + n <= mi_bbitmap_max_bits(bbitmap));\n\n size_t m = n; // bits to go\n size_t count = 0; // chunk count\n while (m > 0) {\n mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count];\n if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL)) {\n goto rollback;\n }\n m = (m <= MI_BCHUNK_BITS ? 0 : m - MI_BCHUNK_BITS);\n count++;\n }\n return true;\n\nrollback:\n // we only need to reset chunks the we just fully cleared\n while (count > 0) {\n count--;\n mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count];\n mi_bchunk_setN(chunk, 0, MI_BCHUNK_BITS, NULL);\n }\n return false;\n}\n\n// Go through the bbitmap to find a sequence of `n` bits and clear them atomically where `n > MI_ARENA_MAX_CHUNK_OBJ_SIZE`\n// Since these are very large object allocations we always search from the start and only consider starting at the start\n// of a chunk (for fragmentation and efficiency).\n// Todo: for now we try to find full empty chunks to cover `n` but we can allow a partial chunk at the end\n// Todo: This scans directly through the chunks -- we might want to consult the cmap as well?\nbool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {\n MI_UNUSED(tseq);\n mi_assert(n > 0); if (n==0) { return false; }\n\n const size_t chunk_max = mi_bbitmap_chunk_count(bbitmap);\n const size_t chunk_req = _mi_divide_up(n, MI_BCHUNK_BITS); // minimal number of chunks needed\n if (chunk_max < chunk_req) { return false; }\n\n // iterate through the chunks\n size_t chunk_idx = 0;\n while (chunk_idx <= chunk_max - chunk_req)\n {\n size_t count = 0; // chunk count\n do {\n mi_assert_internal(chunk_idx + count < chunk_max);\n mi_bchunk_t* const chunk = &bbitmap->chunks[chunk_idx + count];\n if (!mi_bchunk_all_are_set_relaxed(chunk)) {\n break;\n }\n else {\n count++;\n }\n }\n while (count < chunk_req);\n\n // did we find a suitable range?\n if (count == chunk_req) {\n // now try to claim it!\n if (mi_bchunk_try_clearN_(bbitmap, chunk_idx, n)) {\n *pidx = (chunk_idx * MI_BCHUNK_BITS);\n for (size_t i = 0; i < count; i++) {\n mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx + i, MI_CBIN_HUGE);\n }\n mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap));\n return true;\n }\n }\n\n // keep searching but skip the scanned range\n chunk_idx += count+1;\n }\n return false;\n}\n\n\n\n\n\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/bitmap.h", "sha256": "2e7a54ff44592a1fe8fe929e691322622f8aa1e0b1e80954781f332a88dd6cd1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2024 Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* ----------------------------------------------------------------------------\nConcurrent bitmap that can set/reset sequences of bits atomically\n---------------------------------------------------------------------------- */\n#pragma once\n#ifndef MI_BITMAP_H\n#define MI_BITMAP_H\n\n/* --------------------------------------------------------------------------------\n Atomic bitmaps with release/acquire guarantees:\n\n `mi_bfield_t`: is a single machine word that can efficiently be bit counted (usually `size_t`)\n each bit usually represents a single MI_ARENA_SLICE_SIZE in an arena (64 KiB).\n We need 16K bits to represent a 1GiB arena.\n\n `mi_bchunk_t`: a chunk of bfield's of a total of MI_BCHUNK_BITS (= 512 on 64-bit, 256 on 32-bit)\n allocations never span across chunks -- so MI_ARENA_MAX_OBJ_SIZE is the number\n of bits in a chunk times the MI_ARENA_SLICE_SIZE (512 * 64KiB = 32 MiB).\n These chunks are cache-aligned and we can use AVX2/AVX512/NEON/SVE/SVE2/etc. instructions\n to scan for bits (perhaps) more efficiently.\n\n We allocate byte-sized ranges aligned to bytes in the bfield, and bfield-sized\n ranges aligned to a bfield.\n\n Searching linearly through the chunks would be too slow (16K bits per GiB).\n Instead we add a \"chunkmap\" to do a two-level search (more or less a btree of depth 2).\n\n `mi_bchunkmap_t` (== `mi_bchunk_t`): for each chunk we track if it has (potentially) any bit set.\n The chunkmap has 1 bit per chunk that is set if the chunk potentially has a bit set.\n This is used to avoid scanning every chunk. (and thus strictly an optimization)\n It is conservative: it is fine to set a bit in the chunk map even if the chunk turns out\n to have no bits set. It is also allowed to briefly have a clear bit even if the\n chunk has bits set -- as long as we guarantee that the bit will be set later on;\n (this allows us to set the chunkmap bit right after we set a bit in the corresponding chunk).\n\n However, when we clear a bit in a chunk, and the chunk is indeed all clear, we\n cannot safely clear the bit corresponding to the chunk in the chunkmap since it\n may race with another thread setting a bit in the same chunk. Therefore, when\n clearing, we first test if a chunk is clear, then clear the chunkmap bit, and\n then test again to catch any set bits that we may have missed.\n\n Since the chunkmap may thus be briefly out-of-sync, this means that we may sometimes\n not find a free page even though it's there (but we accept this as we avoid taking\n full locks). (Another way to do this is to use an epoch but we like to avoid that complexity\n for now).\n\n `mi_bitmap_t`: a bitmap with N chunks. A bitmap has a chunkmap of MI_BCHUNK_BITS (512)\n and thus has at most 512 chunks (=2^18 bits x 64 KiB slices = 16 GiB max arena size).\n The minimum is 1 chunk which is a 32 MiB arena.\n\n For now, the implementation assumes MI_HAS_FAST_BITSCAN and uses trailing-zero-count\n and pop-count (but we think it can be adapted work reasonably well on older hardware too)\n--------------------------------------------------------------------------------------------- */\n\n// A word-size bit field.\ntypedef size_t mi_bfield_t;\n\n#define MI_BFIELD_BITS_SHIFT (MI_SIZE_SHIFT+3)\n#define MI_BFIELD_BITS (1 << MI_BFIELD_BITS_SHIFT)\n#define MI_BFIELD_SIZE (MI_BFIELD_BITS/8)\n#define MI_BFIELD_LO_BIT8 (((~(mi_bfield_t)0))/0xFF) // 0x01010101 ..\n#define MI_BFIELD_HI_BIT8 (MI_BFIELD_LO_BIT8 << 7) // 0x80808080 ..\n\n#define MI_BCHUNK_SIZE (MI_BCHUNK_BITS / 8)\n#define MI_BCHUNK_FIELDS (MI_BCHUNK_BITS / MI_BFIELD_BITS) // 8 on both 64- and 32-bit\n\n\n// some compiler (msvc in C mode) cannot have expressions in the alignment attribute\n#if MI_BCHUNK_SIZE==64\n#define mi_decl_bchunk_align mi_decl_align(64)\n#elif MI_BCHUNK_SIZE==32\n#define mi_decl_bchunk_align mi_decl_align(32)\n#else\n#define mi_decl_bchunk_align mi_decl_align(MI_BCHUNK_SIZE)\n#endif\n\n\n// A bitmap chunk contains 512 bits on 64-bit (256 on 32-bit)\ntypedef mi_decl_bchunk_align struct mi_bchunk_s {\n _Atomic(mi_bfield_t) bfields[MI_BCHUNK_FIELDS];\n} mi_bchunk_t;\n\n\n// The chunkmap has one bit per corresponding chunk that is set if the chunk potentially has bits set.\n// The chunkmap is itself a chunk.\ntypedef mi_bchunk_t mi_bchunkmap_t;\n\n#define MI_BCHUNKMAP_BITS MI_BCHUNK_BITS\n\n#define MI_BITMAP_MAX_CHUNK_COUNT (MI_BCHUNKMAP_BITS)\n#define MI_BITMAP_MIN_CHUNK_COUNT (1)\n#if MI_SIZE_BITS > 32\n#define MI_BITMAP_DEFAULT_CHUNK_COUNT (64) // 2 GiB on 64-bit -- this is for the page map\n#else\n#define MI_BITMAP_DEFAULT_CHUNK_COUNT (1)\n#endif\n#define MI_BITMAP_MAX_BIT_COUNT (MI_BITMAP_MAX_CHUNK_COUNT * MI_BCHUNK_BITS) // 16 GiB arena\n#define MI_BITMAP_MIN_BIT_COUNT (MI_BITMAP_MIN_CHUNK_COUNT * MI_BCHUNK_BITS) // 32 MiB arena\n#define MI_BITMAP_DEFAULT_BIT_COUNT (MI_BITMAP_DEFAULT_CHUNK_COUNT * MI_BCHUNK_BITS) // 2 GiB arena\n\n\n// An atomic bitmap\ntypedef mi_decl_bchunk_align struct mi_bitmap_s {\n _Atomic(size_t) chunk_count; // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS)\n size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 1]; // suppress warning on msvc\n mi_bchunkmap_t chunkmap;\n mi_bchunk_t chunks[MI_BITMAP_DEFAULT_CHUNK_COUNT]; // usually dynamic MI_BITMAP_MAX_CHUNK_COUNT\n} mi_bitmap_t;\n\n\nstatic inline size_t mi_bitmap_chunk_count(const mi_bitmap_t* bitmap) {\n return mi_atomic_load_relaxed(&((mi_bitmap_t*)bitmap)->chunk_count);\n}\n\nstatic inline size_t mi_bitmap_max_bits(const mi_bitmap_t* bitmap) {\n return (mi_bitmap_chunk_count(bitmap) * MI_BCHUNK_BITS);\n}\n\n\n\n/* --------------------------------------------------------------------------------\n Atomic bitmap operations\n-------------------------------------------------------------------------------- */\n\n// Many operations are generic over setting or clearing the bit sequence: we use `mi_xset_t` for this (true if setting, false if clearing)\ntypedef bool mi_xset_t;\n#define MI_BIT_SET (true)\n#define MI_BIT_CLEAR (false)\n\n\n// Required size of a bitmap to represent `bit_count` bits.\nsize_t mi_bitmap_size(size_t bit_count, size_t* chunk_count);\n\n// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true\n// returns the size of the bitmap.\nsize_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero);\n\n// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks).\n// Not atomic so only use if still local to a thread.\nvoid mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n);\n\n\n// Set a bit in the bitmap; returns `true` if it atomically transitioned from 0 to 1\nbool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx);\n\n// Clear a bit in the bitmap; returns `true` if it atomically transitioned from 1 to 0\nbool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx);\n\n// Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 0's to 1's\n// If `already_set` is not NULL, it is set to count of bits were already all set.\n// (this is used for correct statistics if commiting over a partially committed area)\nbool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* already_set);\n\n// Clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 1's to 0's\nbool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n);\n\n\n// Is a sequence of n bits already all set/cleared?\nbool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n);\n\n// Is the bitmap completely clear?\nbool mi_bitmap_is_all_clear(mi_bitmap_t* bitmap);\n\n// Is a sequence of n bits already set?\n// (Used to check if a memory range is already committed)\nstatic inline bool mi_bitmap_is_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) {\n return mi_bitmap_is_xsetN(MI_BIT_SET, bitmap, idx, n);\n}\n\n// Is a sequence of n bits already clear?\nstatic inline bool mi_bitmap_is_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) {\n return mi_bitmap_is_xsetN(MI_BIT_CLEAR, bitmap, idx, n);\n}\n\nstatic inline bool mi_bitmap_is_set(mi_bitmap_t* bitmap, size_t idx) {\n return mi_bitmap_is_setN(bitmap, idx, 1);\n}\n\nstatic inline bool mi_bitmap_is_clear(mi_bitmap_t* bitmap, size_t idx) {\n return mi_bitmap_is_clearN(bitmap, idx, 1);\n}\n\n// Called once a bit is cleared to see if the memory slice can be claimed.\ntypedef bool (mi_claim_fun_t)(size_t slice_index, mi_arena_t* arena, bool* keep_set);\n\n// Find a set bits in the bitmap, atomically clear it, and check if `claim` returns true.\n// If not claimed, continue on (potentially setting the bit again depending on `keep_set`).\n// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.\nmi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx,\n mi_claim_fun_t* claim, mi_arena_t* arena );\n\n\n// Atomically clear a bit but only if it is set. Will block otherwise until the bit is set.\n// This is used to delay free-ing a page that it at the same time being considered to be\n// allocated from `mi_arena_try_abandoned` (and is in the `claim` function of `mi_bitmap_try_find_and_claim`).\nvoid mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx);\n\n\n// If a bit is set in the bitmap, return `true` and set `idx` to the index of the highest bit.\n// Otherwise return `false` (and `*idx` is undefined).\n// Used for unloading arena's\nbool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx);\n\n// Return count of all set bits in a bitmap.\nsize_t mi_bitmap_popcount(mi_bitmap_t* bitmap);\n\n\ntypedef bool (mi_forall_set_fun_t)(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg2);\n\n// Visit all set bits in a bitmap (`slice_count == 1`)\nbool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);\n\n// Visit all set bits in a bitmap with larger ranges if possible (`slice_count >= 1`)\n// Ranges will never cross chunk boundaries though (and `slice_count <= MI_BCHUNK_BITS`)\nbool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);\n\n// Visit all set bits in a bitmap with at least `rngslices` at a time (and aligned to `rngslices`). \n// This is used by purging to not break up transparent huge pages for example.\n// Ranges will never cross chunk boundaries (and `slice_count <= MI_BCHUNK_BITS`).\nbool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);\n\n// Count all set bits in given range in the bitmap.\nsize_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n);\n\n/* ----------------------------------------------------------------------------\n Binned concurrent bitmap\n Assigns a size class to each chunk such that small blocks don't cause too\n much fragmentation since we keep chunks for larger blocks separate.\n---------------------------------------------------------------------------- */\n\n// mi_chunkbin_t is defined in mimalloc-stats.h\n\nstatic inline mi_chunkbin_t mi_chunkbin_inc(mi_chunkbin_t bbin) {\n mi_assert_internal(bbin < MI_CBIN_COUNT);\n return (mi_chunkbin_t)((int)bbin + 1);\n}\n\nstatic inline mi_chunkbin_t mi_chunkbin_dec(mi_chunkbin_t bbin) {\n mi_assert_internal(bbin > MI_CBIN_NONE);\n return (mi_chunkbin_t)((int)bbin - 1);\n}\n\nstatic inline mi_chunkbin_t mi_chunkbin_of(size_t slice_count) {\n if (slice_count==1) return MI_CBIN_SMALL;\n if (slice_count==8) return MI_CBIN_MEDIUM;\n #if MI_ENABLE_LARGE_PAGES\n if (slice_count==MI_BFIELD_BITS) return MI_CBIN_LARGE;\n #endif\n if (slice_count > MI_BCHUNK_BITS) return MI_CBIN_HUGE;\n return MI_CBIN_OTHER;\n}\n\n// An atomic \"binned\" bitmap for the free slices where we keep chunks reserved for particular size classes\ntypedef mi_decl_bchunk_align struct mi_bbitmap_s {\n _Atomic(size_t) chunk_count; // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS)\n _Atomic(size_t) chunk_max_accessed; // max chunk index that was once cleared or set\n #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 2\n size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 2]; // suppress warning on msvc by aligning manually\n #endif\n mi_bchunkmap_t chunkmap;\n mi_bchunkmap_t chunkmap_bins[MI_CBIN_COUNT - 1]; // chunkmaps with bit set if the chunk is in that size class (excluding MI_CBIN_NONE)\n mi_bchunk_t chunks[MI_BITMAP_DEFAULT_CHUNK_COUNT]; // usually dynamic MI_BITMAP_MAX_CHUNK_COUNT\n} mi_bbitmap_t;\n\n\nstatic inline size_t mi_bbitmap_chunk_count(const mi_bbitmap_t* bbitmap) {\n return mi_atomic_load_relaxed(&((mi_bbitmap_t*)bbitmap)->chunk_count);\n}\n\nstatic inline size_t mi_bbitmap_max_bits(const mi_bbitmap_t* bbitmap) {\n return (mi_bbitmap_chunk_count(bbitmap) * MI_BCHUNK_BITS);\n}\n\nmi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunk_t* chunkmap_bins, size_t chunk_idx);\n\nsize_t mi_bbitmap_size(size_t bit_count, size_t* chunk_count);\n\n// If a bit is clear in the bitmap, return `true` and set `idx` to the index of the highest bit that was clear.\n// Otherwise return `false` (and `*idx` is undefined).\n// Used for debug output.\nbool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx);\n\n// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true\n// returns the size of the bitmap.\nsize_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero);\n\n// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks).\n// Not atomic so only use if still local to a thread.\nvoid mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n);\n\n\n// Set a sequence of `n` bits in the bbitmap; returns `true` if atomically transitioned from all 0's to 1's\nbool mi_bbitmap_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n);\n\n\n// Is a sequence of n bits already all set/cleared?\nbool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_t n);\n\n// Is a sequence of n bits already set?\n// (Used to check if a memory range is already committed)\nstatic inline bool mi_bbitmap_is_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {\n return mi_bbitmap_is_xsetN(MI_BIT_SET, bbitmap, idx, n);\n}\n\n// Is a sequence of n bits already clear?\nstatic inline bool mi_bbitmap_is_clearN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {\n return mi_bbitmap_is_xsetN(MI_BIT_CLEAR, bbitmap, idx, n);\n}\n\n\n// Try to atomically transition `n` bits from all set to all clear. Returns `true` on succes.\n// `n` cannot cross chunk boundaries, where `n <= MI_CHUNK_BITS`.\nbool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n);\n\n\n// Specialized versions for common bit sequence sizes\nbool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 1-bit\nbool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 8-bits\n// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // MI_BFIELD_BITS\nbool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // < MI_BFIELD_BITS\nbool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BFIELD_BITS <= MI_BCHUNK_BITS\nbool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BCHUNK_BITS\n\n// Find a sequence of `n` bits in the bbitmap with all bits set, and try to atomically clear all.\n// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.\nmi_decl_nodiscard static inline bool mi_bbitmap_try_find_and_clearN(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {\n if (n==1) return mi_bbitmap_try_find_and_clear(bbitmap, tseq, pidx); // small pages\n if (n==8) return mi_bbitmap_try_find_and_clear8(bbitmap, tseq, pidx); // medium pages\n // if (n==MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearX(bbitmap, tseq, pidx); // large pages\n if (n==0) return false;\n if (n<=MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearNX(bbitmap, tseq, n, pidx);\n if (n<=MI_BCHUNK_BITS) return mi_bbitmap_try_find_and_clearNC(bbitmap, tseq, n, pidx);\n return mi_bbitmap_try_find_and_clearN_(bbitmap, tseq, n, pidx);\n}\n\n\n#endif // MI_BITMAP_H\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/free.c", "sha256": "c9a845dd7b9020f26d73761d235fb75e7bc4a3682576de750050ac4098211b43", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#if !defined(MI_IN_ALLOC_C)\n#error \"this file should be included from 'alloc.c' (so aliases can work from alloc-override)\"\n// add includes help an IDE\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\" // _mi_prim_thread_id()\n#endif\n\n// forward declarations\nstatic void mi_check_padding(const mi_page_t* page, const mi_block_t* block);\nstatic bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block);\nstatic size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded);\nstatic void mi_stat_free(const mi_page_t* page, const mi_block_t* block);\n\n\n// ------------------------------------------------------\n// Free\n// ------------------------------------------------------\n\n// regular free of a (thread local) block pointer\n// fast path written carefully to prevent spilling on the stack\nstatic inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full)\n{\n MI_UNUSED(was_guarded);\n // checks \n if mi_unlikely(mi_check_is_double_free(page, block)) return;\n if (!was_guarded) { mi_check_padding(page, block); }\n if (track_stats) { mi_stat_free(page, block); }\n #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN\n memset(block, MI_DEBUG_FREED, mi_page_block_size(page));\n #endif\n if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned\n\n // actual free: push on the local free list\n mi_block_set_next(page, block, page->local_free);\n page->local_free = block;\n if mi_unlikely(--page->used == 0) {\n if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page)\n _mi_page_retire(page); \n }\n }\n else if mi_unlikely(check_full && mi_page_is_in_full(page)) {\n _mi_page_unfull(page);\n }\n}\n\n// Forward declaration for multi-threaded collect\nstatic void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept;\n\n// Free a block multi-threaded\nstatic inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded) mi_attr_noexcept\n{\n MI_UNUSED(was_guarded);\n // adjust stats (after padding check and potentially recursive `mi_free` above)\n mi_stat_free(page, block); // stat_free may access the padding\n mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded));\n\n // _mi_padding_shrink(page, block, sizeof(mi_block_t));\n#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading\n if (!was_guarded) {\n size_t dbgsize = mi_usable_size(block);\n if (dbgsize > MI_MiB) { dbgsize = MI_MiB; }\n _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize);\n }\n#endif\n\n // push atomically on the page thread free list\n mi_thread_free_t tf_new;\n mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);\n do {\n mi_block_set_next(page, block, mi_tf_block(tf_old));\n tf_new = mi_tf_create(block, true /* always use owned: try to claim it if the page is abandoned */);\n } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough?\n\n // and atomically try to collect the page if it was abandoned\n const bool is_owned_now = !mi_tf_is_owned(tf_old);\n if (is_owned_now) {\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_free_try_collect_mt(page,block);\n }\n}\n\n\n// Adjust a block that was allocated aligned, to the actual start of the block in the page.\n// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the\n// `page_start` and `block_size` fields; however these are constant and the page won't be\n// deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently.\nmi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {\n mi_assert_internal(page!=NULL && p!=NULL);\n\n const size_t diff = (uint8_t*)p - mi_page_start(page);\n const size_t block_size = mi_page_block_size(page);\n const size_t adjust = (_mi_is_power_of_two(block_size) ? diff & (block_size - 1) : diff % block_size);\n return (mi_block_t*)((uintptr_t)p - adjust);\n}\n\n// forward declaration for a MI_GUARDED build\n#if MI_GUARDED\nstatic void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration\nstatic inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {\n if (mi_block_ptr_is_guarded(block, p)) { \n mi_block_unguard(page, block, p); \n return true;\n }\n else {\n return false;\n }\n}\n#else\nstatic inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {\n MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);\n return false;\n}\n#endif\n\nstatic inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, void* p ) {\n mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer\n #if MI_SECURE > 0\n // in secure mode we always unalign to guard against free-ing interior pointers\n return _mi_page_ptr_unalign(page,p);\n #else\n MI_UNUSED(page);\n return (mi_block_t*)p;\n #endif\n}\n\n\n// free a local pointer (page parameter comes first for better codegen)\nstatic void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept {\n mi_assert_internal(p!=NULL && page != NULL);\n mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));\n const bool was_guarded = mi_block_check_unguard(page, block, p);\n mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */);\n}\n\n// free a pointer owned by another thread (page parameter comes first for better codegen)\nstatic void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p) mi_attr_noexcept {\n mi_assert_internal(p!=NULL && page != NULL);\n mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));\n const bool was_guarded = mi_block_check_unguard(page, block, p);\n mi_free_block_mt(page, block, was_guarded);\n}\n\n// generic free (for runtime integration)\nvoid mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) mi_attr_noexcept {\n if (is_local) mi_free_generic_local(page,p);\n else mi_free_generic_mt(page,p);\n}\n\n\n// Get the page belonging to a pointer\n// Does further checks in debug mode to see if this was a valid pointer.\nstatic inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg)\n{\n MI_UNUSED_RELEASE(msg);\n #if MI_DEBUG\n if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) {\n _mi_error_message(EINVAL, \"%s: invalid (unaligned) pointer: %p\\n\", msg, p);\n return NULL;\n }\n mi_page_t* page = _mi_safe_ptr_page(p);\n if (p != NULL && page == NULL) {\n _mi_error_message(EINVAL, \"%s: invalid pointer: %p\\n\", msg, p);\n }\n return page;\n #else\n return _mi_ptr_page(p);\n #endif\n}\n\n// Free a block\n// Fast path written carefully to prevent register spilling on the stack\nstatic mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* page) \n{\n if mi_unlikely(page==NULL) return; // page will be NULL if p==NULL\n mi_assert_internal(p!=NULL && page!=NULL);\n if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }\n\n const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page));\n if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0`\n // thread-local, aligned, and not a full page\n mi_block_t* const block = mi_validate_block_from_ptr(page,p);\n mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */);\n }\n else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0`\n // page is local, but is full or contains (inner) aligned blocks; use generic path\n mi_free_generic_local(page, p);\n }\n // free-ing in a page owned by a theap in another thread, or an abandoned page (not belonging to a theap)\n else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0`\n // blocks are aligned (and not a full page); push on the thread_free list\n mi_block_t* const block = mi_validate_block_from_ptr(page,p);\n mi_free_block_mt(page,block,false /* was_guarded */);\n }\n else {\n // page is full or contains (inner) aligned blocks; use generic multi-thread path\n mi_free_generic_mt(page, p);\n }\n}\n\nvoid mi_free(void* p) mi_attr_noexcept {\n mi_page_t* const page = mi_validate_ptr_page(p,\"mi_free\"); \n mi_free_ex(p, NULL, page);\n}\n\nvoid mi_ufree(void* p, size_t* usable) mi_attr_noexcept {\n mi_page_t* const page = mi_validate_ptr_page(p,\"mi_ufree\"); \n mi_free_ex(p, usable, page);\n}\n\nvoid mi_free_small(void* p) mi_attr_noexcept {\n // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`.\n // If we keep page info in front of the page area for small objects, we can find the info\n // just by aligning down the pointer instead of looking it up in the page map.\n #if MI_PAGE_META_ALIGNED_FREE_SMALL \n #if MI_GUARDED \n #warning \"MI_PAGE_META_ALIGNED_FREE_SMALL ignored as MI_GUARDED is defined\"\n mi_free(p);\n #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE\n #warning \"MI_PAGE_META_ALIGNED_FREE_SMALL ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size\"\n mi_free(p);\n #else\n mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE);\n mi_assert(page == mi_validate_ptr_page(p,\"mi_free_small\"));\n mi_assert((void*)page == _mi_align_down_ptr(page->page_start,MI_SMALL_PAGE_SIZE));\n mi_assert(page->block_size <= MI_SMALL_SIZE_MAX); // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small`\n mi_free_ex(p, NULL, page);\n #endif\n #else\n mi_free(p);\n #endif \n}\n\n\n// --------------------------------------------------------------------------------------------\n// `mi_free_try_collect_mt`: Potentially collect a page in a free in an abandoned page.\n// 1. if the page becomes empty, free it\n// 2. if it can be reclaimed, reclaim it in our theap\n// 3. if it went to < 7/8th used, re-abandon to be mapped (so it can be found by theaps looking for free pages)\n// --------------------------------------------------------------------------------------------\n\n// Helper for mi_free_try_collect_mt: free if the page has no more used blocks (this is updated by `_mi_page_free_collect(_partly)`)\nstatic bool mi_abandoned_page_try_free(mi_page_t* page)\n{\n if (!mi_page_all_free(page)) return false;\n // first remove it from the abandoned pages in the arena (if mapped, this might wait for any readers to finish)\n _mi_arenas_page_unabandon(page,NULL);\n _mi_arenas_page_free(page,NULL); // we can now free the page directly\n return true;\n}\n\n// Helper for mi_free_try_collect_mt: try if we can reabandon a previously abandoned mostly full page to be mapped\nstatic bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page)\n{\n // if the page is unmapped, try to reabandon so it can possibly be mapped and found for allocations\n // We only reabandon if a full page starts to have enough blocks available to prevent immediate re-abandon of a full page\n if (mi_page_is_mostly_used(page)) return false; // not too full\n if (page->memid.memkind != MI_MEM_ARENA || mi_page_is_abandoned_mapped(page)) return false; // and not already mapped (or unmappable)\n\n mi_assert(!mi_page_is_full(page));\n return _mi_arenas_page_try_reabandon_to_mapped(page);\n}\n\n// Release ownership of a page. This may free or reabandoned the page if other blocks are concurrently\n// freed in the meantime. Returns `true` if the page was freed.\n// By passing the captured `expected_thread_free`, we can often avoid calling `mi_page_free_collect`.\nstatic void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free) {\n mi_assert_internal(mi_page_is_owned(page));\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_assert_internal(!mi_page_all_free(page));\n // try to cas atomically the original free list (`mt_free`) back with the ownership cleared.\n mi_thread_free_t tf_expect = mi_tf_create(expected_thread_free, true);\n mi_thread_free_t tf_new = mi_tf_create(expected_thread_free, false);\n while mi_unlikely(!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_expect, tf_new)) {\n mi_assert_internal(mi_tf_is_owned(tf_expect));\n // while the xthread_free list is not empty..\n while (mi_tf_block(tf_expect) != NULL) {\n // if there were concurrent updates to the thread-free list, we retry to free or reabandon to mapped (if it became !mosty_used).\n _mi_page_free_collect(page,false); // update used count\n if (mi_abandoned_page_try_free(page)) return;\n if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;\n // otherwise continue un-owning\n tf_expect = mi_atomic_load_relaxed(&page->xthread_free);\n }\n // and try again to release ownership\n mi_assert_internal(mi_tf_block(tf_expect)==NULL);\n tf_new = mi_tf_create(NULL, false);\n }\n}\n\nstatic inline bool mi_page_queue_len_is_atmost( mi_theap_t* theap, size_t block_size, long atmost) {\n if (atmost < 0) return false;\n mi_page_queue_t* const pq = mi_page_queue(theap,block_size);\n mi_assert_internal(pq!=NULL);\n return (pq->count <= (size_t)atmost);\n}\n\n// Helper for mi_free_try_collect_mt: try to reclaim the page for ourselves\nstatic mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long reclaim_on_free) mi_attr_noexcept\n{\n // note: reclaiming can improve benchmarks like `larson` or `rbtree-ck` a lot even in the single-threaded case,\n // since free-ing from an owned page avoids atomic operations. However, if we reclaim too eagerly in\n // a multi-threaded scenario we may start to hold on to too much memory and reduce reuse among threads.\n // If the current theap is where the page originally came from, we reclaim much more eagerly while\n // 'cross-thread' reclaiming on free is by default off (and we only 'reclaim' these by finding the abandoned\n // pages when we allocate a fresh page).\n mi_assert_internal(mi_page_is_owned(page));\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_assert_internal(!mi_page_all_free(page));\n mi_assert_internal(page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE);\n mi_assert_internal(reclaim_on_free >= 0);\n\n // dont reclaim if we just have terminated this thread and we should\n // not reinitialize the theap for this thread. (can happen due to thread-local destructors for example -- issue #944)\n if (!_mi_thread_is_initialized()) return false;\n\n // get our theap \n mi_theap_t* const theap = _mi_page_associated_theap_peek(page);\n if (theap==NULL || !theap->allow_page_reclaim) return false;\n \n // todo: cache `is_in_threadpool` and `exclusive_arena` directly in the theap for performance?\n // set max_reclaim limit\n long max_reclaim = 0;\n if mi_likely(theap == page->theap) { // did this page originate from the current theap? (and thus allocated from this thread)\n // originating theap\n max_reclaim = _mi_option_get_fast(theap->tld->is_in_threadpool ? mi_option_page_cross_thread_max_reclaim : mi_option_page_max_reclaim);\n }\n else if (reclaim_on_free == 1 && // if cross-thread is allowed\n !theap->tld->is_in_threadpool && // and we are not part of a threadpool\n !mi_page_is_mostly_used(page) && // and the page is not too full\n _mi_arena_memid_is_suitable(page->memid, _mi_theap_heap(theap)->exclusive_arena)) { // and it fits our memory\n // across threads\n max_reclaim = _mi_option_get_fast(mi_option_page_cross_thread_max_reclaim);\n }\n\n // are we within the reclaim limit?\n if (max_reclaim >= 0 && !mi_page_queue_len_is_atmost(theap, page->block_size, max_reclaim)) {\n return false;\n }\n\n // reclaim the page into this theap\n // first remove it from the abandoned pages in the arena -- this might wait for any readers to finish\n _mi_arenas_page_unabandon(page, theap);\n _mi_theap_page_reclaim(theap, page);\n mi_theap_stat_counter_increase(theap, pages_reclaim_on_free, 1);\n return true;\n}\n\n\n// We freed a block in an abandoned page (that was not owned). Try to collect\nstatic void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept\n{\n mi_assert_internal(mi_page_is_owned(page));\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_assert_internal(mt_free != NULL);\n // we own the page now, and it is safe to collect the thread atomic free list\n if (page->block_size <= MI_SMALL_SIZE_MAX) {\n // use the `_partly` version to avoid atomic operations since we already have the `mt_free` pointing into the thread free list\n // (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted).\n // however, if the page became completely free, the used count is guaranteed to be 0.)\n mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used.\n _mi_page_free_collect_partly(page, mt_free); \n }\n else {\n // for larger blocks we use the regular collect \n _mi_page_free_collect(page,false /* no force */);\n mt_free = NULL; // expected page->xthread_free value after collection\n }\n const long reclaim_on_free = _mi_option_get_fast(mi_option_page_reclaim_on_free);\n #if MI_DEBUG > 1\n if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); }\n if (mi_page_is_full(page)) { mi_assert(mi_page_is_mostly_used(page)); }\n #endif\n\n // try to: 1. free it, 2. reclaim it, or 3. reabandon it to be mapped\n if (mi_abandoned_page_try_free(page)) return;\n if (page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE && reclaim_on_free >= 0) { // early test for better codegen\n if (mi_abandoned_page_try_reclaim(page, reclaim_on_free)) return;\n }\n if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;\n \n // otherwise unown the page again\n mi_abandoned_page_unown_from_free(page, mt_free);\n}\n\n\n// ------------------------------------------------------\n// Usable size\n// ------------------------------------------------------\n\n// Bytes available in a block\nstatic size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept {\n const mi_block_t* block = _mi_page_ptr_unalign(page, p);\n const bool is_guarded = mi_block_ptr_is_guarded(block,p);\n const size_t size = mi_page_usable_size_of(page, block, is_guarded);\n const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;\n mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);\n const size_t aligned_size = (size - adjust); \n return aligned_size;\n}\n\nstatic inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_attr_noexcept {\n if mi_unlikely(page==NULL) return 0;\n if mi_likely(!mi_page_has_interior_pointers(page)) {\n const mi_block_t* block = (const mi_block_t*)p;\n return mi_page_usable_size_of(page, block, false /* is guarded */);\n }\n else {\n // split out to separate routine for improved code generation\n return mi_page_usable_aligned_size_of(page, p);\n }\n}\n\nmi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept {\n const mi_page_t* const page = mi_validate_ptr_page(p,\"mi_usable_size\");\n return _mi_usable_size(p,page);\n}\n\n\n// ------------------------------------------------------\n// Free variants\n// ------------------------------------------------------\n\nvoid mi_free_size(void* p, size_t size) mi_attr_noexcept {\n MI_UNUSED_RELEASE(size);\n #if MI_DEBUG\n const mi_page_t* const page = mi_validate_ptr_page(p,\"mi_free_size\"); \n const size_t available = _mi_usable_size(p,page);\n mi_assert(p == NULL || size <= available || available == 0 /* invalid pointer */ );\n #endif\n mi_free(p);\n}\n\nvoid mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {\n MI_UNUSED_RELEASE(alignment);\n mi_assert(((uintptr_t)p % alignment) == 0);\n mi_free_size(p,size);\n}\n\nvoid mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {\n MI_UNUSED_RELEASE(alignment);\n mi_assert(((uintptr_t)p % alignment) == 0);\n mi_free(p);\n}\n\n\n// ------------------------------------------------------\n// Check for double free in secure and debug mode\n// This is somewhat expensive so only enabled for secure mode 4\n// ------------------------------------------------------\n\n#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0))\n// linear check if the free list contains a specific element\nstatic bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) {\n while (list != NULL) {\n if (elem==list) return true;\n list = mi_block_next(page, list);\n }\n return false;\n}\n\nstatic mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) {\n // The decoded value is in the same page (or NULL).\n // Walk the free lists to verify positively if it is already freed\n if (mi_list_contains(page, page->free, block) ||\n mi_list_contains(page, page->local_free, block) ||\n mi_list_contains(page, mi_page_thread_free(page), block))\n {\n _mi_error_message(EAGAIN, \"double free detected of block %p with size %zu\\n\", block, mi_page_block_size(page));\n return true;\n }\n return false;\n}\n\n#define mi_track_page(page,access) { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); }\n\nstatic inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {\n bool is_double_free = false;\n mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field\n if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer?\n (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL?\n {\n // Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free?\n // (continue in separate function to improve code generation)\n is_double_free = mi_check_is_double_freex(page, block);\n }\n return is_double_free;\n}\n#else\nstatic inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {\n MI_UNUSED(page);\n MI_UNUSED(block);\n return false;\n}\n#endif\n\n\n// ---------------------------------------------------------------------------\n// Check for theap block overflow by setting up padding at the end of the block\n// ---------------------------------------------------------------------------\n\n#if MI_PADDING // && !MI_TRACK_ENABLED\nstatic bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) {\n *bsize = mi_page_usable_block_size(page);\n const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize);\n mi_track_mem_defined(padding,sizeof(mi_padding_t));\n *delta = padding->delta;\n uint32_t canary = padding->canary;\n uintptr_t keys[2];\n keys[0] = page->keys[0];\n keys[1] = page->keys[1];\n bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize);\n mi_track_mem_noaccess(padding,sizeof(mi_padding_t));\n return ok;\n}\n\n// Return the exact usable size of a block.\nstatic size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {\n if (is_guarded) {\n const size_t bsize = mi_page_block_size(page);\n return (bsize - _mi_os_page_size());\n }\n else {\n size_t bsize;\n size_t delta;\n bool ok = mi_page_decode_padding(page, block, &delta, &bsize);\n mi_assert_internal(ok); mi_assert_internal(delta <= bsize);\n return (ok ? bsize - delta : 0);\n }\n}\n\n// When a non-thread-local block is freed, it becomes part of the thread delayed free\n// list that is freed later by the owning theap. If the exact usable size is too small to\n// contain the pointer for the delayed list, then shrink the padding (by decreasing delta)\n// so it will later not trigger an overflow error in `mi_free_block`.\nvoid _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {\n size_t bsize;\n size_t delta;\n bool ok = mi_page_decode_padding(page, block, &delta, &bsize);\n mi_assert_internal(ok);\n if (!ok || (bsize - delta) >= min_size) return; // usually already enough space\n mi_assert_internal(bsize >= min_size);\n if (bsize < min_size) return; // should never happen\n size_t new_delta = (bsize - min_size);\n mi_assert_internal(new_delta < bsize);\n mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize);\n mi_track_mem_defined(padding,sizeof(mi_padding_t));\n padding->delta = (uint32_t)new_delta;\n mi_track_mem_noaccess(padding,sizeof(mi_padding_t));\n}\n#else\nstatic size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {\n MI_UNUSED(is_guarded); MI_UNUSED(block);\n return mi_page_usable_block_size(page);\n}\n\nvoid _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {\n MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(min_size);\n}\n#endif\n\n#if MI_PADDING && MI_PADDING_CHECK\n\nstatic bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) {\n size_t bsize;\n size_t delta;\n bool ok = mi_page_decode_padding(page, block, &delta, &bsize);\n *size = *wrong = bsize;\n if (!ok) return false;\n mi_assert_internal(bsize >= delta);\n *size = bsize - delta;\n if (!mi_page_is_huge(page)) {\n uint8_t* fill = (uint8_t*)block + bsize - delta;\n const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes\n mi_track_mem_defined(fill, maxpad);\n for (size_t i = 0; i < maxpad; i++) {\n if (fill[i] != MI_DEBUG_PADDING) {\n *wrong = bsize - delta + i;\n ok = false;\n break;\n }\n }\n mi_track_mem_noaccess(fill, maxpad);\n }\n return ok;\n}\n\nstatic void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {\n size_t size;\n size_t wrong;\n if (!mi_verify_padding(page,block,&size,&wrong)) {\n _mi_error_message(EFAULT, \"buffer overflow in theap block %p of size %zu: write after %zu bytes\\n\", block, size, wrong );\n }\n}\n\n#else\n\nstatic void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {\n MI_UNUSED(page);\n MI_UNUSED(block);\n}\n\n#endif\n\n// only maintain stats for smaller objects if requested\n#if (MI_STAT>0)\nstatic void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {\n MI_UNUSED(block);\n mi_theap_t* const theap = _mi_theap_default();\n if (!mi_theap_is_initialized(theap)) return; // (for now) skip statistics if free'd after thread_done was called (usually a thread cleanup call by the OS)\n\n const size_t bsize = mi_page_usable_block_size(page);\n // #if (MI_STAT>1)\n // const size_t usize = mi_page_usable_size_of(page, block);\n // mi_theap_stat_decrease(theap, malloc_requested, usize);\n // #endif\n if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {\n mi_theap_stat_decrease(theap, malloc_normal, bsize);\n #if (MI_STAT > 1)\n mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], 1);\n #endif\n }\n else {\n const size_t bpsize = mi_page_block_size(page); // match stat in page.c:mi_huge_page_alloc\n mi_theap_stat_decrease(theap, malloc_huge, bpsize);\n }\n}\n#else\nvoid mi_stat_free(const mi_page_t* page, const mi_block_t* block) {\n MI_UNUSED(page); MI_UNUSED(block);\n}\n#endif\n\n\n// Remove guard page when building with MI_GUARDED\n#if MI_GUARDED\nstatic void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {\n MI_UNUSED(p);\n mi_assert_internal(mi_block_ptr_is_guarded(block, p));\n mi_assert_internal(mi_page_has_interior_pointers(page));\n mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t));\n mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED);\n\n const size_t bsize = mi_page_block_size(page);\n const size_t psize = _mi_os_page_size();\n mi_assert_internal(bsize > psize);\n mi_assert_internal(!page->memid.is_pinned);\n void* gpage = (uint8_t*)block + bsize - psize;\n mi_assert_internal(_mi_is_aligned(gpage, psize));\n _mi_os_unprotect(gpage, psize);\n}\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/heap.c", "sha256": "ae8e88737c9861e8e071fe5f1e7acc52093bcffd136a359a6b473101e968c14f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/*----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\" // _mi_theap_default\n\n\n/* -----------------------------------------------------------\n Heap's\n----------------------------------------------------------- */\n\nmi_theap_t* mi_heap_theap(mi_heap_t* heap) {\n return _mi_heap_theap(heap);\n}\n\nvoid mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node) {\n if (heap==NULL) { heap = mi_heap_main(); }\n heap->numa_node = (numa_node < 0 ? -1 : numa_node % _mi_os_numa_node_count());\n}\n\nvoid mi_heap_stats_merge_to_subproc(mi_heap_t* heap) {\n if (heap==NULL) { heap = mi_heap_main(); }\n _mi_stats_merge_into(&heap->subproc->stats, &heap->stats);\n}\n\nvoid mi_heap_stats_merge_to_main(mi_heap_t* heap) {\n if (heap==NULL) return;\n _mi_stats_merge_into(&mi_heap_main()->stats, &heap->stats);\n}\n\nstatic mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_heap)\n{\n mi_heap_t* heap = (mi_heap_t*)const_heap;\n mi_assert_internal(heap!=NULL);\n\n if (_mi_is_heap_main(heap)) {\n // this can be called if the (main) thread is not yet initialized (as no allocation happened)\n // but `theap_main_init_get()` will call `mi_thread_init()`\n mi_theap_t* const theap = _mi_theap_main_safe();\n mi_assert_internal(theap!=NULL && _mi_is_heap_main(_mi_theap_heap(theap)));\n return theap;\n }\n\n // otherwise initialize the theap for this heap\n // get the thread local\n mi_assert_internal(heap->theap != 0);\n if (heap->theap==0) { // paranoia\n _mi_error_message(EFAULT, \"no thread-local reserved for heap (%p)\\n\", heap);\n return NULL;\n }\n mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);\n\n // create a fresh theap?\n if (theap==NULL) {\n // set first an invalid value to ensure the thread local storage is allocated\n if (!_mi_thread_local_set(heap->theap, (mi_theap_t*)1)) {\n _mi_error_message(EFAULT, \"unable to allocate memory for thread local storage\\n\");\n return NULL;\n } \n // then allocate the theap\n theap = _mi_theap_create(heap, _mi_theap_default_safe()->tld);\n _mi_thread_local_set(heap->theap, theap); // Cannot fail now as it was set before. Always set so the local is valid or NULL (and not 1)\n if (theap==NULL) {\n _mi_error_message(EFAULT, \"unable to allocate memory for a thread local heap\\n\");\n return NULL;\n } \n }\n return theap;\n}\n\n\n// get the theap for a heap without initializing (and return NULL in that case)\nmi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap) {\n if (heap==NULL || _mi_is_heap_main(heap)) {\n return _mi_theap_main_safe(); \n }\n else {\n return (mi_theap_t*)_mi_thread_local_get(heap->theap);\n }\n}\n\n// get (and possibly create) the theap belonging to a heap\nmi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap)\n{\n mi_theap_t* theap = _mi_heap_theap_peek(heap);\n if mi_unlikely(theap==NULL) {\n theap = mi_heap_init_theap(heap);\n if (theap==NULL) { return (mi_theap_t*)&_mi_theap_empty_wrong; } // this will return NULL from page.c:_mi_malloc_generic\n }\n _mi_theap_cached_set(theap);\n return theap;\n}\n\n\nmi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) {\n // always allocate heap data in the (subprocess) main heap\n mi_heap_t* const heap_main = mi_heap_main();\n // todo: allocate heap data in the exclusive arena ?\n mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) );\n if (heap==NULL) return NULL;\n\n // reserve a thread local slot for this heap (see also issue #1230)\n const mi_thread_local_t theap_slot = _mi_thread_local_create();\n if (theap_slot == 0) {\n _mi_error_message(EFAULT, \"unable to dynamically create a thread local for a heap\\n\");\n mi_free(heap);\n return NULL;\n }\n\n // init fields\n heap->theap = theap_slot;\n heap->subproc = heap_main->subproc;\n heap->heap_seq = mi_atomic_increment_relaxed(&heap_main->subproc->heap_total_count);\n heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id);\n heap->numa_node = -1; // no initial affinity\n mi_stats_header_init(&heap->stats);\n mi_lock_init(&heap->theaps_lock);\n mi_lock_init(&heap->os_abandoned_pages_lock);\n mi_lock_init(&heap->arena_pages_lock);\n\n // push onto the subproc heaps\n mi_lock(&heap->subproc->heaps_lock) {\n mi_heap_t* head = heap->subproc->heaps;\n heap->prev = NULL;\n heap->next = head;\n if (head!=NULL) { head->prev = heap; }\n heap->subproc->heaps = heap;\n }\n mi_atomic_increment_relaxed(&heap_main->subproc->heap_count);\n mi_subproc_stat_increase(heap_main->subproc, heaps, 1);\n return heap;\n}\n\nmi_heap_t* mi_heap_new(void) {\n return mi_heap_new_in_arena(0);\n}\n\n// free all theaps belonging to this heap (without deleting their pages as we do this arena wise for efficiency)\nstatic void mi_heap_free_theaps(mi_heap_t* heap) {\n // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`), \n // and we need to ensure we free theaps atomically.\n // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock \n // potential concurrent thread termination which tries to remove the theap from our theaps list.\n bool all_freed;\n do {\n all_freed = true;\n mi_theap_t* theap = NULL;\n mi_lock(&heap->theaps_lock) { \n theap = heap->theaps; \n while(theap != NULL) {\n mi_theap_t* next = theap->hnext;\n if (!_mi_theap_free(theap, false /* dont re-acquire the heap->theaps_lock */, true /* acquire the tld->theaps_lock though */ )) {\n all_freed = false;\n }\n theap = next;\n } \n }\n if (!all_freed) { \n mi_heap_stat_counter_increase(heap,heaps_delete_wait,1); \n _mi_prim_thread_yield();\n }\n else { \n mi_assert_internal(heap->theaps==NULL); \n } \n }\n while(!all_freed);\n}\n\n// free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed)\nstatic void mi_heap_free(mi_heap_t* heap) {\n mi_assert_internal(heap!=NULL && !_mi_is_heap_main(heap));\n\n // free all arena pages infos\n mi_lock(&heap->arena_pages_lock) {\n for (size_t i = 0; i < MI_MAX_ARENAS; i++) {\n mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);\n if (arena_pages!=NULL) {\n mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL);\n mi_free(arena_pages);\n }\n }\n }\n\n // remove the heap from the subproc\n mi_heap_stats_merge_to_main(heap);\n mi_atomic_decrement_relaxed(&heap->subproc->heap_count);\n mi_subproc_stat_decrease(heap->subproc, heaps, 1);\n mi_lock(&heap->subproc->heaps_lock) {\n if (heap->next!=NULL) { heap->next->prev = heap->prev; }\n if (heap->prev!=NULL) { heap->prev->next = heap->next; }\n else { heap->subproc->heaps = heap->next; }\n }\n\n _mi_thread_local_free(heap->theap);\n mi_lock_done(&heap->theaps_lock);\n mi_lock_done(&heap->os_abandoned_pages_lock);\n mi_lock_done(&heap->arena_pages_lock);\n mi_free(heap);\n}\n\nvoid mi_heap_delete(mi_heap_t* heap) {\n if (heap==NULL) return;\n if (_mi_is_heap_main(heap)) {\n _mi_warning_message(\"cannot delete the main heap\\n\");\n return;\n }\n mi_heap_free_theaps(heap);\n _mi_heap_move_pages(heap, mi_heap_main());\n mi_heap_free(heap);\n}\n\nvoid _mi_heap_force_destroy(mi_heap_t* heap) {\n if (heap==NULL) return;\n mi_heap_free_theaps(heap);\n _mi_heap_destroy_pages(heap);\n if (!_mi_is_heap_main(heap)) { mi_heap_free(heap); } // todo: release locks of the main heap?\n}\n\nvoid mi_heap_destroy(mi_heap_t* heap) {\n if (heap==NULL) return;\n if (_mi_is_heap_main(heap)) {\n _mi_warning_message(\"cannot destroy the main heap\\n\");\n return;\n }\n _mi_heap_force_destroy(heap);\n}\n\nmi_heap_t* mi_heap_of(const void* p) {\n mi_page_t* page = _mi_safe_ptr_page(p);\n if (page==NULL) return NULL;\n return mi_page_heap(page);\n}\n\nbool mi_any_heap_contains(const void* p) {\n return (mi_heap_of(p)!=NULL);\n}\n\nbool mi_heap_contains(const mi_heap_t* heap, const void* p) {\n if (heap==NULL) { heap = mi_heap_main(); }\n return (heap==mi_heap_of(p));\n}\n\n// deprecated\nbool mi_check_owned(const void* p) {\n return mi_any_heap_contains(p);\n}\n\n// unsafe heap utilization function for DragonFly (see issue #1258)\n// If the page of pointer `p` belongs to `heap` (or `heap==NULL`) and has less than `perc_threshold` used blocks in its used area return `true`.\n// This function is unsafe in general as it assumes we are the only thread accessing the page of `p`.\nbool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept {\n if (p==NULL) return false;\n const mi_page_t* const page = _mi_safe_ptr_page(p); // Get the page containing this pointer\n if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false;\n // If the page is the head of the queue, it is currently being used for \n // allocations; we skip it to avoid immediate thrashing.\n if (page->prev == NULL) return false;\n\n // match heap?\n const mi_heap_t* const page_heap = mi_page_heap(page);\n if (page_heap==NULL) return false;\n if (heap!=NULL && page_heap!=heap) return false;\n \n // check utilization\n if (page->capacity==0) return false;\n if (perc_threshold>=100) return true;\n return (perc_threshold >= ((100UL*page->used) / page->capacity));\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/init.c", "sha256": "eca1fe77d72ab2aaccc447c8119056e9c4e8851bda9a95472b573539f76107ad", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\"\n\n#include // memcpy, memset\n#include // atexit\n\n#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ }\n#define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC)\n\n// Empty page used to initialize the small free pages array\nconst mi_page_t _mi_page_empty = {\n MI_ATOMIC_VAR_INIT(0), // xthread_id\n NULL, // free\n 0, // used\n 0, // capacity\n 0, // reserved capacity\n 0, // retire_expire\n false, // is_zero\n NULL, // local_free\n MI_ATOMIC_VAR_INIT(0), // xthread_free\n 0, // block_size\n NULL, // page_start\n #if (MI_PADDING || MI_ENCODE_FREELIST)\n { 0, 0 }, // keys\n #endif\n NULL, // theap\n NULL, // heap\n NULL, NULL, // next, prev\n MI_ARENA_SLICE_SIZE, // page_committed\n MI_MEMID_STATIC // memid\n};\n\n#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty)\n\n#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8)\n#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }\n#elif (MI_PADDING>0)\n#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }\n#else\n#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() }\n#endif\n\n\n// Empty page queues for every bin\n#define QNULL(sz) { NULL, NULL, 0, (sz)*sizeof(uintptr_t) }\n#define MI_PAGE_QUEUES_EMPTY \\\n { QNULL(1), \\\n QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \\\n QNULL( 10), QNULL( 12), QNULL( 14), QNULL( 16), QNULL( 20), QNULL( 24), QNULL( 28), QNULL( 32), /* 16 */ \\\n QNULL( 40), QNULL( 48), QNULL( 56), QNULL( 64), QNULL( 80), QNULL( 96), QNULL( 112), QNULL( 128), /* 24 */ \\\n QNULL( 160), QNULL( 192), QNULL( 224), QNULL( 256), QNULL( 320), QNULL( 384), QNULL( 448), QNULL( 512), /* 32 */ \\\n QNULL( 640), QNULL( 768), QNULL( 896), QNULL( 1024), QNULL( 1280), QNULL( 1536), QNULL( 1792), QNULL( 2048), /* 40 */ \\\n QNULL( 2560), QNULL( 3072), QNULL( 3584), QNULL( 4096), QNULL( 5120), QNULL( 6144), QNULL( 7168), QNULL( 8192), /* 48 */ \\\n QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \\\n QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \\\n QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \\\n QNULL(MI_LARGE_MAX_OBJ_WSIZE + 1 /* 655360, Huge queue */), \\\n QNULL(MI_LARGE_MAX_OBJ_WSIZE + 2) /* Full queue */ }\n\n#define MI_STAT_COUNT_NULL() {0,0,0}\n\n// Empty statistics\n#define MI_STAT_COUNT(stat) {0,0,0},\n#define MI_STAT_COUNTER(stat) {0},\n\n#define MI_STATS_NULL \\\n MI_STAT_FIELDS() \\\n \\\n { MI_INIT4(MI_STAT_COUNT_NULL) }, \\\n { { 0 }, { 0 }, { 0 }, { 0 } }, \\\n \\\n { MI_INIT74(MI_STAT_COUNT_NULL) }, \\\n { MI_INIT74(MI_STAT_COUNT_NULL) }, \\\n { MI_INIT5(MI_STAT_COUNT_NULL) }\n\n// --------------------------------------------------------\n// Statically allocate an empty theap as the initial\n// thread local value for the default theap,\n// and statically allocate the backing theap for the main\n// thread so it can function without doing any allocation\n// itself (as accessing a thread local for the first time\n// may lead to allocation itself on some platforms)\n// --------------------------------------------------------\n\nstatic mi_decl_cache_align mi_subproc_t subproc_main\n#if __cplusplus\n = { }; // empty initializer to prevent running the constructor (with msvc)\n#else\n = { 0 }; // C zero initialize\n#endif\n\nstatic mi_subproc_t* subprocs = &subproc_main;\nstatic mi_lock_t subprocs_lock;\n\nstatic mi_decl_cache_align mi_tld_t tld_empty = {\n 0, // thread_id\n 0, // thread_seq\n 0, // default numa node\n &subproc_main, // subproc\n NULL, // theaps list\n MI_LOCK_INITIALIZER, // theaps lock\n false, // recurse\n false, // is_in_threadpool\n MI_MEMID_STATIC // memid\n};\n\nmi_decl_cache_align const mi_theap_t _mi_theap_empty = {\n &tld_empty, // tld\n MI_ATOMIC_VAR_INIT(NULL), // heap\n MI_ATOMIC_VAR_INIT(1), // refcount\n 0, // heartbeat\n 0, // cookie\n { {0}, {0}, 0, true }, // random\n 0, // page count\n MI_BIN_FULL, 0, // page retired min/max\n 0, // pages_full_size\n 0, 0, // generic count\n NULL, NULL, // tnext, tprev\n NULL, NULL, // hnext, hprev\n 0, // full page retain\n false, // allow reclaim\n true, // allow abandon\n #if MI_GUARDED\n 0, 0, 0, 1, // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`)\n #endif\n MI_SMALL_PAGES_EMPTY,\n MI_PAGE_QUEUES_EMPTY,\n MI_MEMID_STATIC,\n { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats\n};\n\nmi_decl_cache_align const mi_theap_t _mi_theap_empty_wrong = {\n &tld_empty, // tld\n MI_ATOMIC_VAR_INIT(NULL), // heap\n MI_ATOMIC_VAR_INIT(1), // refcount\n 0, // heartbeat\n 0, // cookie\n { {0}, {0}, 0, true }, // random\n 0, // page count\n MI_BIN_FULL, 0, // page retired min/max\n 0, // pages_full_size\n 0, 0, // generic count\n NULL, NULL, // tnext, tprev\n NULL, NULL, // hnext, hprev\n 0, // full page retain\n false, // allow reclaim\n true, // allow abandon\n #if MI_GUARDED\n 0, 0, 0, 1, // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`)\n #endif\n MI_SMALL_PAGES_EMPTY,\n MI_PAGE_QUEUES_EMPTY,\n MI_MEMID_STATIC,\n { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats\n};\n\n// Heap for the main thread\n\nextern mi_decl_hidden mi_decl_cache_align mi_theap_t theap_main;\nextern mi_decl_hidden mi_decl_cache_align mi_heap_t heap_main;\n\nstatic mi_decl_cache_align mi_tld_t tld_main = {\n 0, // thread_id\n 0, // thread_seq\n 0, // numa node\n &subproc_main, // subproc\n &theap_main, // theaps list\n MI_LOCK_INITIALIZER, // theaps lock\n false, // recurse\n false, // is_in_threadpool\n MI_MEMID_STATIC // memid\n};\n\nmi_decl_cache_align mi_theap_t theap_main = {\n &tld_main, // thread local data\n MI_ATOMIC_VAR_INIT(&heap_main), // main heap\n MI_ATOMIC_VAR_INIT(1), // refcount\n 0, // heartbeat\n 0, // initial cookie\n { {0x846ca68b}, {0}, 0, true }, // random\n 0, // page count\n MI_BIN_FULL, 0, // page retired min/max\n 0, // pages_full_size\n 0, 0, // generic count\n NULL, NULL, // tnext, tprev\n NULL, NULL, // hnext, hprev\n 2, // full page retain\n true, // allow page reclaim\n true, // allow page abandon\n #if MI_GUARDED\n 0, 0, 0, 0,\n #endif\n MI_SMALL_PAGES_EMPTY,\n MI_PAGE_QUEUES_EMPTY,\n MI_MEMID_STATIC,\n { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats\n};\n\nmi_decl_cache_align mi_heap_t heap_main\n#if __cplusplus\n = { }; // empty initializer to prevent running the constructor (with msvc)\n#else\n = { 0 }; // C zero initialize\n#endif\n\n// the theap belonging to the main heap\nmi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main = NULL;\n\nmi_threadid_t _mi_thread_id(void) mi_attr_noexcept {\n mi_threadid_t tid = _mi_prim_thread_id();\n mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits\n return tid;\n}\n\n#if MI_TLS_MODEL_THREAD_LOCAL\n// the thread-local main theap for allocation\nmi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty;\n// the last used non-main theap\nmi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty;\n#endif\n\nbool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.\n\nmi_stats_t _mi_stats_main = { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL };\n\n#undef MI_STAT_COUNT\n#undef MI_STAT_COUNTER\n\n\n#if MI_GUARDED\nmi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {\n theap->guarded_sample_rate = sample_rate;\n theap->guarded_sample_count = sample_rate; // count down samples\n if (theap->guarded_sample_rate > 1) {\n if (seed == 0) {\n seed = _mi_theap_random_next(theap);\n }\n theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate`\n }\n}\n\nmi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {\n theap->guarded_size_min = min;\n theap->guarded_size_max = (min > max ? min : max);\n}\n\nvoid _mi_theap_guarded_init(mi_theap_t* theap) {\n mi_theap_guarded_set_sample_rate(theap,\n (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX),\n (size_t)mi_option_get(mi_option_guarded_sample_seed));\n mi_theap_guarded_set_size_bound(theap,\n (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX),\n (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) );\n}\n#else\nmi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {\n MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed);\n}\n\nmi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {\n MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max);\n}\nvoid _mi_theap_guarded_init(mi_theap_t* theap) {\n MI_UNUSED(theap);\n}\n#endif\n\n/* -----------------------------------------------------------\n Initialization\n Note: on some platforms lock_init or just a thread local access\n can cause allocation and induce recursion during initialization.\n----------------------------------------------------------- */\n\n\n// Initialize main subproc\nstatic void mi_subproc_main_init(void) {\n if (subproc_main.memid.memkind != MI_MEM_STATIC) {\n subproc_main.memid = _mi_memid_create(MI_MEM_STATIC);\n subproc_main.heaps = &heap_main;\n subproc_main.heap_total_count = 1;\n subproc_main.heap_count = 1;\n mi_atomic_store_ptr_release(mi_heap_t, &subproc_main.heap_main, &heap_main);\n __mi_stat_increase_mt(&subproc_main.stats.heaps, 1);\n mi_stats_header_init(&subproc_main.stats);\n mi_lock_init(&subproc_main.arena_reserve_lock);\n mi_lock_init(&subproc_main.heaps_lock);\n mi_lock_init(&subprocs_lock);\n mi_lock_init(&tld_empty.theaps_lock); \n }\n}\n\n// Initialize main tld\nstatic void mi_tld_main_init(void) {\n if (tld_main.thread_id == 0) {\n tld_main.thread_id = _mi_prim_thread_id();\n mi_lock_init(&tld_main.theaps_lock);\n }\n}\n\nvoid _mi_theap_options_init(mi_theap_t* theap) {\n theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0);\n theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0);\n theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32);\n}\n\n// Initialization of the (statically allocated) main theap, and the main tld and subproc.\nstatic void mi_theap_main_init(void) {\n if mi_unlikely(theap_main.memid.memkind != MI_MEM_STATIC) {\n // theap\n theap_main.memid = _mi_memid_create(MI_MEM_STATIC);\n #if defined(__APPLE__) || defined(_WIN32) && !defined(MI_SHARED_LIB)\n _mi_random_init_weak(&theap_main.random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)\n #else\n _mi_random_init(&theap_main.random);\n #endif\n theap_main.cookie = _mi_theap_random_next(&theap_main);\n _mi_theap_options_init(&theap_main);\n _mi_theap_guarded_init(&theap_main);\n }\n}\n\n// Initialize main heap\nstatic void mi_heap_main_init(void) {\n if mi_unlikely(heap_main.subproc == NULL) {\n heap_main.subproc = &subproc_main;\n heap_main.theaps = &theap_main;\n\n mi_theap_main_init();\n mi_subproc_main_init();\n mi_tld_main_init();\n\n mi_lock_init(&heap_main.theaps_lock);\n mi_lock_init(&heap_main.os_abandoned_pages_lock);\n mi_lock_init(&heap_main.arena_pages_lock);\n }\n}\n\n\n/* -----------------------------------------------------------\n Thread local data\n----------------------------------------------------------- */\n\n// Allocate fresh tld\nstatic mi_tld_t* mi_tld_alloc(void) {\n if (_mi_is_main_thread()) {\n mi_atomic_increment_relaxed(&tld_main.subproc->thread_count);\n return &tld_main;\n }\n else {\n // allocate tld meta-data\n // note: we need to be careful to not access the tld from `_mi_meta_zalloc`\n // (and in turn from `_mi_arena_alloc_aligned` and `_mi_os_alloc_aligned`).\n mi_memid_t memid;\n mi_tld_t* tld = (mi_tld_t*)_mi_meta_zalloc(sizeof(mi_tld_t), &memid);\n if (tld==NULL) {\n _mi_error_message(ENOMEM, \"unable to allocate memory for thread local data\\n\");\n return NULL;\n }\n tld->memid = memid;\n tld->theaps = NULL;\n mi_lock_init(&tld->theaps_lock);\n tld->subproc = &subproc_main;\n tld->numa_node = _mi_os_numa_node();\n tld->thread_id = _mi_prim_thread_id();\n tld->thread_seq = mi_atomic_increment_relaxed(&tld->subproc->thread_total_count);\n tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool();\n mi_atomic_increment_relaxed(&tld->subproc->thread_count);\n return tld;\n }\n}\n\n#define MI_TLD_INVALID ((mi_tld_t*)1)\n\nmi_decl_noinline static void mi_tld_free(mi_tld_t* tld) {\n mi_lock_done(&tld->theaps_lock);\n if (tld != NULL && tld != MI_TLD_INVALID) {\n mi_atomic_decrement_relaxed(&tld->subproc->thread_count);\n _mi_meta_free(tld, sizeof(mi_tld_t), tld->memid);\n }\n #if 0\n // do not read/write to `thread_tld` on older macOS <= 14 as that will re-initialize the thread local storage\n // (since we are calling this during pthread shutdown)\n // (and this could happen on other systems as well, so let's never do it)\n thread_tld = MI_TLD_INVALID;\n #endif\n}\n\n// return the thread local heap ensuring it is initialized (and not `NULL` or `&_mi_theap_empty`);\nmi_theap_t* _mi_theap_default_safe(void) {\n mi_theap_t* theap = _mi_theap_default();\n if mi_likely(mi_theap_is_initialized(theap)) return theap;\n mi_thread_init();\n mi_assert_internal(mi_theap_is_initialized(_mi_theap_default()));\n return _mi_theap_default();\n}\n\n// return the main theap ensuring it is initialized. \nmi_theap_t* _mi_theap_main_safe(void) {\n mi_theap_t* theap = __mi_theap_main;\n if mi_unlikely(theap==NULL) { // if thread_init or default_set was never called\n mi_thread_init(); // sets the default slot to the main theap\n theap = _mi_theap_default();\n mi_assert_internal(theap!=NULL);\n mi_assert_internal(_mi_is_theap_main(theap));\n if (_mi_is_theap_main(theap)) {\n __mi_theap_main = theap;\n }\n }\n mi_assert_internal(theap!=NULL && _mi_is_theap_main(theap)); \n return theap;\n}\n\n\nmi_subproc_t* _mi_subproc_main(void) {\n return &subproc_main;\n}\n\nmi_subproc_t* _mi_subproc(void) {\n // should work without doing initialization (as it may be called from `_mi_tld -> mi_tld_alloc ... -> os_alloc -> _mi_subproc()`\n // todo: this will still fail on OS systems where the first access to a thread-local causes allocation.\n // on such systems we can check for this with the _mi_prim_get_default_theap as those are protected (by being\n // stored in a TLS slot for example)\n mi_theap_t* theap = _mi_theap_default();\n if (theap == NULL) {\n return _mi_subproc_main();\n }\n else {\n return theap->tld->subproc; // avoid using thread local storage (`thread_tld`)\n }\n}\n\nmi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) {\n mi_heap_t* heap = mi_atomic_load_ptr_relaxed(mi_heap_t,&subproc->heap_main);\n if mi_likely(heap!=NULL) {\n return heap;\n }\n else {\n mi_heap_main_init();\n mi_assert_internal(mi_atomic_load_relaxed(&subproc->heap_main) != NULL);\n return mi_atomic_load_ptr_relaxed(mi_heap_t,&subproc->heap_main);\n }\n}\n\nmi_heap_t* mi_heap_main(void) {\n return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init\n}\n\nbool _mi_is_heap_main(const mi_heap_t* heap) {\n mi_assert_internal(heap!=NULL);\n return (_mi_subproc_heap_main(heap->subproc) == heap);\n}\n\nbool _mi_is_theap_main(const mi_theap_t* theap) {\n return (mi_theap_is_initialized(theap) && _mi_is_heap_main(_mi_theap_heap(theap)));\n}\n\n/* -----------------------------------------------------------\n Sub process\n----------------------------------------------------------- */\n\n\nmi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) {\n return (mi_subproc_t*)(subproc_id._mi_subproc_id);\n}\n\nmi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) {\n mi_subproc_id_t id = { subproc };\n return id;\n}\n\nmi_subproc_id_t mi_subproc_main(void) {\n return _mi_subproc_to_id(_mi_subproc_main());\n}\n\nmi_subproc_id_t mi_subproc_current(void) {\n return _mi_subproc_to_id(_mi_subproc());\n}\n\nmi_subproc_id_t mi_subproc_new(void) {\n static _Atomic(size_t) subproc_total_count;\n mi_memid_t memid;\n mi_subproc_t* subproc = (mi_subproc_t*)_mi_meta_zalloc(sizeof(mi_subproc_t),&memid);\n if (subproc == NULL) return _mi_subproc_to_id(NULL);\n subproc->memid = memid;\n subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count) + 1;\n mi_stats_header_init(&subproc->stats);\n mi_lock_init(&subproc->arena_reserve_lock);\n mi_lock_init(&subproc->heaps_lock);\n mi_lock(&subprocs_lock) {\n // push on subproc list\n subproc->next = subprocs;\n if (subprocs!=NULL) { subprocs->prev = subproc; }\n subprocs = subproc;\n }\n return _mi_subproc_to_id(subproc);\n}\n\n// destroy all subproc resources including arena's, heap's etc.\nstatic void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock)\n{\n if (subproc==NULL) return;\n\n // remove from the subproc list\n mi_lock_maybe(&subprocs_lock, acquire_subprocs_lock) {\n if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; }\n if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; }\n else { mi_assert_internal(subprocs==subproc); subprocs = subproc->next; }\n }\n\n // destroy all subproc heaps\n mi_lock(&subproc->heaps_lock) {\n mi_heap_t* heap = subproc->heaps;\n while (heap != NULL) {\n mi_heap_t* next = heap->next;\n if (heap!=subproc->heap_main) { mi_heap_destroy(heap); }\n heap = next;\n }\n mi_assert_internal(subproc->heaps == subproc->heap_main);\n _mi_heap_force_destroy(subproc->heap_main); // no warning if destroying the main heap\n }\n\n // remove associated arenas\n _mi_arenas_unsafe_destroy_all(subproc);\n\n // merge stats back into the main subproc?\n if (subproc!=&subproc_main) {\n _mi_stats_merge_into(&subproc_main.stats, &subproc->stats);\n }\n\n // safe to release\n // todo: should we refcount subprocesses?\n mi_lock_done(&subproc->arena_reserve_lock);\n mi_lock_done(&subproc->heaps_lock);\n if (subproc!=&subproc_main) {\n _mi_meta_free(subproc, sizeof(mi_subproc_t), subproc->memid);\n }\n else {\n // for the main subproc, also release the global page map\n _mi_page_map_unsafe_destroy(&subproc_main);\n }\n}\n\nvoid mi_subproc_destroy(mi_subproc_id_t subproc_id) {\n mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);\n if (subproc==NULL || subproc==&subproc_main) return;\n mi_subproc_unsafe_destroy(subproc, true /* take lock */);\n}\n\nstatic void mi_subprocs_unsafe_destroy_all(void) {\n mi_lock(&subprocs_lock) {\n mi_subproc_t* subproc = subprocs;\n while (subproc!=NULL) {\n mi_subproc_t* next = subproc->next;\n if (subproc!=&subproc_main) {\n mi_subproc_unsafe_destroy(subproc, false /* take subprocs lock */);\n }\n subproc = next;\n }\n } \n mi_subproc_unsafe_destroy(&subproc_main, true /* take subprocs lock */);\n}\n\n\nvoid mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) {\n mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);\n mi_tld_t* const tld = _mi_theap_default_safe()->tld;\n mi_assert(tld->subproc== &subproc_main);\n if (tld->subproc != &subproc_main) {\n _mi_warning_message(\"unable to add thread to the subprocess as it was already in another subprocess (id: %p)\\n\", subproc);\n return;\n }\n tld->subproc = subproc;\n tld->thread_seq = mi_atomic_increment_relaxed(&subproc->thread_total_count);\n mi_atomic_decrement_relaxed(&subproc_main.thread_count);\n mi_atomic_increment_relaxed(&subproc->thread_count);\n}\n\n\nbool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) {\n mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);\n if (subproc==NULL) return false;\n bool ok = true;\n mi_lock(&subproc->heaps_lock) {\n for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) {\n ok = (*visitor)(heap, arg);\n }\n }\n return ok;\n}\n\n\n/* -----------------------------------------------------------\n Allocate theap data\n----------------------------------------------------------- */\n\n// Initialize the thread local default theap, called from `mi_thread_init`\nstatic mi_theap_t* _mi_thread_init_theap_default(void) {\n mi_theap_t* theap = _mi_theap_default();\n if (mi_theap_is_initialized(theap)) return theap;\n if (_mi_is_main_thread()) {\n mi_heap_main_init();\n theap = &theap_main;\n }\n else {\n // allocates tld data\n // note: we cannot access thread-locals yet as that can cause (recursive) allocation\n // (on macOS <= 14 for example where the loader allocates thread-local data on demand).\n mi_tld_t* tld = mi_tld_alloc();\n if (tld==NULL) return NULL; // things are very wrong if this fails (out of memory)\n // allocate and initialize the theap for the main heap\n theap = _mi_theap_create(mi_heap_main(), tld);\n }\n // associate the theap with this thread\n // (this is safe, on macOS for example, the theap is set in a dedicated TLS slot and thus does not cause recursive allocation)\n _mi_theap_default_set(theap);\n return theap;\n}\n\n\n// Free the thread local theaps\nstatic void mi_thread_theaps_done(mi_tld_t* tld)\n{\n // reset the thread local theaps\n _mi_theap_default_set((mi_theap_t*)&_mi_theap_empty);\n _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty);\n __mi_theap_main = NULL;\n \n // abandon the pages of all theaps in this thread\n mi_lock(&tld->theaps_lock) {\n mi_theap_t* theap = tld->theaps;\n while (theap != NULL) {\n mi_theap_t* next = theap->tnext; \n // never destroy theaps; if a dll is linked statically with mimalloc,\n // there may still be delete/free calls after the mi_fls_done is called. Issue #207\n _mi_theap_collect_abandon(theap);\n mi_assert_internal(theap->page_count==0);\n theap = next;\n }\n }\n\n // free the theaps of this thread.\n // This can run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically.\n // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock \n // potential concurrent `mi_heap_free_theaps` which tries to remove the theap from our theaps list.\n bool all_freed;\n do {\n all_freed = true;\n mi_lock(&tld->theaps_lock) {\n mi_theap_t* theap = tld->theaps;\n while (theap != NULL) {\n mi_theap_t* next = theap->tnext;\n mi_assert_internal(theap->page_count==0);\n if (!_mi_theap_free(theap, true /* acquire heap->theaps_lock */, false /* dont re-acquire the tld->theaps_lock*/ )) {\n all_freed = false;\n }\n theap = next;\n }\n }\n if (!all_freed) { \n mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1); \n _mi_prim_thread_yield(); \n }\n else { \n mi_assert_internal(tld->theaps==NULL); \n }\n } while (!all_freed);\n\n mi_assert(_mi_theap_default()==(mi_theap_t*)&_mi_theap_empty); // careful to not re-initialize the default theap during theap_delete\n mi_assert(!mi_theap_is_initialized(_mi_theap_default()));\n}\n\n\n\n// --------------------------------------------------------\n// Try to run `mi_thread_done()` automatically so any memory\n// owned by the thread but not yet released can be abandoned\n// and re-owned by another thread.\n//\n// 1. windows dynamic library:\n// call from DllMain on DLL_THREAD_DETACH\n// 2. windows static library:\n// use special linker section to call a destructor when the thread is done\n// 3. unix, pthreads:\n// use a pthread key to call a destructor when a pthread is done\n//\n// In the last two cases we also need to call `mi_process_init`\n// to set up the thread local keys.\n// --------------------------------------------------------\n\n// Set up handlers so `mi_thread_done` is called automatically\nstatic void mi_process_setup_auto_thread_done(void) {\n mi_atomic_do_once {\n _mi_prim_thread_init_auto_done();\n _mi_theap_default_set(&theap_main);\n }\n}\n\n\nbool _mi_is_main_thread(void) {\n return (tld_main.thread_id==0 || tld_main.thread_id == _mi_thread_id());\n}\n\n\n// Initialize thread\nvoid mi_thread_init(void) mi_attr_noexcept\n{\n // ensure our process has started already\n mi_process_init();\n // if the theap_default is already set we have already initialized\n if (_mi_thread_is_initialized()) return;\n\n // initialize the default theap\n if (_mi_thread_init_theap_default() == NULL) return; // out-of-memory on tld/theap allocation\n\n mi_heap_stat_increase(mi_heap_main(), threads, 1);\n // _mi_verbose_message(\"thread init: 0x%zx\\n\", _mi_thread_id());\n}\n\nvoid mi_thread_done(void) mi_attr_noexcept {\n _mi_thread_done(NULL);\n}\n\nvoid _mi_thread_done(mi_theap_t* _theap_main)\n{\n // NULL can be passed on some platforms\n if (_theap_main==NULL) {\n _theap_main = __mi_theap_main; // don't call `mi_theap_main_safe` as that re-initializes the thread\n if (_theap_main==NULL) { // can happen if `mi_theap_main_safe` is never called; but then the default is main\n _theap_main = _mi_theap_default();\n mi_assert_internal(_theap_main==NULL || _mi_is_theap_main(_theap_main));\n }\n }\n\n // prevent re-entrancy through theap_done/theap_set_default_direct (issue #699)\n if (!mi_theap_is_initialized(_theap_main)) {\n return;\n }\n\n // release dynamic thread_local's\n _mi_thread_locals_thread_done();\n\n // note: we store the tld as we should avoid reading `thread_tld` at this point (to avoid reinitializing the thread local storage)\n mi_tld_t* const tld = _theap_main->tld;\n\n // adjust stats\n mi_heap_stat_decrease(_mi_subproc_heap_main(tld->subproc), threads, 1); // todo: or `_theap_main->heap`?\n\n // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local theaps...\n if (tld->thread_id != _mi_prim_thread_id()) return;\n\n // delete the thread local theaps\n mi_thread_theaps_done(tld);\n\n // free thread local data\n mi_tld_free(tld);\n}\n\n\nmi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) {\n return (mi_theap_t*)&_mi_theap_empty;\n}\n\n#if MI_TLS_MODEL_DYNAMIC_WIN32\n\n// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed)\n// See for the offsets.\n#if MI_SIZE_SIZE==4\n#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_SIZE_SIZE)\n#else\n#define MI_TLS_DIRECT_FIRST (0x1480 / MI_SIZE_SIZE)\n#endif\n#define MI_TLS_DIRECT_SLOTS (64)\n#define MI_TLS_EXPANSION_SLOTS (1024)\n\n#if !MI_WIN_DIRECT_TLS\n#define MI_TLS_INITIAL_SLOT MI_TLS_EXPANSION_SLOT\n#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1)\n#else\n// with only direct entries, use the \"arbitrary user data\" field \n// and assume it is NULL (see also )\n#define MI_TLS_INITIAL_SLOT (5)\n#define MI_TLS_INITIAL_EXPANSION_SLOT (0)\n#endif\n\n// we initially use the last of the expansion slots as the default NULL.\n// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc (which is quite unlikely)\nmi_decl_hidden mi_decl_cache_align size_t _mi_theap_default_slot = MI_TLS_INITIAL_SLOT;\nmi_decl_hidden size_t _mi_theap_default_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT;\nmi_decl_hidden size_t _mi_theap_cached_slot = MI_TLS_INITIAL_SLOT;\nmi_decl_hidden size_t _mi_theap_cached_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT;\n\nstatic DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES;\nstatic DWORD mi_tls_raw_index_cached = TLS_OUT_OF_INDEXES;\n\nstatic bool mi_win_tls_slot_alloc(size_t* slot, size_t* extended, DWORD* raw_index) {\n const DWORD index = TlsAlloc();\n *raw_index = index;\n if (index==TLS_OUT_OF_INDEXES) {\n *extended = 0;\n *slot = 0;\n return false;\n }\n else if (index= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT);\n if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) {\n mi_prim_tls_slot_set(slot, value);\n }\n else {\n mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS);\n TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value); // use TlsSetValue to initialize the TlsExpansion array if needed\n }\n}\n\n#elif MI_TLS_MODEL_DYNAMIC_PTHREADS\n\n// only for pthreads for now\nmi_decl_hidden pthread_key_t _mi_theap_default_key = 0;\nmi_decl_hidden pthread_key_t _mi_theap_cached_key = 0;\n\n// create a non-zero pthread key\nstatic int mi_pthread_key_create( pthread_key_t* pkey ) {\n pthread_key_t key;\n int err = pthread_key_create(&key, NULL);\n if (err!=0) return err;\n if (key==0) {\n // if we get a zero key, create another one as we use 0 for an invalid key\n pthread_key_t key2;\n err = pthread_key_create(&key2, NULL);\n pthread_key_delete(key); // delete the old key\n if (err!=0) return err;\n key = key2;\n }\n mi_assert_internal(key!=0); \n *pkey = key;\n return 0;\n}\n\nstatic void mi_tls_slots_init(void) {\n mi_atomic_do_once {\n int err = mi_pthread_key_create(&_mi_theap_default_key);\n if (err==0) {\n err = mi_pthread_key_create(&_mi_theap_cached_key);\n }\n if (err!=0) {\n _mi_error_message(EFAULT, \"unable to allocate pthread keys (error %d)\\n\", err);\n }\n }\n}\n\nstatic void mi_tls_slots_done(void) {\n if (_mi_theap_default_key != 0) {\n pthread_key_delete(_mi_theap_default_key);\n _mi_theap_default_key = 0;\n }\n if (_mi_theap_cached_key != 0) {\n pthread_key_delete(_mi_theap_cached_key);\n _mi_theap_cached_key = 0;\n }\n}\n\n#else\n\nstatic void mi_tls_slots_init(void) {\n // nothing\n}\n\nstatic void mi_tls_slots_done(void) {\n // nothing\n}\n\n#endif\n\nvoid _mi_theap_cached_set(mi_theap_t* theap) {\n mi_theap_t* prev = _mi_theap_cached();\n if (prev==theap) return;\n // set\n mi_tls_slots_init();\n #if MI_TLS_MODEL_THREAD_LOCAL\n __mi_theap_cached = theap;\n #elif MI_TLS_MODEL_FIXED_SLOT\n mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_CACHED, theap);\n #elif MI_TLS_MODEL_DYNAMIC_WIN32\n mi_win_tls_slot_set(_mi_theap_cached_slot, _mi_theap_cached_expansion_slot, theap);\n #elif MI_TLS_MODEL_DYNAMIC_PTHREADS\n if (_mi_theap_cached_key!=0) pthread_setspecific(_mi_theap_cached_key, theap);\n #endif\n // update refcounts (so cached theap memory keeps available until no longer cached)\n _mi_theap_incref(theap);\n _mi_theap_decref(prev); \n}\n\nvoid _mi_theap_default_set(mi_theap_t* theap) {\n mi_theap_t* const theap_old = _mi_theap_default();\n mi_assert_internal(theap != NULL);\n mi_assert_internal(theap->tld->thread_id==0 || theap->tld->thread_id==_mi_thread_id());\n mi_tls_slots_init();\n #if MI_TLS_MODEL_THREAD_LOCAL\n __mi_theap_default = theap;\n #elif MI_TLS_MODEL_FIXED_SLOT\n mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_DEFAULT, theap);\n #elif MI_TLS_MODEL_DYNAMIC_WIN32\n mi_win_tls_slot_set(_mi_theap_default_slot, _mi_theap_default_expansion_slot, theap);\n #elif MI_TLS_MODEL_DYNAMIC_PTHREADS\n if (_mi_theap_default_key!=0) pthread_setspecific(_mi_theap_default_key, theap);\n #endif\n\n // set theap main if needed\n if (mi_theap_is_initialized(theap)) {\n // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again)\n _mi_prim_thread_associate_default_theap(theap);\n if (_mi_is_heap_main(_mi_theap_heap(theap))) {\n __mi_theap_main = theap;\n }\n }\n\n // ensure either the default slot contains the main theap, or __mi_theap_main is initialized \n if (mi_theap_is_initialized(theap_old) && _mi_is_heap_main(_mi_theap_heap(theap_old))) {\n __mi_theap_main = theap_old;\n }\n}\n\nvoid mi_thread_set_in_threadpool(void) mi_attr_noexcept {\n mi_theap_t* theap = _mi_theap_default_safe();\n theap->tld->is_in_threadpool = true;\n}\n\n// --------------------------------------------------------\n// Run functions on process init/done, and thread init/done\n// --------------------------------------------------------\nstatic bool os_preloading = true; // true until this module is initialized\n\n// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false.\nbool mi_decl_noinline _mi_preloading(void) {\n return os_preloading;\n}\n\n// Returns true if mimalloc was redirected\nmi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept {\n return _mi_is_redirected();\n}\n\n// Called once by the process loader from `src/prim/prim.c`\nvoid _mi_auto_process_init(void) {\n // mi_heap_main_init();\n // #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD)\n // volatile mi_theap_t* dummy = __mi_theap_default; // access TLS to allocate it before setting tls_initialized to true;\n // if (dummy == NULL) return; // use dummy or otherwise the access may get optimized away (issue #697)\n // #endif\n\n os_preloading = false;\n mi_assert_internal(_mi_is_main_thread());\n\n mi_process_init();\n mi_process_setup_auto_thread_done();\n _mi_thread_locals_init();\n _mi_options_post_init(); // now we can print to stderr\n if (_mi_is_redirected()) _mi_verbose_message(\"malloc is redirected.\\n\");\n\n // show message from the redirector (if present)\n const char* msg = NULL;\n _mi_allocator_init(&msg);\n if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) {\n _mi_fputs(NULL,NULL,NULL,msg);\n }\n\n // reseed random\n _mi_random_reinit_if_weak(&theap_main.random);\n}\n\n// CPU features\nmi_decl_cache_align size_t _mi_cpu_movsb_max = 0; // for size <= max, rep movsb is fast\nmi_decl_cache_align size_t _mi_cpu_stosb_max = 0; // for size <= max, rep stosb is fast\nmi_decl_cache_align bool _mi_cpu_has_popcnt = false;\n\n#if (MI_ARCH_X64 || MI_ARCH_X86)\n#if defined(__GNUC__)\n// #include \nstatic bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {\n // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx)\n // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect).\n uint32_t eax, ebx, ecx, edx;\n __asm __volatile(\"cpuid\" : \"=a\"(eax), \"=b\"(ebx), \"=c\"(ecx), \"=d\"(edx) : \"a\"(level), \"c\"(sublevel) : );\n regs4[0] = eax;\n regs4[1] = ebx;\n regs4[2] = ecx;\n regs4[3] = edx;\n return true;\n}\n\n#elif defined(_MSC_VER)\nstatic bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {\n __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel);\n return true;\n}\n#else\nstatic bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {\n MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel);\n return false;\n}\n#endif\n\nstatic void mi_detect_cpu_features(void) {\n // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))\n // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?)\n // FSRS for fast short rep stosb\n bool amd = false;\n bool fsrm = false;\n // bool erms = false;\n bool fsrs = false;\n uint32_t cpu_info[4];\n if (mi_cpuid(cpu_info, 0, 0)) {\n amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD)\n }\n if (mi_cpuid(cpu_info, 7, 0)) {\n fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see \n // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see \n }\n if (mi_cpuid(cpu_info, 7, 1)) {\n fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see \n }\n if (mi_cpuid(cpu_info, 1, 0)) {\n _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see \n }\n\n if (fsrm) {\n _mi_cpu_movsb_max = 127;\n }\n if (fsrs || (amd && fsrm)) { // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803\n _mi_cpu_stosb_max = 127;\n }\n}\n\n#else\nstatic void mi_detect_cpu_features(void) {\n #if MI_ARCH_ARM64\n _mi_cpu_has_popcnt = true;\n #endif\n}\n#endif\n\n\n// Initialize the process; called by thread_init or the process loader\nstatic void mi_process_init_once(void) mi_attr_noexcept {\n _mi_process_is_initialized = true; \n _mi_verbose_message(\"process init: 0x%zx\\n\", _mi_thread_id());\n\n mi_detect_cpu_features();\n _mi_options_init();\n _mi_stats_init();\n _mi_os_init();\n // the following can potentially allocate (on freeBSD for pthread keys)\n // todo: do 2-phase so we can use stats at first, then later init the keys?\n mi_heap_main_init(); // before page_map_init so stats are working\n _mi_page_map_init(); // todo: this could fail.. should we abort in that case?\n mi_thread_init();\n _mi_process_is_initialized = true;\n\n #if defined(_WIN32) && defined(MI_WIN_USE_FLS)\n // On windows, when building as a static lib the FLS cleanup happens to early for the main thread.\n // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup\n // will not call _mi_thread_done on the (still executing) main thread. See issue #508.\n _mi_prim_thread_associate_default_theap(NULL);\n #endif\n\n // mi_stats_reset(); // only call stat reset *after* thread init (or the theap tld == NULL)\n mi_track_init();\n if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {\n size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024);\n int reserve_at = (int)mi_option_get_clamp(mi_option_reserve_huge_os_pages_at, -1, INT_MAX);\n if (reserve_at != -1) {\n mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500);\n } else {\n mi_reserve_huge_os_pages_interleave(pages, 0, pages*500);\n }\n }\n if (mi_option_is_enabled(mi_option_reserve_os_memory)) {\n long ksize = mi_option_get(mi_option_reserve_os_memory);\n if (ksize > 0) {\n mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true);\n }\n }\n}\n\n// Initialize the process; called by thread_init or the process loader\nvoid mi_process_init(void) mi_attr_noexcept {\n // #if _MSC_VER < 1920\n\t// mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main\n\t// #endif\n mi_atomic_do_once {\n mi_process_init_once();\n }\n}\n\n// Called when the process is done (cdecl as it is used with `at_exit` on some platforms)\nvoid mi_cdecl mi_process_done(void) mi_attr_noexcept {\n // only shutdown if we were initialized\n if (!_mi_process_is_initialized) return;\n // ensure we are called once\n static bool process_done = false;\n if (process_done) return;\n process_done = true;\n\n // free dynamic thread locals (if used at all)\n _mi_thread_locals_done();\n\n // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread\n _mi_prim_thread_done_auto_done();\n\n #ifndef MI_SKIP_COLLECT_ON_EXIT\n #if (MI_DEBUG || !defined(MI_SHARED_LIB))\n // free all memory if possible on process exit. This is not needed for a stand-alone process\n // but should be done if mimalloc is statically linked into another shared library which\n // is repeatedly loaded/unloaded, see issue #281.\n mi_theap_collect(_mi_theap_default(), true /* force */);\n #endif\n #endif\n\n // done with tracking tools\n mi_track_done()\n\n // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free\n // since after process_done there might still be other code running that calls `free` (like at_exit routines,\n // or C-runtime termination code.\n if (mi_option_is_enabled(mi_option_destroy_on_exit)) {\n mi_subprocs_unsafe_destroy_all(); // destroys all subprocs, arenas, and the page_map! \n }\n else {\n mi_heap_stats_merge_to_subproc(mi_heap_main());\n }\n \n // careful now to no longer access any allocator functionality \n if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {\n mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL);\n }\n mi_lock_done(&subprocs_lock);\n mi_tls_slots_done();\n _mi_allocator_done();\n _mi_verbose_message(\"process done: 0x%zx\\n\", tld_main.thread_id);\n os_preloading = true; // don't call the C runtime anymore\n}\n\nvoid mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept {\n if (_mi_option_get_fast(mi_option_destroy_on_exit)>1) return;\n mi_process_done();\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/libc.c", "sha256": "ebd95b475f15d5387e6ef3c88a9cb47b88e64d7e55998ae1dc656614676a443b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// --------------------------------------------------------\n// This module defines various std libc functions to reduce\n// the dependency on libc, and also prevent errors caused\n// by some libc implementations when called before `main`\n// executes (due to malloc redirection)\n// --------------------------------------------------------\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\" // mi_prim_getenv\n\nchar _mi_toupper(char c) {\n if (c >= 'a' && c <= 'z') return (c - 'a' + 'A');\n else return c;\n}\n\nint _mi_strnicmp(const char* s, const char* t, size_t n) {\n mi_assert_internal(s!=NULL && t!=NULL);\n if (n == 0) return 0;\n for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) {\n if (_mi_toupper(*s) != _mi_toupper(*t)) break;\n }\n return (n == 0 ? 0 : *s - *t);\n}\n\nbool _mi_streq(const char* s, const char* t) {\n if (s==NULL && t==NULL) return true;\n if (s==NULL || t==NULL) return false;\n for (; *s != 0 && *t != 0; s++, t++) {\n if (*s != *t) break;\n }\n return (*s == *t);\n}\n\nvoid _mi_strlcpy(char* dest, const char* src, size_t dest_size) {\n if (dest==NULL || src==NULL || dest_size == 0) return;\n // copy until end of src, or when dest is (almost) full\n while (*src != 0 && dest_size > 1) {\n *dest++ = *src++;\n dest_size--;\n }\n // always zero terminate\n *dest = 0;\n}\n\nvoid _mi_strlcat(char* dest, const char* src, size_t dest_size) {\n if (dest==NULL || src==NULL || dest_size == 0) return;\n // find end of string in the dest buffer\n while (*dest != 0 && dest_size > 1) {\n dest++;\n dest_size--;\n }\n // and catenate\n _mi_strlcpy(dest, src, dest_size);\n}\n\nsize_t _mi_strnlen(const char* s, size_t max_len) {\n if (s==NULL) return 0;\n size_t len = 0;\n while(s[len] != 0 && len < max_len) { len++; }\n return len;\n}\n\nsize_t _mi_strlen(const char* s) {\n return _mi_strnlen(s,PTRDIFF_MAX);\n}\n\nchar* _mi_strnstr(char* s, size_t max_len, const char* pat) {\n if (s==NULL) return NULL;\n if (pat==NULL) return s;\n const size_t m = _mi_strnlen(s, max_len);\n const size_t n = _mi_strlen(pat); \n for (size_t start = 0; start + n <= m; start++) {\n size_t i = 0;\n while (itid);\n if mi_likely(once_tid == 1) {\n return false; // already executed\n }\n const mi_threadid_t current_tid = _mi_thread_id();\n if (once_tid == current_tid) {\n return false; // recursive invocation; we need this for process_init for example\n } \n\n mi_lock_acquire(&once->lock);\n uintptr_t expected = 0;\n if (mi_atomic_cas_strong_acq_rel(&once->tid, &expected, current_tid)) { // could use atomic_load/store as well\n return true; // should execute and release\n } \n else {\n mi_lock_release(&once->lock);\n return false; // already another thread entered and released\n }\n}\n\nvoid _mi_atomic_once_release(mi_atomic_once_t* once) {\n if (mi_atomic_load_acquire(&once->tid)>1) { // paranoia\n mi_atomic_store_release(&once->tid,1); // done executing\n mi_lock_release(&once->lock);\n } \n}\n\n\n// --------------------------------------------------------\n// Define our own limited `_mi_vsnprintf` and `_mi_snprintf`\n// This is mostly to avoid calling these when libc is not yet\n// initialized (and to reduce dependencies)\n//\n// format: d i, p x u, s\n// prec: z l ll L\n// width: 10\n// align-left: -\n// fill: 0\n// plus: +\n// --------------------------------------------------------\n\nstatic void mi_outc(char c, char** out, char* end) {\n char* p = *out;\n if (p >= end) return;\n *p = c;\n *out = p + 1;\n}\n\nstatic void mi_outs(const char* s, char** out, char* end) {\n if (s == NULL) return;\n char* p = *out;\n while (*s != 0 && p < end) {\n *p++ = *s++;\n }\n *out = p;\n}\n\nstatic void mi_out_fill(char fill, size_t len, char** out, char* end) {\n char* p = *out;\n for (size_t i = 0; i < len && p < end; i++) {\n *p++ = fill;\n }\n *out = p;\n}\n\nstatic void mi_out_alignright(char fill, char* start, size_t len, size_t extra, char* end) {\n if (len == 0 || extra == 0) return;\n if (start + len + extra >= end) return;\n // move `len` characters to the right (in reverse since it can overlap)\n for (size_t i = 1; i <= len; i++) {\n start[len + extra - i] = start[len - i];\n }\n // and fill the start\n for (size_t i = 0; i < extra; i++) {\n start[i] = fill;\n }\n}\n\n\nstatic void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* end)\n{\n if (x == 0 || base == 0 || base > 16) {\n if (prefix != 0) { mi_outc(prefix, out, end); }\n mi_outc('0',out,end);\n }\n else {\n // output digits in reverse\n char* start = *out;\n while (x > 0) {\n char digit = (char)(x % base);\n mi_outc((digit <= 9 ? '0' + digit : 'A' + digit - 10),out,end);\n x = x / base;\n }\n if (prefix != 0) {\n mi_outc(prefix, out, end);\n }\n size_t len = *out - start;\n // and reverse in-place\n for (size_t i = 0; i < (len / 2); i++) {\n char c = start[len - i - 1];\n start[len - i - 1] = start[i];\n start[i] = c;\n }\n }\n}\n\n\n#define MI_NEXTC() c = *in; if (c==0) break; in++;\n\nint _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {\n if (buf == NULL || bufsize == 0 || fmt == NULL) return 0;\n buf[bufsize - 1] = 0;\n char* const end = buf + (bufsize - 1);\n const char* in = fmt;\n char* out = buf;\n while (true) {\n if (out >= end) break;\n char c;\n MI_NEXTC();\n if (c != '%') {\n if (c == '\\\\') {\n MI_NEXTC();\n switch (c) {\n case 'e': mi_outc('\\x1B', &out, end); break;\n case 't': mi_outc('\\t', &out, end); break;\n case 'n': mi_outc('\\n', &out, end); break;\n case 'r': mi_outc('\\r', &out, end); break;\n case '\\\\': mi_outc('\\\\', &out, end); break;\n default: /* ignore */ break;\n }\n }\n else if ((c >= ' ' && c <= '~') || c=='\\n' || c=='\\r' || c=='\\t' || c=='\\x1b') { // output visible ascii or standard control only\n mi_outc(c, &out, end);\n }\n }\n else {\n MI_NEXTC();\n char fill = ' ';\n size_t width = 0;\n char numtype = 'd';\n char numplus = 0;\n bool alignright = true;\n if (c == '+' || c == ' ') { numplus = c; MI_NEXTC(); }\n if (c == '-') { alignright = false; MI_NEXTC(); }\n if (c == '0') { fill = '0'; MI_NEXTC(); }\n if (c >= '1' && c <= '9') {\n width = (c - '0'); MI_NEXTC();\n while (c >= '0' && c <= '9') {\n width = (10 * width) + (c - '0'); MI_NEXTC();\n }\n if (c == 0) break; // extra check due to while\n }\n if (c == 'z' || c == 't' || c == 'L') { numtype = c; MI_NEXTC(); }\n else if (c == 'l') {\n numtype = c; MI_NEXTC();\n if (c == 'l') { numtype = 'L'; MI_NEXTC(); }\n }\n\n char* start = out;\n if (c == '%') {\n mi_outc('%', &out, end);\n }\n else if (c == 's') {\n // string\n const char* s = va_arg(args, const char*);\n mi_outs(s, &out, end);\n }\n else if (c == 'p' || c == 'x' || c == 'u') {\n // unsigned\n uintmax_t x = 0;\n if (c == 'x' || c == 'u') {\n if (numtype == 'z') x = va_arg(args, size_t);\n else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t\n else if (numtype == 'L') x = va_arg(args, unsigned long long);\n else if (numtype == 'l') x = va_arg(args, unsigned long);\n else x = va_arg(args, unsigned int);\n }\n else if (c == 'p') {\n void* const p = va_arg(args, void*);\n x = (uintptr_t)p;\n mi_outs(\"0x\", &out, end);\n start = out;\n width = (width >= 2 ? width - 2 : 0);\n }\n if (width == 0 && (c == 'x' || c == 'p')) {\n if (c == 'p') { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); }\n if (width == 0) { width = 2; }\n fill = '0';\n }\n mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end);\n }\n else if (c == 'i' || c == 'd') {\n // signed\n intmax_t x = 0;\n if (numtype == 'z') x = va_arg(args, intptr_t );\n else if (numtype == 't') x = va_arg(args, ptrdiff_t);\n else if (numtype == 'L') x = va_arg(args, long long);\n else if (numtype == 'l') x = va_arg(args, long);\n else x = va_arg(args, int);\n char pre = 0;\n if (x < 0) {\n pre = '-';\n if (x > INTMAX_MIN) { x = -x; }\n }\n else if (numplus != 0) {\n pre = numplus;\n }\n mi_out_num((uintmax_t)x, 10, pre, &out, end);\n }\n else if (c >= ' ' && c <= '~') {\n // unknown format\n mi_outc('%', &out, end);\n mi_outc(c, &out, end);\n }\n\n // fill & align\n mi_assert_internal(out <= end);\n mi_assert_internal(out >= start);\n const size_t len = out - start;\n if (len < width) {\n mi_out_fill(fill, width - len, &out, end);\n if (alignright && out <= end) {\n mi_out_alignright(fill, start, len, width - len, end);\n }\n }\n }\n }\n mi_assert_internal(out <= end);\n *out = 0;\n return (int)(out - buf);\n}\n\nint _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) {\n va_list args;\n va_start(args, fmt);\n const int written = _mi_vsnprintf(buf, buflen, fmt, args);\n va_end(args);\n return written;\n}\n\n\n\n// --------------------------------------------------------\n// generic trailing and leading zero count, and popcount\n// --------------------------------------------------------\n\n#if !MI_HAS_FAST_BITSCAN\n\nstatic size_t mi_ctz_generic32(uint32_t x) {\n // de Bruijn multiplication, see \n static const uint8_t debruijn[32] = {\n 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,\n 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9\n };\n if (x==0) return 32;\n return debruijn[(uint32_t)((x & -(int32_t)x) * (uint32_t)(0x077CB531U)) >> 27];\n}\n\nstatic size_t mi_clz_generic32(uint32_t x) {\n // de Bruijn multiplication, see \n static const uint8_t debruijn[32] = {\n 31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1,\n 23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0\n };\n if (x==0) return 32;\n x |= x >> 1;\n x |= x >> 2;\n x |= x >> 4;\n x |= x >> 8;\n x |= x >> 16;\n return debruijn[(uint32_t)(x * (uint32_t)(0x07C4ACDDU)) >> 27];\n}\n\nsize_t _mi_ctz_generic(size_t x) {\n if (x==0) return MI_SIZE_BITS;\n #if (MI_SIZE_BITS <= 32)\n return mi_ctz_generic32((uint32_t)x);\n #else\n const uint32_t lo = (uint32_t)x;\n if (lo != 0) {\n return mi_ctz_generic32(lo);\n }\n else {\n return (32 + mi_ctz_generic32((uint32_t)(x>>32)));\n }\n #endif\n}\n\nsize_t _mi_clz_generic(size_t x) {\n if (x==0) return MI_SIZE_BITS;\n #if (MI_SIZE_BITS <= 32)\n return mi_clz_generic32((uint32_t)x);\n #else\n const uint32_t hi = (uint32_t)(x>>32);\n if (hi != 0) {\n return mi_clz_generic32(hi);\n }\n else {\n return 32 + mi_clz_generic32((uint32_t)x);\n }\n #endif\n}\n\n#endif // bit scan\n\n\n#if MI_SIZE_SIZE == 4\n#define mi_mask_even_bits32 (0x55555555)\n#define mi_mask_even_pairs32 (0x33333333)\n#define mi_mask_even_nibbles32 (0x0F0F0F0F)\n\n// sum of all the bytes in `x` if it is guaranteed that the sum < 256!\nstatic size_t mi_byte_sum32(uint32_t x) {\n // perform `x * 0x01010101`: the highest byte contains the sum of all bytes.\n x += (x << 8);\n x += (x << 16);\n return (size_t)(x >> 24);\n}\n\nstatic size_t mi_popcount_generic32(uint32_t x) {\n // first count each 2-bit group `a`, where: a==0b00 -> 00, a==0b01 -> 01, a==0b10 -> 01, a==0b11 -> 10\n // in other words, `a - (a>>1)`; to do this in parallel, we need to mask to prevent spilling a bit pair\n // into the lower bit-pair:\n x = x - ((x >> 1) & mi_mask_even_bits32);\n // add the 2-bit pair results\n x = (x & mi_mask_even_pairs32) + ((x >> 2) & mi_mask_even_pairs32);\n // add the 4-bit nibble results\n x = (x + (x >> 4)) & mi_mask_even_nibbles32;\n // each byte now has a count of its bits, we can sum them now:\n return mi_byte_sum32(x);\n}\n\nmi_decl_noinline size_t _mi_popcount_generic(size_t x) {\n if (x<=1) return x;\n if (~x==0) return MI_SIZE_BITS;\n return mi_popcount_generic32(x);\n}\n\n#else\n#define mi_mask_even_bits64 (0x5555555555555555)\n#define mi_mask_even_pairs64 (0x3333333333333333)\n#define mi_mask_even_nibbles64 (0x0F0F0F0F0F0F0F0F)\n\n// sum of all the bytes in `x` if it is guaranteed that the sum < 256!\nstatic size_t mi_byte_sum64(uint64_t x) {\n x += (x << 8);\n x += (x << 16);\n x += (x << 32);\n return (size_t)(x >> 56);\n}\n\nstatic size_t mi_popcount_generic64(uint64_t x) {\n x = x - ((x >> 1) & mi_mask_even_bits64);\n x = (x & mi_mask_even_pairs64) + ((x >> 2) & mi_mask_even_pairs64);\n x = (x + (x >> 4)) & mi_mask_even_nibbles64;\n return mi_byte_sum64(x);\n}\n\nmi_decl_noinline size_t _mi_popcount_generic(size_t x) {\n if (x<=1) return x;\n if (~x==0) return MI_SIZE_BITS;\n return mi_popcount_generic64(x);\n}\n#endif\n\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/options.c", "sha256": "96ef01e41f4b3043118e8c801e660b1d3f185d6fb824e5298a038aee89b6a392", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\" // mi_prim_out_stderr\n\n#include // stdin/stdout\n#include // abort\n\nstatic long mi_max_error_count = 16; // stop outputting errors after this (use < 0 for no limit)\nstatic long mi_max_warning_count = 16; // stop outputting warnings after this (use < 0 for no limit)\n\nstatic void mi_add_stderr_output(void);\n\nint mi_version(void) mi_attr_noexcept {\n return MI_MALLOC_VERSION;\n}\n\n\n// --------------------------------------------------------\n// Options\n// These can be accessed by multiple threads and may be\n// concurrently initialized, but an initializing data race\n// is ok since they resolve to the same value.\n// --------------------------------------------------------\n\n\n#define MI_OPTION(opt) mi_option_##opt, #opt, NULL\n#define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy\n\n// Some options can be set at build time for statically linked libraries\n// (use `-DMI_EXTRA_CPPDEFS=\"opt1=val1;opt2=val2\"`)\n//\n// This is useful if we cannot pass them as environment variables\n// (and setting them programmatically would be too late)\n\n#ifndef MI_DEFAULT_VERBOSE\n#define MI_DEFAULT_VERBOSE 0\n#endif\n\n#ifndef MI_DEFAULT_ARENA_EAGER_COMMIT\n#define MI_DEFAULT_ARENA_EAGER_COMMIT 2\n#endif\n\n// in KiB\n#ifndef MI_DEFAULT_ARENA_RESERVE\n #if (MI_INTPTR_SIZE>4)\n #define MI_DEFAULT_ARENA_RESERVE 1024L*1024L\n #else\n #define MI_DEFAULT_ARENA_RESERVE 128L*1024L\n #endif\n#endif\n\n#ifndef MI_DEFAULT_ARENA_MAX_OBJECT_SIZE\n#define MI_DEFAULT_ARENA_MAX_OBJECT_SIZE ((MI_SIZE_BITS * MI_ARENA_MAX_CHUNK_OBJ_SIZE)/MI_KiB) /* 2 GiB (or 256 MiB on 32-bit), larger than this is alloc'd by the OS */\n#endif\n\n#ifndef MI_DEFAULT_DISALLOW_ARENA_ALLOC\n#define MI_DEFAULT_DISALLOW_ARENA_ALLOC 0\n#endif\n\n#ifndef MI_DEFAULT_ALLOW_LARGE_OS_PAGES\n#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 0\n#endif\n\n#ifndef MI_DEFAULT_RESERVE_HUGE_OS_PAGES\n#define MI_DEFAULT_RESERVE_HUGE_OS_PAGES 0\n#endif\n\n#ifndef MI_DEFAULT_RESERVE_OS_MEMORY\n#define MI_DEFAULT_RESERVE_OS_MEMORY 0\n#endif\n\n#ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE\n#if MI_GUARDED && !MI_DEBUG\n#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000\n#else\n#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0\n#endif\n#endif\n\n#ifndef MI_DEFAULT_PAGEMAP_COMMIT\n#if defined(__APPLE__) // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access\n#define MI_DEFAULT_PAGEMAP_COMMIT 1\n#else\n#define MI_DEFAULT_PAGEMAP_COMMIT 0\n#endif\n#endif\n\n#ifndef MI_DEFAULT_PAGE_MAX_RECLAIM\n#define MI_DEFAULT_PAGE_MAX_RECLAIM (-1) // unlimited\n#endif\n\n#ifndef MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM\n#define MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM 32\n#endif\n\n#ifndef MI_DEFAULT_ALLOW_THP\n#if defined(__ANDROID__)\n#define MI_DEFAULT_ALLOW_THP 0\n#else\n#define MI_DEFAULT_ALLOW_THP 1\n#endif\n#endif\n\n// Static options\nstatic mi_option_desc_t mi_options[_mi_option_last] =\n{\n // stable options\n#if MI_DEBUG || defined(MI_SHOW_ERRORS)\n { 1, MI_OPTION_UNINIT, MI_OPTION(show_errors) },\n#else\n { 0, MI_OPTION_UNINIT, MI_OPTION(show_errors) },\n#endif\n { 0, MI_OPTION_UNINIT, MI_OPTION(show_stats) },\n { MI_DEFAULT_VERBOSE, MI_OPTION_UNINIT, MI_OPTION(verbose) },\n\n // some of the following options are experimental and not all combinations are allowed.\n { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit) }, \n { MI_DEFAULT_ARENA_EAGER_COMMIT,\n MI_OPTION_UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux)\n { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit)\n { MI_DEFAULT_ALLOW_LARGE_OS_PAGES,\n MI_OPTION_UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's\n { MI_DEFAULT_RESERVE_HUGE_OS_PAGES,\n MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages\n {-1, MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N\n { MI_DEFAULT_RESERVE_OS_MEMORY,\n MI_OPTION_UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`)\n { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread\n { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free\n { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_abandoned_page_purge) }, \n { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_reset) }, // reset segment memory on free (needs eager commit)\n { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit_delay) }, \n { 1000,MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds\n { 0, MI_OPTION_UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.\n { 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas)\n { 100, MI_OPTION_UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose\n { 32, MI_OPTION_UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output\n { 32, MI_OPTION_UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output\n { 10, MI_OPTION_UNINIT, MI_OPTION(deprecated_max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try.\n { 0, MI_OPTION_UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees!\n { MI_DEFAULT_ARENA_RESERVE, MI_OPTION_UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)\n { 1, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's\n { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) },\n { MI_DEFAULT_DISALLOW_ARENA_ALLOC, MI_OPTION_UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's)\n { 400, MI_OPTION_UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries.\n#if defined(MI_VISIT_ABANDONED)\n { 1, MI_OPTION_INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting theap blocks in abandoned segments; requires taking locks during reclaim.\n#else\n { 0, MI_OPTION_UNINIT, MI_OPTION(visit_abandoned) },\n#endif\n { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects\n { MI_GiB, MI_OPTION_UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects\n { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)\n { MI_DEFAULT_GUARDED_SAMPLE_RATE,\n MI_OPTION_UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded (=4000)\n { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_sample_seed)},\n { 10000, MI_OPTION_UNINIT, MI_OPTION(generic_collect) }, // collect theaps every N (=10000) generic allocation calls\n { 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(page_reclaim_on_free, abandoned_reclaim_on_free) },// reclaim abandoned (small) pages on a free: -1 = disable completely, 0 = only reclaim into the originating theap, 1 = reclaim on free across theaps\n { 2, MI_OPTION_UNINIT, MI_OPTION(page_full_retain) }, // number of (small) pages to retain in the free page queues\n { 4, MI_OPTION_UNINIT, MI_OPTION(page_max_candidates) }, // max search to find a best page candidate\n { 0, MI_OPTION_UNINIT, MI_OPTION(max_vabits) }, // max virtual address space bits\n { MI_DEFAULT_PAGEMAP_COMMIT,\n MI_OPTION_UNINIT, MI_OPTION(pagemap_commit) }, // commit the full pagemap upfront?\n { 0, MI_OPTION_UNINIT, MI_OPTION(page_commit_on_demand) }, // commit pages on-demand (2 disables this only on overcommit systems (like Linux))\n { MI_DEFAULT_PAGE_MAX_RECLAIM,\n MI_OPTION_UNINIT, MI_OPTION(page_max_reclaim) }, // don't reclaim (small) pages of the same originating theap if we already own N pages in that size class\n { MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM,\n MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class\n { MI_DEFAULT_ALLOW_THP,\n MI_OPTION_UNINIT, MI_OPTION(allow_thp) }, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.\n { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if `mi_option_allow_thp==2`).\n { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE, \n MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). \n { 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation\n};\n\nstatic void mi_option_init(mi_option_desc_t* desc);\n\nstatic bool mi_option_has_size_in_kib(mi_option_t option) {\n return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve || \n option == mi_option_minimal_purge_size || option == mi_option_arena_max_object_size);\n}\n\nvoid _mi_options_init(void) {\n // called on process load\n for(int i = 0; i < _mi_option_last; i++ ) {\n mi_option_t option = (mi_option_t)i;\n long l = mi_option_get(option); MI_UNUSED(l); // initialize\n }\n mi_max_error_count = mi_option_get(mi_option_max_errors);\n mi_max_warning_count = mi_option_get(mi_option_max_warnings);\n #if MI_GUARDED\n if (mi_option_get(mi_option_guarded_sample_rate) > 0) {\n if (mi_option_is_enabled(mi_option_allow_large_os_pages)) {\n mi_option_disable(mi_option_allow_large_os_pages);\n _mi_warning_message(\"option 'allow_large_os_pages' is disabled to allow for guarded objects\\n\");\n }\n }\n #endif \n}\n\n// called at actual process load, it should be safe to print now\nvoid _mi_options_post_init(void) {\n mi_add_stderr_output(); // now it safe to use stderr for output\n if (mi_option_is_enabled(mi_option_verbose)) { mi_options_print(); }\n}\n\n#define mi_stringifyx(str) #str // and stringify\n#define mi_stringify(str) mi_stringifyx(str) // expand\n\nmi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept\n{\n // show version\n const int vermajor = MI_MALLOC_VERSION/10000;\n const int verminor = (MI_MALLOC_VERSION%10000)/100;\n const int verpatch = (MI_MALLOC_VERSION%100);\n _mi_fprintf(out, arg, \"v%i.%i.%i%s%s (built on %s, %s)\\n\", vermajor, verminor, verpatch,\n #if defined(MI_CMAKE_BUILD_TYPE)\n \", \" mi_stringify(MI_CMAKE_BUILD_TYPE)\n #else\n \"\"\n #endif\n ,\n #if defined(MI_GIT_DESCRIBE)\n \", git \" mi_stringify(MI_GIT_DESCRIBE)\n #else\n \"\"\n #endif\n , __DATE__, __TIME__);\n\n // show options\n for (int i = 0; i < _mi_option_last; i++) {\n mi_option_t option = (mi_option_t)i;\n long l = mi_option_get(option); MI_UNUSED(l); // possibly initialize\n mi_option_desc_t* desc = &mi_options[option];\n _mi_fprintf(out, arg, \"option '%s': %ld %s\\n\", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? \"KiB\" : \"\"));\n }\n\n // show build configuration\n _mi_fprintf(out, arg, \"debug level : %d\\n\", MI_DEBUG );\n _mi_fprintf(out, arg, \"secure level: %d\\n\", MI_SECURE );\n _mi_fprintf(out, arg, \"mem tracking: %s\\n\", MI_TRACK_TOOL);\n #if MI_GUARDED\n _mi_fprintf(out, arg, \"guarded build: %s\\n\", mi_option_get(mi_option_guarded_sample_rate) != 0 ? \"enabled\" : \"disabled\");\n #endif\n #if MI_TSAN\n _mi_fprintf(out, arg, \"thread santizer enabled\\n\");\n #endif\n}\n\nmi_decl_export void mi_options_print(void) mi_attr_noexcept {\n mi_options_print_out(NULL, NULL);\n}\n\nlong _mi_option_get_fast(mi_option_t option) {\n mi_assert(option >= 0 && option < _mi_option_last);\n mi_option_desc_t* desc = &mi_options[option];\n mi_assert(desc->option == option); // index should match the option\n //mi_assert(desc->init != MI_OPTION_UNINIT);\n return desc->value;\n}\n\n\nmi_decl_nodiscard long mi_option_get(mi_option_t option) {\n mi_assert(option >= 0 && option < _mi_option_last);\n if (option < 0 || option >= _mi_option_last) return 0;\n mi_option_desc_t* desc = &mi_options[option];\n mi_assert(desc->option == option); // index should match the option\n if mi_unlikely(desc->init == MI_OPTION_UNINIT) {\n mi_option_init(desc);\n }\n return desc->value;\n}\n\nmi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long max) {\n long x = mi_option_get(option);\n return (x < min ? min : (x > max ? max : x));\n}\n\nmi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) {\n const long x = mi_option_get(option);\n size_t size = (x < 0 ? 0 : (size_t)x);\n if (mi_option_has_size_in_kib(option)) {\n if (mi_mul_overflow(size, MI_KiB, &size)) {\n size = MI_MAX_ALLOC_SIZE;\n }\n }\n return size;\n}\n\nvoid mi_option_set(mi_option_t option, long value) {\n mi_assert(option >= 0 && option < _mi_option_last);\n if (option < 0 || option >= _mi_option_last) return;\n mi_option_desc_t* desc = &mi_options[option];\n mi_assert(desc->option == option); // index should match the option\n desc->value = value;\n desc->init = MI_OPTION_INITIALIZED;\n // ensure min/max range; be careful to not recurse.\n if (desc->option == mi_option_guarded_min && _mi_option_get_fast(mi_option_guarded_max) < value) {\n mi_option_set(mi_option_guarded_max, value);\n }\n else if (desc->option == mi_option_guarded_max && _mi_option_get_fast(mi_option_guarded_min) > value) {\n mi_option_set(mi_option_guarded_min, value);\n }\n}\n\nvoid mi_option_set_default(mi_option_t option, long value) {\n mi_assert(option >= 0 && option < _mi_option_last);\n if (option < 0 || option >= _mi_option_last) return;\n mi_option_desc_t* desc = &mi_options[option];\n if (desc->init != MI_OPTION_INITIALIZED) {\n desc->value = value;\n }\n}\n\nmi_decl_nodiscard bool mi_option_is_enabled(mi_option_t option) {\n return (mi_option_get(option) != 0);\n}\n\nvoid mi_option_set_enabled(mi_option_t option, bool enable) {\n mi_option_set(option, (enable ? 1 : 0));\n}\n\nvoid mi_option_set_enabled_default(mi_option_t option, bool enable) {\n mi_option_set_default(option, (enable ? 1 : 0));\n}\n\nvoid mi_option_enable(mi_option_t option) {\n mi_option_set_enabled(option,true);\n}\n\nvoid mi_option_disable(mi_option_t option) {\n mi_option_set_enabled(option,false);\n}\n\nstatic void mi_cdecl mi_out_stderr(const char* msg, void* arg) {\n MI_UNUSED(arg);\n if (msg != NULL && msg[0] != 0) {\n _mi_prim_out_stderr(msg);\n }\n}\n\n// Since an output function can be registered earliest in the `main`\n// function we also buffer output that happens earlier. When\n// an output function is registered it is called immediately with\n// the output up to that point.\n#ifndef MI_MAX_DELAY_OUTPUT\n#define MI_MAX_DELAY_OUTPUT ((size_t)(16*1024))\n#endif\nstatic char out_buf[MI_MAX_DELAY_OUTPUT+1];\nstatic _Atomic(size_t) out_len;\nstatic mi_lock_t out_buf_lock = MI_LOCK_INITIALIZER;\n\nstatic void mi_cdecl mi_out_buf(const char* msg, void* arg) {\n MI_UNUSED(arg);\n if (msg==NULL) return;\n if (mi_atomic_load_acquire(&out_len)>=MI_MAX_DELAY_OUTPUT) return;\n size_t n = _mi_strlen(msg);\n if (n==0 || n >= MI_MAX_DELAY_OUTPUT) return;\n // copy msg into the buffer\n mi_lock(&out_buf_lock) {\n const size_t start = mi_atomic_add_acq_rel(&out_len, n);\n if (start < MI_MAX_DELAY_OUTPUT) {\n // check bound\n if (start+n >= MI_MAX_DELAY_OUTPUT) {\n n = MI_MAX_DELAY_OUTPUT-start-1;\n }\n _mi_memcpy(&out_buf[start], msg, n);\n }\n }\n}\n\nstatic void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) {\n if (out==NULL) return;\n // claim (if `no_more_buf == true`, no more output will be added after this point)\n mi_lock(&out_buf_lock) {\n size_t count = mi_atomic_add_acq_rel(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1));\n // and output the current contents\n if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT;\n out_buf[count] = 0;\n out(out_buf,arg);\n if (!no_more_buf) {\n out_buf[count] = '\\n'; // if continue with the buffer, insert a newline\n }\n }\n}\n\n\n// Once this module is loaded, switch to this routine\n// which outputs to stderr and the delayed output buffer.\nstatic void mi_cdecl mi_out_buf_stderr(const char* msg, void* arg) {\n mi_out_stderr(msg,arg);\n mi_out_buf(msg,arg);\n}\n\n\n\n// --------------------------------------------------------\n// Default output handler\n// --------------------------------------------------------\n\n// The program should only install a single output handler from a single thread\n// since otherwise the argument and output function may not match.\nstatic _Atomic(void*) mi_out_default; // = // is `mi_output_fun*` (but some platforms don't support atomic function pointers)\nstatic _Atomic(void*) mi_out_arg; // = NULL\n\nstatic mi_output_fun* mi_out_get_default(void** parg) {\n mi_output_fun* const out = (mi_output_fun*)mi_atomic_load_ptr_acquire(void,&mi_out_default);\n if (parg != NULL) { *parg = mi_atomic_load_ptr_acquire(void,&mi_out_arg); }\n return (out == NULL ? &mi_out_buf : out);\n}\n\nvoid mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept {\n mi_atomic_store_ptr_release(void,&mi_out_default, (void*)(out == NULL ? &mi_out_stderr : out)); // stop using the delayed output buffer\n mi_atomic_store_ptr_release(void,&mi_out_arg, arg);\n if (out!=NULL) { mi_out_buf_flush(out,true,arg); } // output all the delayed output now\n}\n\n// add stderr to the delayed output after the module is loaded\nstatic void mi_add_stderr_output(void) {\n mi_assert_internal(mi_out_default == NULL);\n mi_out_buf_flush(&mi_out_stderr, false, NULL); // flush current contents to stderr\n mi_atomic_store_ptr_release(void,&mi_out_default,(void*)&mi_out_buf_stderr); // and add stderr to the delayed output\n mi_atomic_store_ptr_release(void,&mi_out_arg,NULL);\n}\n\n// --------------------------------------------------------\n// Messages, all end up calling `_mi_fputs`.\n// --------------------------------------------------------\nstatic _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors\nstatic _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings\n\n// When overriding malloc, we may recurse into mi_vfprintf if an allocation\n// inside the C runtime causes another message.\n// In some cases (like on macOS) the loader already allocates which\n// calls into mimalloc; if we then access thread locals (like `recurse`)\n// this may crash as the access may call _tlv_bootstrap that tries to\n// (recursively) invoke malloc again to allocate space for the thread local\n// variables on demand. This is why we use a _mi_preloading test on such\n// platforms. However, C code generator may move the initial thread local address\n// load before the `if` and we therefore split it out in a separate function.\nstatic mi_decl_thread bool recurse = false;\n\nstatic mi_decl_noinline bool mi_recurse_enter_prim(void) {\n if (recurse) return false;\n recurse = true;\n return true;\n}\n\nstatic mi_decl_noinline void mi_recurse_exit_prim(void) {\n recurse = false;\n}\n\nstatic bool mi_recurse_enter(void) {\n #if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD)\n if (_mi_preloading()) return false;\n #endif\n return mi_recurse_enter_prim();\n}\n\nstatic void mi_recurse_exit(void) {\n #if defined(__APPLE__) || defined(__ANDROID__) || defined(MI_TLS_RECURSE_GUARD)\n if (_mi_preloading()) return;\n #endif\n mi_recurse_exit_prim();\n}\n\nvoid _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) {\n if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // TODO: use mi_out_stderr for stderr?\n if (!mi_recurse_enter()) return;\n out = mi_out_get_default(&arg);\n if (prefix != NULL) out(prefix, arg);\n out(message, arg);\n mi_recurse_exit();\n }\n else {\n if (prefix != NULL) out(prefix, arg);\n out(message, arg);\n }\n}\n\n// Define our own limited `fprintf` that avoids memory allocation.\n// We do this using `_mi_vsnprintf` with a limited buffer.\nstatic void mi_vfprintf( mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args ) {\n char buf[992];\n if (fmt==NULL) return;\n if (!mi_recurse_enter()) return;\n _mi_vsnprintf(buf, sizeof(buf)-1, fmt, args);\n mi_recurse_exit();\n _mi_fputs(out,arg,prefix,buf);\n}\n\nvoid _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) {\n va_list args;\n va_start(args,fmt);\n mi_vfprintf(out,arg,NULL,fmt,args);\n va_end(args);\n}\n\nstatic void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) {\n if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) {\n char tprefix[64];\n _mi_snprintf(tprefix, sizeof(tprefix), \"%sthread 0x%tx: \", prefix, (uintptr_t)_mi_thread_id());\n mi_vfprintf(out, arg, tprefix, fmt, args);\n }\n else {\n mi_vfprintf(out, arg, prefix, fmt, args);\n }\n}\n\nvoid _mi_raw_message(const char* fmt, ...) {\n va_list args;\n va_start(args, fmt);\n mi_vfprintf(NULL, NULL, NULL, fmt, args);\n va_end(args);\n}\n\nvoid _mi_message(const char* fmt, ...) {\n va_list args;\n va_start(args, fmt);\n mi_vfprintf_thread(NULL, NULL, \"mimalloc: \", fmt, args);\n va_end(args);\n}\n\nvoid _mi_trace_message(const char* fmt, ...) {\n if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher\n va_list args;\n va_start(args, fmt);\n mi_vfprintf_thread(NULL, NULL, \"mimalloc: \", fmt, args);\n va_end(args);\n}\n\nvoid _mi_verbose_message(const char* fmt, ...) {\n if (!mi_option_is_enabled(mi_option_verbose)) return;\n va_list args;\n va_start(args,fmt);\n mi_vfprintf(NULL, NULL, \"mimalloc: \", fmt, args);\n va_end(args);\n}\n\nstatic void mi_show_error_message(const char* fmt, va_list args) {\n if (!mi_option_is_enabled(mi_option_verbose)) {\n if (!mi_option_is_enabled(mi_option_show_errors)) return;\n if (mi_max_error_count >= 0 && (long)mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return;\n }\n mi_vfprintf_thread(NULL, NULL, \"mimalloc: error: \", fmt, args);\n}\n\nvoid _mi_warning_message(const char* fmt, ...) {\n if (!mi_option_is_enabled(mi_option_verbose)) {\n if (!mi_option_is_enabled(mi_option_show_errors)) return;\n if (mi_max_warning_count >= 0 && (long)mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return;\n }\n va_list args;\n va_start(args,fmt);\n mi_vfprintf_thread(NULL, NULL, \"mimalloc: warning: \", fmt, args);\n va_end(args);\n}\n\n\n#if MI_DEBUG\nmi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) mi_attr_noexcept {\n _mi_fprintf(NULL, NULL, \"mimalloc: assertion failed: at \\\"%s\\\":%u, %s\\n assertion: \\\"%s\\\"\\n\", fname, line, (func==NULL?\"\":func), assertion);\n abort();\n}\n#endif\n\n// --------------------------------------------------------\n// Errors\n// --------------------------------------------------------\n\nstatic mi_error_fun* volatile mi_error_handler; // = NULL\nstatic _Atomic(void*) mi_error_arg; // = NULL\n\nstatic void mi_error_default(int err) {\n MI_UNUSED(err);\n#if (MI_DEBUG>0)\n if (err==EFAULT) {\n #ifdef _MSC_VER\n __debugbreak();\n #endif\n abort();\n }\n#endif\n#if (MI_SECURE>0)\n if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data)\n abort();\n }\n#endif\n#if defined(MI_XMALLOC)\n if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode\n abort();\n }\n#endif\n}\n\nvoid mi_register_error(mi_error_fun* fun, void* arg) {\n mi_error_handler = fun; // can be NULL\n mi_atomic_store_ptr_release(void,&mi_error_arg, arg);\n}\n\nvoid _mi_error_message(int err, const char* fmt, ...) {\n // show detailed error message\n va_list args;\n va_start(args, fmt);\n mi_show_error_message(fmt, args);\n va_end(args);\n // and call the error handler which may abort (or return normally)\n if (mi_error_handler != NULL) {\n mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg));\n }\n else {\n mi_error_default(err);\n }\n}\n\n// --------------------------------------------------------\n// Initialize options by checking the environment\n// --------------------------------------------------------\n\n// TODO: implement ourselves to reduce dependencies on the C runtime\n#include // strtol\n#include // strstr\n\n\nstatic void mi_option_init(mi_option_desc_t* desc) {\n // Read option value from the environment\n char s[64 + 1];\n char buf[64+1];\n _mi_strlcpy(buf, \"mimalloc_\", sizeof(buf));\n _mi_strlcat(buf, desc->name, sizeof(buf));\n bool found = _mi_getenv(buf, s, sizeof(s));\n if (!found && desc->legacy_name != NULL) {\n _mi_strlcpy(buf, \"mimalloc_\", sizeof(buf));\n _mi_strlcat(buf, desc->legacy_name, sizeof(buf));\n found = _mi_getenv(buf, s, sizeof(s));\n if (found) {\n _mi_warning_message(\"environment option \\\"mimalloc_%s\\\" is deprecated -- use \\\"mimalloc_%s\\\" instead.\\n\", desc->legacy_name, desc->name);\n }\n }\n\n if (found) {\n size_t len = _mi_strnlen(s, sizeof(buf) - 1);\n for (size_t i = 0; i < len; i++) {\n buf[i] = _mi_toupper(s[i]);\n }\n buf[len] = 0;\n if (buf[0] == 0 || _mi_streq(buf,\"1\") || _mi_streq(buf,\"TRUE\") || _mi_streq(buf,\"YES\") || _mi_streq(buf,\"ON\")) {\n desc->value = 1;\n desc->init = MI_OPTION_INITIALIZED;\n }\n else if (_mi_streq(buf,\"0\") || _mi_streq(buf,\"FALSE\") || _mi_streq(buf,\"NO\") || _mi_streq(buf,\"OFF\")) { \n desc->value = 0;\n desc->init = MI_OPTION_INITIALIZED;\n }\n else {\n char* end = buf;\n long value = strtol(buf, &end, 10);\n if (mi_option_has_size_in_kib(desc->option)) {\n // this option is interpreted in KiB to prevent overflow of `long` for large allocations\n // (long is 32-bit on 64-bit windows, which allows for 4TiB max.)\n size_t size = (value < 0 ? 0 : (size_t)value);\n bool overflow = false;\n if (*end == 'K') { end++; }\n else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; }\n else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; }\n else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; }\n else { size = (size + MI_KiB - 1) / MI_KiB; }\n if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB\n else if (*end == 'B') { end++; } // Kb, Mb, Gb, Tb\n if (overflow || size > (MI_MAX_ALLOC_SIZE / MI_KiB)) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); }\n value = (size > LONG_MAX ? LONG_MAX : (long)size);\n }\n if (*end == 0) {\n mi_option_set(desc->option, value);\n }\n else {\n // set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose.\n desc->init = MI_OPTION_DEFAULTED;\n if (desc->option == mi_option_verbose && desc->value == 0) {\n // if the 'mimalloc_verbose' env var has a bogus value we'd never know\n // (since the value defaults to 'off') so in that case briefly enable verbose\n desc->value = 1;\n _mi_warning_message(\"environment option mimalloc_%s has an invalid value.\\n\", desc->name);\n desc->value = 0;\n }\n else {\n _mi_warning_message(\"environment option mimalloc_%s has an invalid value.\\n\", desc->name);\n }\n }\n }\n mi_assert_internal(desc->init != MI_OPTION_UNINIT);\n }\n else if (!_mi_preloading()) {\n desc->init = MI_OPTION_DEFAULTED;\n }\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/os.c", "sha256": "011e965f28c0e2c885a6834d51bd756a458dd37ebfed6fd4d9ad6607ce0390ed", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\"\n\n/* -----------------------------------------------------------\n Initialization.\n----------------------------------------------------------- */\n#ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB\n#if MI_INTPTR_SIZE < 8\n#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB // 4 GiB\n#else\n#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB // 32 GiB\n#endif\n#endif\n\nstatic mi_os_mem_config_t mi_os_mem_config = {\n 4096, // page size\n 0, // large page size (usually 2MiB)\n 4096, // allocation granularity\n MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB,\n MI_MAX_VABITS, // in `bits.h`\n true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems)\n false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)\n true, // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory)\n false // has transparent huge pages? (if true we purge in (aligned) large page size chunks only to not fragment such pages)\n};\n\nbool _mi_os_has_overcommit(void) {\n return mi_os_mem_config.has_overcommit;\n}\n\nbool _mi_os_has_virtual_reserve(void) {\n return mi_os_mem_config.has_virtual_reserve;\n}\n\n\n// OS (small) page size\nsize_t _mi_os_page_size(void) {\n return mi_os_mem_config.page_size;\n}\n\n// if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB)\nsize_t _mi_os_large_page_size(void) {\n return (mi_os_mem_config.large_page_size != 0 ? mi_os_mem_config.large_page_size : _mi_os_page_size());\n}\n\n// minimal purge size. Can be larger than the page size if transparent huge pages are enabled.\nsize_t _mi_os_minimal_purge_size(void) {\n size_t minsize = mi_option_get_size(mi_option_minimal_purge_size);\n if (minsize != 0) {\n return _mi_align_up(minsize, _mi_os_page_size());\n }\n else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) {\n return _mi_os_large_page_size();\n }\n else {\n return _mi_os_page_size();\n }\n}\n\nsize_t _mi_os_guard_page_size(void) {\n const size_t gsize = _mi_os_page_size();\n mi_assert(gsize <= (MI_ARENA_SLICE_SIZE/4)); // issue #1166\n return gsize;\n}\n\nsize_t _mi_os_virtual_address_bits(void) {\n const size_t vbits = mi_os_mem_config.virtual_address_bits;\n mi_assert(vbits <= MI_MAX_VABITS);\n return vbits;\n}\n\nbool _mi_os_canuse_large_page(size_t size, size_t alignment) {\n // if we have access, check the size and alignment requirements\n if (mi_os_mem_config.large_page_size == 0) return false;\n return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0);\n}\n\n// round to a good OS allocation size (bounded by max 12.5% waste)\nsize_t _mi_os_good_alloc_size(size_t size) {\n size_t align_size;\n if (size < 512*MI_KiB) align_size = _mi_os_page_size();\n else if (size < 2*MI_MiB) align_size = 64*MI_KiB;\n else if (size < 8*MI_MiB) align_size = 256*MI_KiB;\n else if (size < 32*MI_MiB) align_size = 1*MI_MiB;\n else align_size = 4*MI_MiB;\n if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow?\n return _mi_align_up(size, align_size);\n}\n\nvoid _mi_os_init(void) {\n _mi_prim_mem_init(&mi_os_mem_config);\n}\n\n\n/* -----------------------------------------------------------\n Util\n-------------------------------------------------------------- */\nbool _mi_os_decommit(void* addr, size_t size);\nbool _mi_os_commit(void* addr, size_t size, bool* is_zero);\n\n// On systems with enough virtual address bits, we can do efficient aligned allocation by using\n// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address\n// space (64TiB) we use this technique. (but see issue #939)\n#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID)\n\n// Return a MI_HINT_ALIGN (4MiB) aligned address that is probably available.\n// If this returns NULL, the OS will determine the address but on some OS's that may not be\n// properly aligned which can be more costly as it needs to be adjusted afterwards.\n// For a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization;\n// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses\n// in the middle of the 2TiB - 6TiB address range (see issue #372))\n\n#define MI_HINT_ALIGN ((uintptr_t)4 << 20) // 4MiB alignment\n#define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start\n#define MI_HINT_AREA ((uintptr_t)4 << 40) // upto (2+4) 6TiB (since before win8 there is \"only\" 8TiB available to processes)\n#define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages)\n\nvoid* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)\n{\n static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0\n\n // todo: perhaps only do alignment hints if THP is enabled?\n if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > MI_HINT_ALIGN) return NULL;\n if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space\n size = _mi_align_up(size, MI_HINT_ALIGN);\n if (size > 16*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at least 1/(MI_HINT_AREA / 1<<34)\n size += MI_HINT_ALIGN; // put in virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.\n\n uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);\n if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize\n uintptr_t init = MI_HINT_BASE;\n #if (MI_SECURE>=1 || defined(NDEBUG)) // security: randomize start of aligned allocations unless in debug mode\n mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)\n if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point\n const uintptr_t r = _mi_theap_random_next(theap);\n init = init + ((MI_HINT_ALIGN * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB\n #endif\n uintptr_t expected = hint + size;\n mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);\n hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all\n }\n mi_assert_internal(hint%MI_HINT_ALIGN == 0);\n if (hint%try_alignment != 0) return NULL;\n return (void*)hint;\n}\n#else\nvoid* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {\n MI_UNUSED(try_alignment); MI_UNUSED(size);\n return NULL;\n}\n#endif\n\n\n/* -----------------------------------------------------------\n Guard page allocation\n----------------------------------------------------------- */\n\n// In secure mode, return the size of a guard page, otherwise 0\nsize_t _mi_os_secure_guard_page_size(void) {\n #if MI_SECURE > 0\n return _mi_os_guard_page_size();\n #else\n return 0;\n #endif\n}\n\n// In secure mode, try to decommit an area and output a warning if this fails.\nbool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) {\n if (addr == NULL) return true;\n #if MI_SECURE > 0\n bool ok = false;\n if (!memid.is_pinned) {\n mi_arena_t* const arena = mi_memid_arena(memid);\n if (arena != NULL && arena->commit_fun != NULL) {\n ok = (*(arena->commit_fun))(false /* decommit */, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);\n }\n else {\n ok = _mi_os_decommit(addr, _mi_os_secure_guard_page_size());\n }\n }\n if (!ok) {\n _mi_error_message(EINVAL, \"secure level %d, but failed to commit guard page (at %p of size %zu)\\n\", MI_SECURE, addr, _mi_os_secure_guard_page_size());\n }\n return ok;\n #else\n MI_UNUSED(memid);\n return true;\n #endif\n}\n\n// In secure mode, try to decommit an area and output a warning if this fails.\nbool _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid) {\n return _mi_os_secure_guard_page_set_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);\n}\n\n// In secure mode, try to recommit an area\nbool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid) {\n if (addr == NULL) return true;\n #if MI_SECURE > 0\n if (!memid.is_pinned) {\n mi_arena_t* const arena = mi_memid_arena(memid);\n if (arena != NULL && arena->commit_fun != NULL) {\n return (*(arena->commit_fun))(true, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);\n }\n else {\n return _mi_os_commit(addr, _mi_os_secure_guard_page_size(), NULL);\n }\n }\n #else\n MI_UNUSED(memid);\n #endif\n return true;\n}\n\n// In secure mode, try to recommit an area\nbool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid) {\n return _mi_os_secure_guard_page_reset_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);\n}\n\n\n/* -----------------------------------------------------------\n Free memory\n-------------------------------------------------------------- */\n\nstatic void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc);\n\nstatic void mi_os_prim_free(void* addr, size_t size, size_t commit_size, mi_subproc_t* subproc) {\n mi_assert_internal((size % _mi_os_page_size()) == 0);\n if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr)\n int err = _mi_prim_free(addr, size); // allow size==0 (issue #1041)\n if (err != 0) {\n _mi_warning_message(\"unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\\n\", err, err, size, addr);\n }\n if (subproc == NULL) { subproc = _mi_subproc(); } // from `mi_arenas_unsafe_destroy` we pass subproc_main explicitly as we can no longer use the theap pointer\n if (commit_size > 0) {\n mi_subproc_stat_decrease(subproc, committed, commit_size);\n }\n mi_subproc_stat_decrease(subproc, reserved, size);\n}\n\nvoid _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc /* can be NULL */) {\n if (mi_memkind_is_os(memid.memkind)) {\n size_t csize = memid.mem.os.size;\n if (csize==0) { csize = _mi_os_good_alloc_size(size); }\n mi_assert_internal(csize >= size);\n size_t commit_size = (still_committed ? csize : 0);\n void* base = addr;\n // different base? (due to alignment)\n if (memid.mem.os.base != base) {\n mi_assert(memid.mem.os.base <= addr);\n base = memid.mem.os.base;\n const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base;\n if (memid.mem.os.size==0) {\n csize += diff;\n }\n if (still_committed) {\n commit_size -= diff; // the (addr-base) part was already un-committed\n }\n }\n // free it\n if (memid.memkind == MI_MEM_OS_HUGE) {\n mi_assert(memid.is_pinned);\n mi_os_free_huge_os_pages(base, csize, subproc);\n }\n else {\n mi_os_prim_free(base, csize, (still_committed ? commit_size : 0), subproc);\n }\n }\n else {\n // nothing to do\n mi_assert(memid.memkind < MI_MEM_OS);\n }\n}\n\nvoid _mi_os_free(void* p, size_t size, mi_memid_t memid) {\n _mi_os_free_ex(p, size, true, memid, NULL);\n}\n\n\n/* -----------------------------------------------------------\n Primitive allocation from the OS.\n-------------------------------------------------------------- */\n\n// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.\n// Also `hint_addr` is a hint and may be ignored.\nstatic void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {\n mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);\n mi_assert_internal(is_zero != NULL);\n mi_assert_internal(is_large != NULL);\n if (size == 0) return NULL;\n if (!commit) { allow_large = false; }\n if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning\n\n // try to align along large OS page size for larger allocations\n const size_t large_page_size = mi_os_mem_config.large_page_size;\n if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && _mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) {\n try_alignment = large_page_size;\n }\n\n *is_zero = false;\n void* p = NULL;\n int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p);\n if (err != 0) {\n _mi_warning_message(\"unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\\n\", err, err, hint_addr, size, try_alignment, commit, allow_large);\n }\n\n mi_os_stat_counter_increase(mmap_calls, 1);\n if (p != NULL) {\n mi_os_stat_increase(reserved, size);\n if (commit) {\n mi_os_stat_increase(committed, size);\n // seems needed for asan (or `mimalloc-test-api` fails)\n #ifdef MI_TRACK_ASAN\n if (*is_zero) { mi_track_mem_defined(p,size); }\n else { mi_track_mem_undefined(p,size); }\n #endif\n }\n }\n return p;\n}\n\nstatic void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {\n return mi_os_prim_alloc_at(NULL, size, try_alignment, commit, allow_large, is_large, is_zero);\n}\n\n\n// Primitive aligned allocation from the OS.\n// This function guarantees the allocated memory is aligned.\nstatic void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {\n mi_assert_internal(memid!=NULL);\n mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));\n mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);\n *memid = _mi_memid_none();\n if (!commit) allow_large = false;\n if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;\n size = _mi_align_up(size, _mi_os_page_size());\n\n // try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size.\n const bool try_direct_alloc = (alignment <= mi_os_mem_config.alloc_granularity || alignment <= size/4);\n\n bool os_is_large = false;\n bool os_is_zero = false;\n void* os_base = NULL;\n size_t os_size = size;\n void* p = NULL;\n if (try_direct_alloc) {\n p = mi_os_prim_alloc(size, alignment, commit, allow_large, &os_is_large, &os_is_zero);\n }\n\n // aligned already?\n if (p != NULL && _mi_is_aligned(p,alignment)) {\n os_base = p;\n }\n else {\n // if not aligned, free it, overallocate, and unmap around it\n #if !MI_TRACK_ASAN\n if (try_direct_alloc) {\n _mi_warning_message(\"unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\\n\", size, p, alignment, commit);\n }\n #endif\n if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0), NULL); }\n if (size >= (SIZE_MAX - alignment)) return NULL; // overflow\n const size_t over_size = size + alignment;\n\n if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block\n // over-allocate uncommitted (virtual) memory\n p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero);\n if (p == NULL) return NULL;\n\n // set p to the aligned part in the full region\n // note: Windows VirtualFree needs the actual base pointer\n // this is handled though by having the `base` field in the memid\n os_base = p; // remember the base\n os_size = over_size;\n p = _mi_align_up_ptr(p, alignment);\n\n // explicitly commit only the aligned part\n if (commit) {\n if (!_mi_os_commit(p, size, NULL)) {\n mi_os_prim_free(os_base, over_size, 0, NULL);\n return NULL;\n }\n }\n }\n else { // mmap can free inside an allocation\n // overallocate...\n p = mi_os_prim_alloc(over_size, 1, commit, false, &os_is_large, &os_is_zero);\n if (p == NULL) return NULL;\n\n // and selectively unmap parts around the over-allocated area.\n void* const aligned_p = _mi_align_up_ptr(p, alignment);\n const size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;\n const size_t mid_size = _mi_align_up(size, _mi_os_page_size());\n const size_t post_size = over_size - pre_size - mid_size;\n mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);\n if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0), NULL); }\n if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), NULL); }\n // we can return the aligned pointer on `mmap` systems\n p = aligned_p;\n os_base = aligned_p; // since we freed the pre part, `*base == p`.\n os_size = mid_size;\n }\n }\n\n mi_assert_internal(p != NULL && os_base != NULL && _mi_is_aligned(p,alignment));\n mi_assert_internal(os_base <= p && size <= os_size);\n *memid = _mi_memid_create_os(os_base,os_size,commit,os_is_zero,os_is_large);\n return p;\n}\n\n\n/* -----------------------------------------------------------\n OS API: alloc and alloc_aligned\n----------------------------------------------------------- */\n\nvoid* _mi_os_alloc(size_t size, mi_memid_t* memid) {\n *memid = _mi_memid_none();\n if (size == 0) return NULL;\n size = _mi_os_good_alloc_size(size);\n bool os_is_large = false;\n bool os_is_zero = false;\n void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero);\n if (p == NULL) return NULL;\n\n *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large);\n mi_assert_internal(memid->mem.os.size >= size);\n mi_assert_internal(memid->initially_committed);\n return p;\n}\n\nvoid* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)\n{\n MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings\n *memid = _mi_memid_none();\n if (size == 0) return NULL;\n size = _mi_os_good_alloc_size(size);\n alignment = _mi_align_up(alignment, _mi_os_page_size());\n\n void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, memid );\n if (p == NULL) return NULL;\n\n mi_assert_internal(memid->mem.os.size >= size);\n mi_assert_internal(_mi_is_aligned(p,alignment));\n if (commit) { mi_assert_internal(memid->initially_committed); }\n return p;\n}\n\n\nmi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_t* memid) {\n if (p==NULL || size==0) return p;\n // ensure committed\n if (!memid->initially_committed) {\n bool is_zero = false;\n if (!_mi_os_commit(p, size, &is_zero)) {\n _mi_os_free(p, size, *memid);\n return NULL;\n }\n memid->initially_committed = true;\n }\n // ensure zero'd\n if (memid->initially_zero) return p;\n _mi_memzero_aligned(p,size);\n memid->initially_zero = true;\n return p;\n}\n\nvoid* _mi_os_zalloc(size_t size, mi_memid_t* memid) {\n void* p = _mi_os_alloc(size,memid);\n return mi_os_ensure_zero(p, size, memid);\n}\n\n/* -----------------------------------------------------------\n OS aligned allocation with an offset. This is used\n for large alignments > MI_BLOCK_ALIGNMENT_MAX. We use a large mimalloc\n page where the object can be aligned at an offset from the start of the segment.\n As we may need to overallocate, we need to free such pointers using `mi_free_aligned`\n to use the actual start of the memory region.\n----------------------------------------------------------- */\n\nvoid* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {\n mi_assert(offset <= size);\n mi_assert((alignment % _mi_os_page_size()) == 0);\n *memid = _mi_memid_none();\n if (offset > size) return NULL;\n if (offset == 0) {\n // regular aligned allocation\n return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);\n }\n else {\n // overallocate to align at an offset\n const size_t extra = _mi_align_up(offset, alignment) - offset;\n if (size >= SIZE_MAX - extra) return NULL; // too large\n const size_t oversize = size + extra;\n void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid);\n if (start == NULL) return NULL;\n\n void* const p = (uint8_t*)start + extra;\n mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment));\n // decommit the overallocation at the start\n if (commit && extra >= _mi_os_page_size()) {\n _mi_os_decommit(start, extra);\n }\n return p;\n }\n}\n\n/* -----------------------------------------------------------\n OS memory API: reset, commit, decommit, protect, unprotect.\n----------------------------------------------------------- */\n\n// OS page align within a given area, either conservative (pages inside the area only),\n// or not (straddling pages outside the area is possible)\nstatic void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {\n mi_assert(addr != NULL && size > 0);\n if (newsize != NULL) *newsize = 0;\n if (size == 0 || addr == NULL) return NULL;\n\n // page align conservatively within the range, or liberally straddling pages outside the range\n void* start = (conservative ? _mi_align_up_ptr(addr, _mi_os_page_size())\n : _mi_align_down_ptr(addr, _mi_os_page_size()));\n void* end = (conservative ? _mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())\n : _mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));\n ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;\n if (diff <= 0) return NULL;\n\n mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size));\n if (newsize != NULL) *newsize = (size_t)diff;\n return start;\n}\n\nstatic void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {\n return mi_os_page_align_areax(true, addr, size, newsize);\n}\n\nbool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) {\n if (is_zero != NULL) { *is_zero = false; }\n mi_os_stat_counter_increase(commit_calls, 1);\n\n // page align range\n size_t csize;\n void* start = mi_os_page_align_areax(false /* conservative? */, addr, size, &csize);\n if (csize == 0) return true;\n\n // commit\n bool os_is_zero = false;\n int err = _mi_prim_commit(start, csize, &os_is_zero);\n if (err != 0) {\n _mi_warning_message(\"cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\\n\", err, err, start, csize);\n return false;\n }\n if (os_is_zero && is_zero != NULL) {\n *is_zero = true;\n mi_assert_expensive(mi_mem_is_zero(start, csize));\n }\n // note: the following seems required for asan (otherwise `mimalloc-test-stress` fails)\n #ifdef MI_TRACK_ASAN\n if (os_is_zero) { mi_track_mem_defined(start,csize); }\n else { mi_track_mem_undefined(start,csize); }\n #endif\n mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit\n return true;\n}\n\nbool _mi_os_commit(void* addr, size_t size, bool* is_zero) {\n return _mi_os_commit_ex(addr, size, is_zero, size);\n}\n\nstatic bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) {\n mi_assert_internal(needs_recommit!=NULL);\n\n // page align\n size_t csize;\n void* start = mi_os_page_align_area_conservative(addr, size, &csize);\n if (csize == 0) return true;\n\n // decommit\n *needs_recommit = true;\n int err = _mi_prim_decommit(start,csize,needs_recommit);\n if (err != 0) {\n _mi_warning_message(\"cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\\n\", err, err, start, csize);\n }\n else if (*needs_recommit) {\n mi_os_stat_decrease(committed, stat_size);\n }\n mi_assert_internal(err == 0);\n return (err == 0);\n}\n\nbool _mi_os_decommit(void* addr, size_t size) {\n bool needs_recommit;\n return mi_os_decommit_ex(addr, size, &needs_recommit, size);\n}\n\n\n// Signal to the OS that the address range is no longer in use\n// but may be used later again. This will release physical memory\n// pages and reduce swapping while keeping the memory committed.\n// We page align to a conservative area inside the range to reset.\nbool _mi_os_reset(void* addr, size_t size) {\n // page align conservatively within the range\n size_t csize;\n void* start = mi_os_page_align_area_conservative(addr, size, &csize);\n if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr)\n mi_os_stat_counter_increase(reset, csize);\n mi_os_stat_counter_increase(reset_calls, 1);\n\n #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN\n memset(start, 0, csize); // pretend it is eagerly reset\n #endif\n\n int err = _mi_prim_reset(start, csize);\n if (err != 0) {\n _mi_warning_message(\"cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\\n\", err, err, start, csize);\n }\n return (err == 0);\n}\n\n\nvoid _mi_os_reuse( void* addr, size_t size ) {\n // page align conservatively within the range\n size_t csize = 0;\n void* const start = mi_os_page_align_area_conservative(addr, size, &csize);\n if (csize == 0) return;\n const int err = _mi_prim_reuse(start, csize);\n if (err != 0) {\n _mi_warning_message(\"cannot reuse OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\\n\", err, err, start, csize);\n }\n}\n\n// either resets or decommits memory, returns true if the memory needs\n// to be recommitted if it is to be re-used later on.\nbool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg)\n{\n if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed?\n mi_os_stat_counter_increase(purge_calls, 1);\n mi_os_stat_counter_increase(purged, size);\n\n if (commit_fun != NULL) {\n bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg);\n return decommitted; // needs_recommit?\n }\n else if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit?\n !_mi_preloading()) // don't decommit during preloading (unsafe)\n {\n bool needs_recommit = true;\n mi_os_decommit_ex(p, size, &needs_recommit, stat_size);\n return needs_recommit;\n }\n else {\n if (allow_reset) { // this can sometimes be not allowed if the range is not fully committed (on Windows, we cannot reset uncommitted memory)\n _mi_os_reset(p, size);\n }\n return false; // needs no recommit\n }\n}\n\n// either resets or decommits memory, returns true if the memory needs\n// to be recommitted if it is to be re-used later on.\nbool _mi_os_purge(void* p, size_t size) {\n return _mi_os_purge_ex(p, size, true, size, NULL, NULL);\n}\n\n\n// Protect a region in memory to be not accessible.\nstatic bool mi_os_protectx(void* addr, size_t size, bool protect) {\n // page align conservatively within the range\n size_t csize = 0;\n void* start = mi_os_page_align_area_conservative(addr, size, &csize);\n if (csize == 0) return false;\n /*\n if (_mi_os_is_huge_reserved(addr)) {\n\t _mi_warning_message(\"cannot mprotect memory allocated in huge OS pages\\n\");\n }\n */\n int err = _mi_prim_protect(start,csize,protect);\n if (err != 0) {\n _mi_warning_message(\"cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\\n\", (protect ? \"protect\" : \"unprotect\"), err, err, start, csize);\n }\n return (err == 0);\n}\n\nbool _mi_os_protect(void* addr, size_t size) {\n return mi_os_protectx(addr, size, true);\n}\n\nbool _mi_os_unprotect(void* addr, size_t size) {\n return mi_os_protectx(addr, size, false);\n}\n\n\n\n/* ----------------------------------------------------------------------------\nSupport for allocating huge OS pages (1Gib) that are reserved up-front\nand possibly associated with a specific NUMA node. (use `numa_node>=0`)\n-----------------------------------------------------------------------------*/\n#define MI_HUGE_OS_PAGE_SIZE (MI_GiB)\n\n\n#if (MI_INTPTR_SIZE >= 8)\n// To ensure proper alignment, use our own area for huge OS pages\nstatic mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0\n\n// Claim an aligned address range for huge pages\nstatic uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {\n if (total_size != NULL) *total_size = 0;\n const size_t size = pages * MI_HUGE_OS_PAGE_SIZE;\n\n uintptr_t start = 0;\n uintptr_t end = 0;\n uintptr_t huge_start = mi_atomic_load_relaxed(&mi_huge_start);\n do {\n start = huge_start;\n if (start == 0) {\n // Initialize the start address after the 32TiB area\n start = ((uintptr_t)8 << 40); // 8TiB virtual start address\n #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode\n mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)\n if (mi_theap_is_initialized(theap)) { // todo: or no hint at all if we lack randomness?\n const uintptr_t r = _mi_theap_random_next(theap);\n start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB\n }\n else {\n _mi_warning_message(\"failed to randomize the start address of huge pages allocation (%zu bytes at %p)\", size, start);\n }\n #endif\n }\n end = start + size;\n } while (!mi_atomic_cas_weak_acq_rel(&mi_huge_start, &huge_start, end));\n\n if (total_size != NULL) *total_size = size;\n return (uint8_t*)start;\n}\n#else\nstatic uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {\n MI_UNUSED(pages);\n if (total_size != NULL) *total_size = 0;\n return NULL;\n}\n#endif\n\n// Allocate MI_ARENA_SLICE_ALIGN aligned huge pages\nvoid* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {\n *memid = _mi_memid_none();\n if (psize != NULL) *psize = 0;\n if (pages_reserved != NULL) *pages_reserved = 0;\n size_t size = 0;\n uint8_t* const start = mi_os_claim_huge_pages(pages, &size);\n if (start == NULL) return NULL; // or 32-bit systems\n\n // Allocate one page at the time but try to place them contiguously\n // We allocate one page at the time to be able to abort if it takes too long\n // or to at least allocate as many as available on the system.\n mi_msecs_t start_t = _mi_clock_start();\n size_t page = 0;\n bool all_zero = true;\n while (page < pages) {\n // allocate a page\n bool is_zero = false;\n void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE);\n void* p = NULL;\n int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p);\n if (!is_zero) { all_zero = false; }\n if (err != 0) {\n _mi_warning_message(\"unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\\n\", err, err, addr, MI_HUGE_OS_PAGE_SIZE);\n break;\n }\n\n // Did we succeed at a contiguous address?\n if (p != addr) {\n // no success, issue a warning and break\n if (p != NULL) {\n _mi_warning_message(\"could not allocate contiguous huge OS page %zu at %p\\n\", page, addr);\n mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, NULL);\n }\n break;\n }\n\n // success, record it\n page++; // increase before timeout check (see issue #711)\n mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE);\n mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE);\n\n // check for timeout\n if (max_msecs > 0) {\n mi_msecs_t elapsed = _mi_clock_end(start_t);\n if (page >= 1) {\n mi_msecs_t estimate = ((elapsed / (page+1)) * pages);\n if (estimate > 2*max_msecs) { // seems like we are going to timeout, break\n elapsed = max_msecs + 1;\n }\n }\n if (elapsed > max_msecs) {\n _mi_warning_message(\"huge OS page allocation timed out (after allocating %zu page(s))\\n\", page);\n break;\n }\n }\n }\n const size_t allocated = page * MI_HUGE_OS_PAGE_SIZE;\n mi_assert_internal(allocated <= size);\n if (pages_reserved != NULL) { *pages_reserved = page; }\n if (psize != NULL) { *psize = allocated; }\n if (page != 0) {\n mi_assert(start != NULL);\n *memid = _mi_memid_create_os(start, allocated, true /* is committed */, all_zero, true /* is_large */);\n memid->memkind = MI_MEM_OS_HUGE;\n mi_assert(memid->is_pinned);\n #ifdef MI_TRACK_ASAN\n if (all_zero) { mi_track_mem_defined(start,allocated); }\n #endif\n }\n return (page == 0 ? NULL : start);\n}\n\n// free every huge page in a range individually (as we allocated per page)\n// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.\nstatic void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc) {\n if (p==NULL || size==0) return;\n uint8_t* base = (uint8_t*)p;\n while (size >= MI_HUGE_OS_PAGE_SIZE) {\n mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, subproc);\n size -= MI_HUGE_OS_PAGE_SIZE;\n base += MI_HUGE_OS_PAGE_SIZE;\n }\n}\n\n\n/* ----------------------------------------------------------------------------\nSupport NUMA aware allocation\n-----------------------------------------------------------------------------*/\n\nstatic _Atomic(size_t) mi_numa_node_count; // = 0 // cache the node count\n\nint _mi_os_numa_node_count(void) {\n size_t count = mi_atomic_load_acquire(&mi_numa_node_count);\n if mi_unlikely(count == 0) {\n long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly?\n if (ncount > 0 && ncount < INT_MAX) {\n count = (size_t)ncount;\n }\n else {\n const size_t n = _mi_prim_numa_node_count(); // or detect dynamically\n if (n == 0 || n > INT_MAX) { count = 1; }\n else { count = n; }\n }\n mi_atomic_store_release(&mi_numa_node_count, count); // save it\n if (count>1) { _mi_verbose_message(\"using %zd numa regions\\n\", count); }\n }\n mi_assert_internal(count > 0 && count <= INT_MAX);\n return (int)count;\n}\n\nstatic int mi_os_numa_node_get(void) {\n int numa_count = _mi_os_numa_node_count();\n if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0\n // never more than the node count and >= 0\n const size_t n = _mi_prim_numa_node();\n int numa_node = (n < INT_MAX ? (int)n : 0);\n if (numa_node >= numa_count) { numa_node = numa_node % numa_count; }\n return numa_node;\n}\n\nint _mi_os_numa_node(void) {\n if mi_likely(mi_atomic_load_relaxed(&mi_numa_node_count) == 1) {\n return 0;\n }\n else {\n return mi_os_numa_node_get();\n }\n}\n\n\n/* ----------------------------------------------------------------------------\n Public API\n-----------------------------------------------------------------------------*/\n#if 0\nmi_decl_export void* mi_os_alloc(size_t size, bool commit, size_t* full_size) {\n return mi_os_alloc_aligned(size, mi_os_mem_config.alloc_granularity, commit, NULL, full_size);\n}\n\nstatic void* mi_os_alloc_aligned_ex(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_committed, bool* is_pinned, void** base, size_t* full_size) {\n mi_memid_t memid = _mi_memid_none();\n void* p = _mi_os_alloc_aligned(size, alignment, commit, allow_large, &memid);\n if (p == NULL) return p;\n if (is_committed != NULL) { *is_committed = memid.initially_committed; }\n if (is_pinned != NULL) { *is_pinned = memid.is_pinned; }\n if (base != NULL) { *base = memid.mem.os.base; }\n if (full_size != NULL) { *full_size = memid.mem.os.size; }\n if (!memid.initially_zero && memid.initially_committed) {\n _mi_memzero_aligned(memid.mem.os.base, memid.mem.os.size);\n }\n return p;\n}\n\nmi_decl_export void* mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, void** base, size_t* full_size) {\n return mi_os_alloc_aligned_ex(size, alignment, commit, false, NULL, NULL, base, full_size);\n}\n\nmi_decl_export void* mi_os_alloc_aligned_allow_large(size_t size, size_t alignment, bool commit, bool* is_committed, bool* is_pinned, void** base, size_t* full_size) {\n return mi_os_alloc_aligned_ex(size, alignment, commit, true, is_committed, is_pinned, base, full_size);\n}\n\nmi_decl_export void mi_os_free(void* p, size_t size) {\n if (p==NULL || size == 0) return;\n mi_memid_t memid = _mi_memid_create_os(p, size, true, false, false);\n _mi_os_free(p, size, memid);\n}\n\nmi_decl_export void mi_os_commit(void* p, size_t size) {\n _mi_os_commit(p, size, NULL);\n}\n\nmi_decl_export void mi_os_decommit(void* p, size_t size) {\n _mi_os_decommit(p, size);\n}\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/page-map.c", "sha256": "1be3a208c678fc34187c2a8ffb75f3fe1eecc674bbf8b2af4d1f329506b1fad5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/*----------------------------------------------------------------------------\nCopyright (c) 2023-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"bitmap.h\"\n\nstatic void mi_page_map_cannot_commit(void) {\n _mi_warning_message(\"unable to commit the allocation page-map on-demand\\n\" );\n}\n\n#if MI_PAGE_MAP_FLAT\n\n// The page-map contains a byte for each 64kb slice in the address space.\n// For an address `a` where `ofs = _mi_page_map[a >> 16]`:\n// 0 = unused\n// 1 = the slice at `a & ~0xFFFF` is a mimalloc page.\n// 1 < ofs <= 127 = the slice is part of a page, starting at `(((a>>16) - ofs - 1) << 16)`.\n//\n// 1 byte per slice => 1 TiB address space needs a 2^14 * 2^16 = 16 MiB page map.\n// A full 256 TiB address space (48 bit) needs a 4 GiB page map.\n// A full 4 GiB address space (32 bit) needs only a 64 KiB page map.\n\nmi_decl_cache_align uint8_t* _mi_page_map = NULL;\nstatic void* mi_page_map_max_address = NULL;\nstatic mi_memid_t mi_page_map_memid;\n\n#define MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT MI_ARENA_SLICE_SIZE\nstatic mi_bitmap_t* mi_page_map_commit; // one bit per committed 64 KiB entries\n\nmi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count);\n\nbool _mi_page_map_init(void) {\n size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);\n if (vbits == 0) {\n vbits = _mi_os_virtual_address_bits();\n #if MI_ARCH_X64 // canonical address is limited to the first 128 TiB\n if (vbits >= 48) { vbits = 47; }\n #endif\n }\n if (vbits < MI_ARENA_SLICE_SHIFT) {\n vbits = MI_ARENA_SLICE_SHIFT;\n }\n\n // Allocate the page map and commit bits\n mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));\n const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT));\n const bool commit = (page_map_size <= 1*MI_MiB || mi_option_is_enabled(mi_option_pagemap_commit)); // _mi_os_has_overcommit(); // commit on-access on Linux systems?\n const size_t commit_bits = _mi_divide_up(page_map_size, MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);\n const size_t bitmap_size = (commit ? 0 : mi_bitmap_size(commit_bits, NULL));\n const size_t reserve_size = bitmap_size + page_map_size;\n uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);\n if (base==NULL) {\n _mi_error_message(ENOMEM, \"unable to reserve virtual memory for the page map (%zu KiB)\\n\", page_map_size / MI_KiB);\n return false;\n }\n if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {\n _mi_warning_message(\"internal: the page map was committed but not zero initialized!\\n\");\n _mi_memzero_aligned(base, reserve_size);\n }\n if (bitmap_size > 0) {\n mi_page_map_commit = (mi_bitmap_t*)base;\n if (!_mi_os_commit(mi_page_map_commit, bitmap_size, NULL)) {\n mi_page_map_cannot_commit();\n return false;\n }\n mi_bitmap_init(mi_page_map_commit, commit_bits, true);\n }\n _mi_page_map = base + bitmap_size;\n\n // commit the first part so NULL pointers get resolved without an access violation\n if (!commit) {\n if (!mi_page_map_ensure_committed(0, 1)) {\n mi_page_map_cannot_commit();\n return false;\n }\n }\n _mi_page_map[0] = 1; // so _mi_ptr_page(NULL) == NULL\n mi_assert_internal(_mi_ptr_page(NULL)==NULL);\n return true;\n}\n\nvoid _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {\n mi_assert_internal(subproc != NULL);\n mi_assert_internal(_mi_page_map != NULL);\n if (_mi_page_map == NULL) return;\n _mi_os_free_ex(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);\n _mi_page_map = NULL;\n mi_page_map_commit = NULL;\n mi_page_map_max_address = NULL;\n mi_page_map_memid = _mi_memid_none();\n}\n\n\nstatic bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) {\n // is the page map area that contains the page address committed?\n // we always set the commit bits so we can track what ranges are in-use.\n // we only actually commit if the map wasn't committed fully already.\n if (mi_page_map_commit != NULL) {\n const size_t commit_idx = idx / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;\n const size_t commit_idx_hi = (idx + slice_count - 1) / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;\n for (size_t i = commit_idx; i <= commit_idx_hi; i++) { // per bit to avoid crossing over bitmap chunks\n if (mi_bitmap_is_clear(mi_page_map_commit, i)) {\n // this may race, in which case we do multiple commits (which is ok)\n bool is_zero;\n uint8_t* const start = _mi_page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);\n const size_t size = MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;\n if (!_mi_os_commit(start, size, &is_zero)) {\n mi_page_map_cannot_commit();\n return false;\n }\n if (!is_zero && !mi_page_map_memid.initially_zero) { _mi_memzero(start, size); }\n mi_bitmap_set(mi_page_map_commit, i);\n }\n }\n }\n #if MI_DEBUG > 0\n _mi_page_map[idx] = 0;\n _mi_page_map[idx+slice_count-1] = 0;\n #endif\n return true;\n}\n\n\nstatic size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* slice_count) {\n size_t page_size;\n *page_start = mi_page_area(page, &page_size);\n if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; } // furthest interior pointer\n *slice_count = mi_slice_count_of_size(page_size) + ((*page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks\n return _mi_page_map_index(page);\n}\n\nbool _mi_page_map_register(mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access!\n if mi_unlikely(_mi_page_map == NULL) {\n if (!_mi_page_map_init()) return false;\n }\n mi_assert(_mi_page_map!=NULL);\n uint8_t* page_start;\n size_t slice_count;\n const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);\n\n if (!mi_page_map_ensure_committed(idx, slice_count)) {\n return false;\n }\n\n // set the offsets\n for (size_t i = 0; i < slice_count; i++) {\n mi_assert_internal(i < 128);\n _mi_page_map[idx + i] = (uint8_t)(i+1);\n }\n return true;\n}\n\nvoid _mi_page_map_unregister(mi_page_t* page) {\n mi_assert_internal(_mi_page_map != NULL);\n // get index and count\n uint8_t* page_start;\n size_t slice_count;\n const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);\n // unset the offsets\n _mi_memzero(_mi_page_map + idx, slice_count);\n}\n\nvoid _mi_page_map_unregister_range(void* start, size_t size) {\n const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);\n const uintptr_t index = _mi_page_map_index(start);\n // todo: scan the commit bits and clear only those ranges?\n if (!mi_page_map_ensure_committed(index, slice_count)) { // we commit the range in total;\n return;\n }\n _mi_memzero(&_mi_page_map[index], slice_count);\n}\n\n\nmi_page_t* _mi_safe_ptr_page(const void* p) {\n if mi_unlikely(p >= mi_page_map_max_address) return NULL;\n const uintptr_t idx = _mi_page_map_index(p);\n if mi_unlikely(mi_page_map_commit != NULL && !mi_bitmap_is_set(mi_page_map_commit, idx/MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT)) return NULL;\n const uintptr_t ofs = _mi_page_map[idx];\n if mi_unlikely(ofs == 0) return NULL;\n return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) - ofs + 1) << MI_ARENA_SLICE_SHIFT);\n}\n\nmi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {\n return (_mi_safe_ptr_page(p) != NULL);\n}\n\n#else\n\n// A 2-level page map\n#define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*))\n#define MI_PAGE_MAP_ENTRIES_PER_CBIT (MI_PAGE_MAP_COUNT < MI_BFIELD_BITS ? 1 : (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS))\n\nmi_decl_cache_align _Atomic(mi_submap_t)* _mi_page_map;\nstatic size_t mi_page_map_count;\nstatic void* mi_page_map_max_address;\nstatic mi_memid_t mi_page_map_memid;\nstatic mi_lock_t mi_page_map_lock;\n\n// divide the main map in 64 (`MI_BFIELD_BITS`) parts commit those parts on demand\nstatic _Atomic(mi_bfield_t) mi_page_map_commit;\n\nmi_decl_nodiscard static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) {\n mi_bfield_t commit = mi_atomic_load_relaxed(&mi_page_map_commit);\n const size_t bit_idx = idx/MI_PAGE_MAP_ENTRIES_PER_CBIT;\n mi_assert_internal(bit_idx < MI_BFIELD_BITS);\n if (pbit_idx != NULL) { *pbit_idx = bit_idx; }\n return ((commit & (MI_ZU(1) << bit_idx)) != 0);\n}\n\nmi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, mi_submap_t* submap) {\n mi_assert_internal(submap!=NULL && *submap==NULL);\n size_t bit_idx;\n if mi_unlikely(!mi_page_map_is_committed(idx, &bit_idx)) {\n uint8_t* start = (uint8_t*)&_mi_page_map[bit_idx * MI_PAGE_MAP_ENTRIES_PER_CBIT];\n if (!_mi_os_commit(start, MI_PAGE_MAP_ENTRIES_PER_CBIT * sizeof(mi_submap_t), NULL)) {\n mi_page_map_cannot_commit();\n return false;\n }\n mi_atomic_or_acq_rel(&mi_page_map_commit, MI_ZU(1) << bit_idx);\n }\n *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // acquire _mi_page_map_at(idx);\n return true;\n}\n\n// initialize the page map\nbool _mi_page_map_init(void) {\n size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);\n if (vbits == 0) {\n vbits = _mi_os_virtual_address_bits();\n #if MI_ARCH_X64 // canonical address is limited to the first 128 TiB\n if (vbits >= 48) { vbits = 47; }\n #endif\n }\n if (vbits < MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT) {\n vbits = MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT;\n }\n\n // Allocate the page map and commit bits\n mi_assert(MI_MAX_VABITS >= vbits);\n mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));\n mi_page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));\n mi_assert(mi_page_map_count <= MI_PAGE_MAP_COUNT);\n const size_t os_page_size = _mi_os_page_size();\n const size_t page_map_size = _mi_align_up( mi_page_map_count * sizeof(mi_page_t**), os_page_size);\n const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;\n const size_t reserve_size = page_map_size + submap_size;\n #if MI_SECURE\n const bool commit = true; // the whole page map is valid and we can reliably check any pointer\n #else\n const bool commit = page_map_size <= 64*MI_KiB ||\n mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit();\n #endif\n _mi_page_map = (_Atomic(mi_page_t**)*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);\n if (_mi_page_map==NULL) {\n _mi_error_message(ENOMEM, \"unable to reserve virtual memory for the page map (%zu KiB)\\n\", page_map_size / MI_KiB);\n return false;\n }\n if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {\n _mi_warning_message(\"internal: the page map was committed but not zero initialized!\\n\");\n _mi_memzero_aligned(_mi_page_map, page_map_size);\n }\n mi_atomic_store_release(&mi_page_map_commit, (mi_page_map_memid.initially_committed ? ~MI_ZU(0) : MI_ZU(0)));\n\n // ensure there is a submap for the NULL address\n mi_page_t** const sub0 = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size); // we reserved a submap part at the end already\n if (!mi_page_map_memid.initially_committed) {\n if (!_mi_os_commit(sub0, submap_size, NULL)) { // commit full submap (issue #1087)\n mi_page_map_cannot_commit();\n return false;\n }\n }\n if (!mi_page_map_memid.initially_zero) { // initialize low addresses with NULL\n _mi_memzero_aligned(sub0, submap_size);\n }\n mi_submap_t nullsub = NULL;\n if (!mi_page_map_ensure_committed(0,&nullsub)) {\n mi_page_map_cannot_commit();\n return false;\n }\n mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[0], sub0);\n mi_lock_init(&mi_page_map_lock); // initialize late in case the lock init causes allocation\n \n mi_assert_internal(_mi_ptr_page(NULL)==NULL);\n return true;\n}\n\n\nvoid _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {\n mi_assert_internal(subproc != NULL);\n mi_assert_internal(_mi_page_map != NULL);\n if (_mi_page_map == NULL) return;\n mi_lock_done(&mi_page_map_lock); \n for (size_t idx = 1; idx < mi_page_map_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map)\n // free all sub-maps\n if (mi_page_map_is_committed(idx, NULL)) {\n mi_submap_t sub = _mi_page_map_at(idx);\n if (sub != NULL) {\n mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false);\n _mi_os_free_ex(memid.mem.os.base, memid.mem.os.size, true, memid, subproc);\n mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[idx], NULL);\n }\n }\n }\n _mi_os_free_ex(_mi_page_map, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);\n _mi_page_map = NULL;\n mi_page_map_count = 0;\n mi_page_map_memid = _mi_memid_none();\n mi_page_map_max_address = NULL;\n mi_atomic_store_release(&mi_page_map_commit, (mi_bfield_t)0);\n}\n\n\nmi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submap_t* submap) {\n mi_assert_internal(submap!=NULL && *submap==NULL);\n mi_submap_t sub = NULL;\n if (!mi_page_map_ensure_committed(idx, &sub)) {\n return false;\n }\n if mi_unlikely(sub == NULL) {\n // sub map not yet allocated, alloc now\n mi_lock(&mi_page_map_lock) \n {\n sub = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // reload\n if (sub==NULL) // not yet allocated by another thread? \n {\n mi_memid_t memid;\n const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;\n sub = (mi_submap_t)_mi_os_zalloc(submap_size, &memid); \n if (sub==NULL) {\n _mi_warning_message(\"internal error: unable to extend the page map\\n\"); \n }\n else {\n mi_submap_t expect = NULL;\n if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &_mi_page_map[idx], &expect, sub)) {\n // another thread already allocated it.. free and continue\n _mi_os_free(sub, submap_size, memid);\n sub = expect;\n }\n }\n }\n }\n if (sub==NULL) return false; // unable to allocate the submap..\n }\n mi_assert_internal(sub!=NULL);\n *submap = sub;\n return true;\n}\n\nstatic bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {\n // is the page map area that contains the page address committed?\n while (slice_count > 0) {\n mi_submap_t sub = NULL;\n if (!mi_page_map_ensure_submap_at(idx, &sub)) {\n return false;\n };\n mi_assert_internal(sub!=NULL);\n // set the offsets for the page\n while (slice_count > 0 && sub_idx < MI_PAGE_MAP_SUB_COUNT) {\n sub[sub_idx] = page;\n slice_count--;\n sub_idx++;\n }\n idx++; // potentially wrap around to the next idx\n sub_idx = 0;\n }\n return true;\n}\n\nstatic bool mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {\n if mi_unlikely(!mi_page_map_set_range_prim(page,idx,sub_idx,slice_count)) {\n // failed to commit, call again to reset the page pointer if needed\n if (page!=NULL) {\n mi_page_map_set_range_prim(NULL,idx,sub_idx,slice_count);\n }\n return false;\n }\n return true;\n}\n\nstatic size_t mi_page_map_get_idx(mi_page_t* page, size_t* sub_idx, size_t* slice_count) {\n size_t page_size;\n uint8_t* page_start = mi_page_area(page, &page_size);\n if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; } // furthest interior pointer\n *slice_count = mi_slice_count_of_size(page_size) + ((page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks\n return _mi_page_map_index(page_start, sub_idx);\n}\n\nbool _mi_page_map_register(mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access!\n if mi_unlikely(_mi_page_map == NULL) {\n if (!_mi_page_map_init()) return false;\n }\n mi_assert(_mi_page_map!=NULL);\n size_t slice_count;\n size_t sub_idx;\n const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);\n return mi_page_map_set_range(page, idx, sub_idx, slice_count);\n}\n\nvoid _mi_page_map_unregister(mi_page_t* page) {\n mi_assert_internal(_mi_page_map != NULL);\n mi_assert_internal(page != NULL);\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n if mi_unlikely(_mi_page_map == NULL) return;\n // get index and count\n size_t slice_count;\n size_t sub_idx;\n const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);\n // unset the offsets\n mi_page_map_set_range(NULL, idx, sub_idx, slice_count);\n}\n\nvoid _mi_page_map_unregister_range(void* start, size_t size) {\n if mi_unlikely(_mi_page_map == NULL) return;\n const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);\n size_t sub_idx;\n const uintptr_t idx = _mi_page_map_index(start, &sub_idx);\n mi_page_map_set_range(NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed?\n}\n\n// Return NULL for invalid pointers\nmi_page_t* _mi_safe_ptr_page(const void* p) {\n if (p==NULL) return NULL;\n if mi_unlikely(p >= mi_page_map_max_address) return NULL;\n size_t sub_idx;\n const size_t idx = _mi_page_map_index(p,&sub_idx);\n if mi_unlikely(!mi_page_map_is_committed(idx,NULL)) return NULL;\n mi_page_t** const sub = _mi_page_map[idx];\n if mi_unlikely(sub==NULL) return NULL;\n return sub[sub_idx];\n}\n\nmi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {\n return (_mi_safe_ptr_page(p) != NULL);\n}\n\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/page-queue.c", "sha256": "7ec275805c9f00eb66710ffa99817b3ed1dba9826435d08f4866f7935152f6ae", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/*----------------------------------------------------------------------------\nCopyright (c) 2018-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* -----------------------------------------------------------\n Definition of page queues for each block size\n----------------------------------------------------------- */\n\n#ifndef MI_IN_PAGE_C\n#error \"this file should be included from 'page.c'\"\n// include to help an IDE\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#endif\n\n/* -----------------------------------------------------------\n Minimal alignment in machine words (i.e. `sizeof(void*)`)\n----------------------------------------------------------- */\n\n#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE)\n #error \"define alignment for more than 4x word size for this platform\"\n#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE)\n #define MI_ALIGN4W // 4 machine words minimal alignment\n#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE)\n #define MI_ALIGN2W // 2 machine words minimal alignment\n#else\n // ok, default alignment is 1 word\n#endif\n\n\n/* -----------------------------------------------------------\n Queue query\n----------------------------------------------------------- */\n\n\nstatic inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) {\n return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+sizeof(uintptr_t)));\n}\n\nstatic inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) {\n return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+(2*sizeof(uintptr_t))));\n}\n\nstatic inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) {\n return (pq->block_size > MI_LARGE_MAX_OBJ_SIZE);\n}\n\nstatic inline size_t mi_page_queue_count(const mi_page_queue_t* pq) {\n return pq->count;\n}\n\n/* -----------------------------------------------------------\n Bins\n----------------------------------------------------------- */\n\n// Return the bin for a given field size.\n// Returns MI_BIN_HUGE if the size is too large.\n// We use `wsize` for the size in \"machine word sizes\",\n// i.e. byte size == `wsize*sizeof(void*)`.\nstatic mi_decl_noinline size_t mi_bin(size_t size) {\n size_t wsize = _mi_wsize_from_size(size);\n#if defined(MI_ALIGN4W)\n if mi_likely(wsize <= 4) {\n return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes\n }\n#elif defined(MI_ALIGN2W)\n if mi_likely(wsize <= 8) {\n return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes\n }\n#else\n if mi_likely(wsize <= 8) {\n return (wsize == 0 ? 1 : wsize);\n }\n#endif\n else if mi_unlikely(wsize > MI_LARGE_MAX_OBJ_WSIZE) {\n return MI_BIN_HUGE;\n }\n else {\n #if defined(MI_ALIGN4W)\n if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes\n #endif\n wsize--;\n // find the highest bit\n const size_t b = (MI_SIZE_BITS - 1 - mi_clz(wsize)); // note: wsize != 0\n // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation).\n // - adjust with 3 because we use do not round the first 8 sizes\n // which each get an exact bin\n const size_t bin = ((b << 2) + ((wsize >> (b - 2)) & 0x03)) - 3;\n mi_assert_internal(bin > 0 && bin < MI_BIN_HUGE);\n return bin;\n }\n}\n\n\n\n/* -----------------------------------------------------------\n Queue of pages with free blocks\n----------------------------------------------------------- */\n\nsize_t _mi_bin(size_t size) {\n return mi_bin(size);\n}\n\nsize_t _mi_bin_size(size_t bin) {\n mi_assert_internal(bin <= MI_BIN_HUGE);\n return _mi_theap_empty.pages[bin].block_size;\n}\n\n// Good size for allocation\nmi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept {\n if (size <= MI_LARGE_MAX_OBJ_SIZE) {\n return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE));\n }\n else if (size <= MI_MAX_ALLOC_SIZE) {\n return _mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size());\n }\n else {\n return size;\n }\n}\n\n#if (MI_DEBUG>1)\nstatic bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_page_t* list = queue->first;\n while (list != NULL) {\n mi_assert_internal(list->next == NULL || list->next->prev == list);\n mi_assert_internal(list->prev == NULL || list->prev->next == list);\n if (list == page) break;\n list = list->next;\n }\n return (list == page);\n}\n\n#endif\n\n#if (MI_DEBUG>1)\nstatic bool mi_theap_contains_queue(const mi_theap_t* theap, const mi_page_queue_t* pq) {\n return (pq >= &theap->pages[0] && pq <= &theap->pages[MI_BIN_FULL]);\n}\n#endif\n\nbool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) {\n MI_UNUSED_RELEASE(theap);\n if (pq==NULL) return false;\n size_t count = 0; MI_UNUSED_RELEASE(count);\n mi_page_t* prev = NULL; MI_UNUSED_RELEASE(prev);\n for (mi_page_t* page = pq->first; page != NULL; page = page->next) {\n mi_assert_internal(page->prev == prev);\n if (mi_page_is_in_full(page)) {\n mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2);\n }\n else if (mi_page_is_huge(page)) {\n mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1);\n }\n else {\n mi_assert_internal(mi_page_block_size(page) == pq->block_size);\n }\n mi_assert_internal(page->theap == theap);\n if (page->next == NULL) {\n mi_assert_internal(pq->last == page);\n }\n count++;\n prev = page;\n }\n mi_assert_internal(pq->count == count);\n return true;\n}\n\nstatic size_t mi_page_bin(const mi_page_t* page) {\n const size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page))));\n mi_assert_internal(bin <= MI_BIN_FULL);\n return bin;\n}\n\n// returns the page bin without using MI_BIN_FULL for statistics\nsize_t _mi_page_stats_bin(const mi_page_t* page) {\n const size_t bin = (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)));\n mi_assert_internal(bin <= MI_BIN_HUGE);\n return bin;\n}\n\nstatic mi_page_queue_t* mi_theap_page_queue_of(mi_theap_t* theap, const mi_page_t* page) {\n mi_assert_internal(theap!=NULL);\n const size_t bin = mi_page_bin(page);\n mi_page_queue_t* pq = &theap->pages[bin];\n mi_assert_internal((mi_page_block_size(page) == pq->block_size) ||\n (mi_page_is_huge(page) && mi_page_queue_is_huge(pq)) ||\n (mi_page_is_in_full(page) && mi_page_queue_is_full(pq)));\n return pq;\n}\n\nstatic mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) {\n mi_theap_t* theap = mi_page_theap(page);\n mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);\n mi_assert_expensive(mi_page_queue_contains(pq, page));\n return pq;\n}\n\n// The current small page array is for efficiency and for each\n// small size (up to 256) it points directly to the page for that\n// size without having to compute the bin. This means when the\n// current free page queue is updated for a small bin, we need to update a\n// range of entries in `_mi_page_small_free`.\nstatic inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_queue_t* pq) {\n mi_assert_internal(mi_theap_contains_queue(theap,pq));\n const size_t size = pq->block_size;\n if (size > MI_SMALL_SIZE_MAX) return;\n\n mi_page_t* page = pq->first;\n if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty;\n\n // find index in the right direct page array\n const size_t idx = _mi_wsize_from_size(size);\n mi_page_t** const pages_free = theap->pages_free_direct;\n if (pages_free[idx] == page) return; // already set\n\n // find start slot\n size_t start;\n if (idx<=1) {\n start = 0;\n }\n else {\n // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped\n mi_assert_internal(pq > &theap->pages[0]); // since idx > 1 \n size_t bin = mi_bin(size);\n const mi_page_queue_t* prev = pq - 1;\n while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) {\n prev--;\n }\n start = 1 + _mi_wsize_from_size(prev->block_size);\n if (start > idx) start = idx;\n }\n\n // set size range to the right page\n mi_assert(start <= idx);\n for (size_t sz = start; sz <= idx; sz++) {\n pages_free[sz] = page;\n }\n}\n\n/*\nstatic bool mi_page_queue_is_empty(mi_page_queue_t* queue) {\n return (queue->first == NULL);\n}\n*/\n\nstatic void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_assert_expensive(mi_page_queue_contains(queue, page));\n mi_assert_internal(queue->count >= 1);\n mi_assert_internal(mi_page_block_size(page) == queue->block_size ||\n (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||\n (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));\n mi_theap_t* theap = mi_page_theap(page);\n if (page->prev != NULL) page->prev->next = page->next;\n if (page->next != NULL) page->next->prev = page->prev;\n if (page == queue->last) queue->last = page->prev;\n if (page == queue->first) {\n queue->first = page->next;\n // update first\n mi_assert_internal(mi_theap_contains_queue(theap, queue));\n mi_theap_queue_first_update(theap,queue);\n }\n theap->page_count--;\n queue->count--;\n page->next = NULL;\n page->prev = NULL;\n mi_page_set_in_full(page,false);\n}\n\n\nstatic void mi_page_queue_push(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {\n mi_assert_internal(mi_page_theap(page) == theap);\n mi_assert_internal(!mi_page_queue_contains(queue, page));\n #if MI_HUGE_PAGE_ABANDON\n mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE);\n #endif\n mi_assert_internal(mi_page_block_size(page) == queue->block_size ||\n (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||\n (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));\n\n mi_page_set_in_full(page, mi_page_queue_is_full(queue));\n\n page->next = queue->first;\n page->prev = NULL;\n if (queue->first != NULL) {\n mi_assert_internal(queue->first->prev == NULL);\n queue->first->prev = page;\n queue->first = page;\n }\n else {\n queue->first = queue->last = page;\n }\n queue->count++;\n\n // update direct\n mi_theap_queue_first_update(theap, queue);\n theap->page_count++;\n}\n\nstatic void mi_page_queue_push_at_end(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {\n mi_assert_internal(mi_page_theap(page) == theap);\n mi_assert_internal(!mi_page_queue_contains(queue, page));\n\n mi_assert_internal(mi_page_block_size(page) == queue->block_size ||\n (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) ||\n (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));\n\n mi_page_set_in_full(page, mi_page_queue_is_full(queue));\n\n page->prev = queue->last;\n page->next = NULL;\n if (queue->last != NULL) {\n mi_assert_internal(queue->last->next == NULL);\n queue->last->next = page;\n queue->last = page;\n }\n else {\n queue->first = queue->last = page;\n }\n queue->count++;\n\n // update direct\n if (queue->first == page) {\n mi_theap_queue_first_update(theap, queue);\n }\n theap->page_count++;\n}\n\nstatic void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {\n mi_assert_internal(mi_page_theap(page) == theap);\n mi_assert_internal(mi_page_queue_contains(queue, page));\n if (queue->first == page) return;\n mi_page_queue_remove(queue, page);\n mi_page_queue_push(theap, queue, page);\n mi_assert_internal(queue->first == page);\n}\n\nstatic void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_assert_internal(from->count >= 1);\n mi_assert_expensive(mi_page_queue_contains(from, page));\n mi_assert_expensive(!mi_page_queue_contains(to, page));\n const size_t bsize = mi_page_block_size(page);\n MI_UNUSED(bsize);\n mi_assert_internal((bsize == to->block_size && bsize == from->block_size) ||\n (bsize == to->block_size && mi_page_queue_is_full(from)) ||\n (bsize == from->block_size && mi_page_queue_is_full(to)) ||\n (mi_page_is_huge(page) && mi_page_queue_is_huge(to)) ||\n (mi_page_is_huge(page) && mi_page_queue_is_full(to)));\n\n mi_theap_t* theap = mi_page_theap(page);\n\n // delete from `from`\n if (page->prev != NULL) page->prev->next = page->next;\n if (page->next != NULL) page->next->prev = page->prev;\n if (page == from->last) from->last = page->prev;\n if (page == from->first) {\n from->first = page->next;\n // update first\n mi_assert_internal(mi_theap_contains_queue(theap, from));\n mi_theap_queue_first_update(theap, from);\n }\n from->count--;\n\n // insert into `to`\n to->count++;\n if (enqueue_at_end) {\n // enqueue at the end\n page->prev = to->last;\n page->next = NULL;\n if (to->last != NULL) {\n mi_assert_internal(theap == mi_page_theap(to->last));\n to->last->next = page;\n to->last = page;\n }\n else {\n to->first = page;\n to->last = page;\n mi_theap_queue_first_update(theap, to);\n }\n }\n else {\n if (to->first != NULL) {\n // enqueue at 2nd place\n mi_assert_internal(theap == mi_page_theap(to->first));\n mi_page_t* next = to->first->next;\n page->prev = to->first;\n page->next = next;\n to->first->next = page;\n if (next != NULL) {\n next->prev = page;\n }\n else {\n to->last = page;\n }\n }\n else {\n // enqueue at the head (singleton list)\n page->prev = NULL;\n page->next = NULL;\n to->first = page;\n to->last = page;\n mi_theap_queue_first_update(theap, to);\n }\n }\n\n mi_page_set_in_full(page, mi_page_queue_is_full(to));\n}\n\nstatic void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {\n mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end */, page);\n}\n\nstatic void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {\n // note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`)\n mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page);\n}\n\n// Only called from `mi_theap_absorb`.\nsize_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append) {\n mi_assert_internal(mi_theap_contains_queue(theap,pq));\n mi_assert_internal(pq->block_size == append->block_size);\n\n if (append->first==NULL) return 0;\n\n // set append pages to new theap and count\n size_t count = 0;\n for (mi_page_t* page = append->first; page != NULL; page = page->next) {\n mi_page_set_theap(page, theap);\n count++;\n }\n mi_assert_internal(count == append->count);\n\n if (pq->last==NULL) {\n // take over afresh\n mi_assert_internal(pq->first==NULL);\n pq->first = append->first;\n pq->last = append->last;\n mi_theap_queue_first_update(theap, pq);\n }\n else {\n // append to end\n mi_assert_internal(pq->last!=NULL);\n mi_assert_internal(append->first!=NULL);\n pq->last->next = append->first;\n append->first->prev = pq->last;\n pq->last = append->last;\n }\n pq->count += append->count;\n\n return count;\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/page.c", "sha256": "d3614a59f2407a32e75b051deb1aff31b233f587d325dcc5420ba508551b687e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/*----------------------------------------------------------------------------\nCopyright (c) 2018-2024, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* -----------------------------------------------------------\n The core of the allocator. Every segment contains\n pages of a certain block size. The main function\n exported is `mi_malloc_generic`.\n----------------------------------------------------------- */\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\"\n\n/* -----------------------------------------------------------\n Definition of page queues for each block size\n----------------------------------------------------------- */\n\n#define MI_IN_PAGE_C\n#include \"page-queue.c\"\n#undef MI_IN_PAGE_C\n\n\n/* -----------------------------------------------------------\n Page helpers\n----------------------------------------------------------- */\n\n// Index a block in a page\nstatic inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t block_size, size_t i) {\n MI_UNUSED(page);\n mi_assert_internal(page != NULL);\n mi_assert_internal(i <= page->reserved);\n return (mi_block_t*)((uint8_t*)page_start + (i * block_size));\n}\n\nstatic bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page);\n\n#if (MI_DEBUG>=3)\nstatic size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) {\n mi_assert_internal(_mi_ptr_page(page->page_start) == page);\n const uint8_t* slice_start = mi_page_slice_start(page);\n mi_assert_internal(_mi_is_aligned(slice_start,MI_PAGE_ALIGN));\n size_t count = 0;\n while (head != NULL) {\n mi_assert_internal((uint8_t*)head - slice_start > (ptrdiff_t)MI_LARGE_PAGE_SIZE || page == _mi_ptr_page(head));\n count++;\n head = mi_block_next(page, head);\n }\n return count;\n}\n\n/*\n// Start of the page available memory\nstatic inline uint8_t* mi_page_area(const mi_page_t* page) {\n return _mi_page_start(_mi_page_segment(page), page, NULL);\n}\n*/\n\nstatic bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) {\n size_t psize;\n uint8_t* page_area = mi_page_area(page, &psize);\n mi_block_t* start = (mi_block_t*)page_area;\n mi_block_t* end = (mi_block_t*)(page_area + psize);\n while(p != NULL) {\n if (p < start || p >= end) return false;\n p = mi_block_next(page, p);\n }\n#if MI_DEBUG>3 // generally too expensive to check this\n if (page->free_is_zero) {\n const size_t ubsize = mi_page_usable_block_size(page);\n for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) {\n mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t)));\n }\n }\n#endif\n return true;\n}\n\nstatic bool mi_page_is_valid_init(mi_page_t* page) {\n mi_assert_internal(mi_page_block_size(page) > 0);\n mi_assert_internal(page->used <= page->capacity);\n mi_assert_internal(page->capacity <= page->reserved);\n\n // const size_t bsize = mi_page_block_size(page);\n // uint8_t* start = mi_page_start(page);\n //mi_assert_internal(start + page->capacity*page->block_size == page->top);\n\n mi_assert_internal(mi_page_list_is_valid(page,page->free));\n mi_assert_internal(mi_page_list_is_valid(page,page->local_free));\n\n #if MI_DEBUG>3 // generally too expensive to check this\n if (page->free_is_zero) {\n const size_t ubsize = mi_page_usable_block_size(page);\n for(mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) {\n mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t)));\n }\n }\n #endif\n\n #if !MI_TRACK_ENABLED && !MI_TSAN\n mi_block_t* tfree = mi_page_thread_free(page);\n mi_assert_internal(mi_page_list_is_valid(page, tfree));\n //size_t tfree_count = mi_page_list_count(page, tfree);\n //mi_assert_internal(tfree_count <= page->thread_freed + 1);\n #endif\n\n size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free);\n mi_assert_internal(page->used + free_count == page->capacity);\n\n return true;\n}\n\nextern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called?\n\nbool _mi_page_is_valid(mi_page_t* page) {\n mi_assert_internal(mi_page_is_valid_init(page));\n #if MI_SECURE\n mi_assert_internal(page->keys[0] != 0);\n #endif\n if (!mi_page_is_abandoned(page)) {\n //mi_assert_internal(!_mi_process_is_initialized);\n {\n mi_page_queue_t* pq = mi_page_queue_of(page);\n mi_assert_internal(mi_page_queue_contains(pq, page));\n mi_assert_internal(pq->block_size==mi_page_block_size(page) || mi_page_is_huge(page) || mi_page_is_in_full(page));\n // mi_assert_internal(mi_theap_contains_queue(mi_page_theap(page),pq));\n }\n }\n return true;\n}\n#endif\n\n\n/* -----------------------------------------------------------\n Page collect the `local_free` and `thread_free` lists\n----------------------------------------------------------- */\n\nstatic void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head)\n{\n if (head == NULL) return;\n\n // find the last block in the list -- also to get a proper use count (without data races)\n size_t max_count = page->capacity; // cannot collect more than capacity\n size_t count = 1;\n mi_block_t* last = head;\n mi_block_t* next;\n while ((next = mi_block_next(page, last)) != NULL && count <= max_count) {\n count++;\n last = next;\n }\n\n // if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free)\n if mi_unlikely(count > max_count) {\n _mi_error_message(EFAULT, \"corrupted thread-free list\\n\");\n return; // the thread-free items cannot be freed\n }\n // if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list)\n else if mi_unlikely(count > page->used) {\n _mi_error_message(EFAULT, \"corrupted meta-data in thread-free list\\n\");\n return; // the thread-free items cannot be freed\n }\n\n // and append the current local free list\n mi_block_set_next(page, last, page->local_free);\n page->local_free = head;\n\n // update counts now\n mi_assert_internal(count <= UINT16_MAX);\n mi_assert_internal(page->used >= (uint16_t)count);\n page->used = page->used - (uint16_t)count;\n}\n\n// Collect the local `thread_free` list using an atomic exchange.\nstatic void mi_page_thread_free_collect(mi_page_t* page)\n{\n // atomically capture the thread free list\n mi_block_t* head;\n mi_thread_free_t tfreex;\n mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free);\n do {\n head = mi_tf_block(tfree);\n if mi_likely(head == NULL) return; // return if the list is empty\n tfreex = mi_tf_create(NULL,mi_tf_is_owned(tfree)); // set the thread free list to NULL\n } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex)); // release is enough?\n mi_assert_internal(head != NULL);\n\n // and move it to the local list\n mi_page_thread_collect_to_local(page, head);\n}\n\n// returns `true` if after collection `mi_page_immediate_available` is true.\nstatic bool mi_page_free_quick_collect(mi_page_t* page) {\n if (page->free != NULL) return true;\n if (page->local_free == NULL) return false;\n // move local_free to free\n page->free = page->local_free;\n page->local_free = NULL;\n page->free_is_zero = false;\n return true;\n}\n\nvoid _mi_page_free_collect(mi_page_t* page, bool force) {\n mi_assert_internal(page!=NULL);\n\n // collect the thread free list\n mi_page_thread_free_collect(page);\n\n // and the local free list\n if (page->local_free != NULL) {\n if mi_likely(page->free == NULL) {\n // usual case\n page->free = page->local_free;\n page->local_free = NULL;\n page->free_is_zero = false;\n }\n else if (force) {\n // append -- only on shutdown (force) as this is a linear operation\n mi_block_t* tail = page->local_free;\n mi_block_t* next;\n while ((next = mi_block_next(page, tail)) != NULL) {\n tail = next;\n }\n mi_block_set_next(page, tail, page->free);\n page->free = page->local_free;\n page->local_free = NULL;\n page->free_is_zero = false;\n }\n }\n\n mi_assert_internal(!force || page->local_free == NULL);\n}\n\n// Collect elements in the thread-free list starting at `head`. This is an optimized\n// version of `_mi_page_free_collect` to be used from `free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`.\n//\n// `head` must be in the `xthread_free` list. It will not collect `head` itself\n// so the `used` count is not fully updated in general. However, if the `head` is\n// the last remaining element, it will be collected and the used count will become `0` (so `mi_page_all_free` becomes true).\nvoid _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {\n if (head == NULL) return;\n mi_block_t* next = mi_block_next(page,head); // we cannot collect the head element itself as `page->thread_free` may point to it (and we want to avoid atomic ops)\n if (next != NULL) {\n mi_block_set_next(page, head, NULL);\n mi_page_thread_collect_to_local(page, next);\n if (page->local_free != NULL && page->free == NULL) {\n page->free = page->local_free;\n page->local_free = NULL;\n page->free_is_zero = false;\n }\n }\n if (page->used == 1) {\n // all elements are free'd since we skipped the `head` element itself\n mi_assert_internal(mi_tf_block(mi_atomic_load_relaxed(&page->xthread_free)) == head);\n mi_assert_internal(mi_block_next(page,head) == NULL);\n _mi_page_free_collect(page, false); // collect the final element\n }\n}\n\n\n/* -----------------------------------------------------------\n Page fresh and retire\n----------------------------------------------------------- */\n\n/*\n// called from segments when reclaiming abandoned pages\nvoid _mi_page_reclaim(mi_theap_t* theap, mi_page_t* page) {\n // mi_page_set_theap(page, theap);\n // _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after theap is set)\n _mi_page_free_collect(page, false); // ensure used count is up to date\n\n mi_assert_expensive(mi_page_is_valid_init(page));\n // mi_assert_internal(mi_page_theap(page) == theap);\n // mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE);\n\n // TODO: push on full queue immediately if it is full?\n mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);\n mi_page_queue_push(theap, pq, page);\n mi_assert_expensive(_mi_page_is_valid(page));\n}\n*/\n\n// called from `mi_free` on a reclaim, and fresh_alloc if we get an abandoned page\nvoid _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page)\n{\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);\n mi_assert_internal(mi_page_is_owned(page));\n mi_assert_internal(mi_page_is_abandoned(page));\n\n mi_page_set_theap(page,theap);\n _mi_page_free_collect(page, false); // ensure used count is up to date\n mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);\n mi_page_queue_push_at_end(theap, pq, page);\n mi_assert_expensive(_mi_page_is_valid(page));\n}\n\nvoid _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {\n _mi_page_free_collect(page, false); // ensure used count is up to date\n if (mi_page_all_free(page)) {\n _mi_page_free(page, pq);\n }\n else {\n mi_page_queue_remove(pq, page);\n mi_theap_t* theap = page->theap;\n mi_page_set_theap(page, NULL);\n page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap\n _mi_arenas_page_abandon(page, theap);\n _mi_arenas_collect(false, false, theap->tld); // allow purging\n }\n}\n\n\n// allocate a fresh page from an arena\nstatic mi_page_t* mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, size_t block_size, size_t page_alignment) {\n #if !MI_HUGE_PAGE_ABANDON\n mi_assert_internal(pq != NULL);\n mi_assert_internal(mi_theap_contains_queue(theap, pq));\n mi_assert_internal(page_alignment > 0 || block_size > MI_LARGE_MAX_OBJ_SIZE || block_size == pq->block_size);\n #endif\n mi_page_t* page = _mi_arenas_page_alloc(theap, block_size, page_alignment);\n if (page == NULL) {\n // out-of-memory\n return NULL;\n }\n if (mi_page_is_abandoned(page)) {\n _mi_theap_page_reclaim(theap, page);\n if (!mi_page_immediate_available(page)) {\n if (mi_page_is_expandable(page)) {\n if (!mi_page_extend_free(theap, page)) {\n return NULL; // cannot commit\n };\n }\n else {\n mi_assert(false); // should not happen?\n return NULL;\n }\n }\n }\n else if (pq != NULL) {\n mi_page_queue_push(theap, pq, page);\n }\n mi_assert_internal(pq!=NULL || mi_page_block_size(page) >= block_size);\n mi_assert_expensive(_mi_page_is_valid(page));\n return page;\n}\n\n// Get a fresh page to use\nstatic mi_page_t* mi_page_fresh(mi_theap_t* theap, mi_page_queue_t* pq) {\n mi_assert_internal(mi_theap_contains_queue(theap, pq));\n mi_page_t* page = mi_page_fresh_alloc(theap, pq, pq->block_size, 0);\n if (page==NULL) return NULL;\n mi_assert_internal(pq->block_size==mi_page_block_size(page));\n mi_assert_internal(pq==mi_theap_page_queue_of(theap, page));\n return page;\n}\n\n\n/* -----------------------------------------------------------\n Unfull, abandon, free and retire\n----------------------------------------------------------- */\n\n// Move a page from the full list back to a regular list (called from thread-local mi_free)\nvoid _mi_page_unfull(mi_page_t* page) {\n mi_assert_internal(page != NULL);\n mi_assert_expensive(_mi_page_is_valid(page));\n mi_assert_internal(mi_page_is_in_full(page));\n mi_assert_internal(!mi_page_theap(page)->allow_page_abandon);\n if (!mi_page_is_in_full(page)) return;\n\n mi_theap_t* theap = mi_page_theap(page);\n mi_page_queue_t* pqfull = &theap->pages[MI_BIN_FULL];\n mi_page_set_in_full(page, false); // to get the right queue\n mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);\n mi_page_set_in_full(page, true);\n mi_page_queue_enqueue_from_full(pq, pqfull, page);\n}\n\nstatic void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {\n mi_assert_internal(pq == mi_page_queue_of(page));\n mi_assert_internal(!mi_page_immediate_available(page));\n mi_assert_internal(!mi_page_is_in_full(page));\n\n mi_theap_t* theap = mi_page_theap(page);\n if (theap->allow_page_abandon) {\n // abandon full pages (this is the usual case in order to allow for sharing of memory between theaps)\n _mi_page_abandon(page, pq);\n }\n else if (!mi_page_is_in_full(page)) {\n // put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed)\n mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page);\n _mi_page_free_collect(page, false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set\n }\n}\n\n\n// Free a page with no more free blocks\nvoid _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) {\n mi_assert_internal(page != NULL);\n mi_assert_expensive(_mi_page_is_valid(page));\n mi_assert_internal(pq == mi_page_queue_of(page));\n mi_assert_internal(mi_page_all_free(page));\n // mi_assert_internal(mi_page_thread_free_flag(page)!=MI_DELAYED_FREEING);\n\n // no more aligned blocks in here\n mi_page_set_has_interior_pointers(page, false);\n\n // remove from the page list\n // (no need to do _mi_theap_delayed_free first as all blocks are already free)\n mi_page_queue_remove(pq, page);\n\n // and free it\n mi_theap_t* theap = mi_page_theap(page); mi_assert_internal(theap!=NULL);\n mi_page_set_theap(page,NULL);\n _mi_arenas_page_free(page, theap);\n _mi_arenas_collect(false, false, theap->tld); // allow purging\n}\n\n#define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX // should be less than size for MI_BIN_HUGE\n#define MI_RETIRE_CYCLES (16)\n\n// Retire a page with no more used blocks\n// Important to not retire too quickly though as new\n// allocations might coming.\n//\n// Note: called from `mi_free` and benchmarks often\n// trigger this due to freeing everything and then\n// allocating again so careful when changing this.\nvoid _mi_page_retire(mi_page_t* page) mi_attr_noexcept {\n mi_assert_internal(page != NULL);\n mi_assert_expensive(_mi_page_is_valid(page));\n mi_assert_internal(mi_page_all_free(page));\n\n if (page->retire_expire!=0) return; // already retired, just keep it retired\n mi_page_set_has_interior_pointers(page, false);\n\n // don't retire too often..\n // (or we end up retiring and re-allocating most of the time)\n // NOTE: refine this more: we should not retire if this\n // is the only page left with free blocks. It is not clear\n // how to check this efficiently though...\n // for now, we don't retire if it is the only page left of this size class.\n mi_page_queue_t* pq = mi_page_queue_of(page);\n #if MI_RETIRE_CYCLES > 0\n const size_t bsize = mi_page_block_size(page);\n if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue?\n if (pq->last==page && pq->first==page) { // the only page in the queue?\n mi_theap_t* theap = mi_page_theap(page);\n #if MI_STAT>0\n mi_theap_stat_counter_increase(theap, pages_retire, 1);\n #endif\n page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4);\n mi_assert_internal(pq >= theap->pages);\n const size_t index = pq - theap->pages;\n mi_assert_internal(index < MI_BIN_FULL && index < MI_BIN_HUGE);\n if (index < theap->page_retired_min) theap->page_retired_min = index;\n if (index > theap->page_retired_max) theap->page_retired_max = index;\n mi_assert_internal(mi_page_all_free(page));\n return; // don't free after all\n }\n }\n #endif\n _mi_page_free(page, pq);\n}\n\n// free retired pages: we don't need to look at the entire queues\n// since we only retire pages that are at the head position in a queue.\nvoid _mi_theap_collect_retired(mi_theap_t* theap, bool force) {\n size_t min = MI_BIN_FULL;\n size_t max = 0;\n for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) {\n mi_page_queue_t* pq = &theap->pages[bin];\n mi_page_t* page = pq->first;\n if (page != NULL && page->retire_expire != 0) {\n if (mi_page_all_free(page)) {\n page->retire_expire--;\n if (page->retire_expire == 0 || force) {\n _mi_page_free(page, pq);\n }\n else {\n // keep retired, update min/max\n if (bin < min) min = bin;\n if (bin > max) max = bin;\n }\n }\n else {\n page->retire_expire = 0;\n }\n }\n }\n theap->page_retired_min = min;\n theap->page_retired_max = max;\n}\n\n/*\nstatic void mi_theap_collect_full_pages(mi_theap_t* theap) {\n // note: normally full pages get immediately abandoned and the full queue is always empty\n // this path is only used if abandoning is disabled due to a destroy-able theap or options\n // set by the user.\n mi_page_queue_t* pq = &theap->pages[MI_BIN_FULL];\n for (mi_page_t* page = pq->first; page != NULL; ) {\n mi_page_t* next = page->next; // get next in case we free the page\n _mi_page_free_collect(page, false); // register concurrent free's\n // no longer full?\n if (!mi_page_is_full(page)) {\n if (mi_page_all_free(page)) {\n _mi_page_free(page, pq);\n }\n else {\n _mi_page_unfull(page);\n }\n }\n page = next;\n }\n}\n*/\n\n\n/* -----------------------------------------------------------\n Initialize the initial free list in a page.\n In secure mode we initialize a randomized list by\n alternating between slices.\n----------------------------------------------------------- */\n\n#define MI_MAX_SLICE_SHIFT (6) // at most 64 slices\n#define MI_MAX_SLICES (1UL << MI_MAX_SLICE_SHIFT)\n#define MI_MIN_SLICES (2)\n\nstatic void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* const page, const size_t bsize, const size_t extend) {\n #if (MI_SECURE<3)\n mi_assert_internal(page->free == NULL);\n mi_assert_internal(page->local_free == NULL);\n #endif\n mi_assert_internal(page->capacity + extend <= page->reserved);\n mi_assert_internal(bsize == mi_page_block_size(page));\n void* const page_area = mi_page_start(page);\n\n // initialize a randomized free list\n // set up `slice_count` slices to alternate between\n size_t shift = MI_MAX_SLICE_SHIFT;\n while ((extend >> shift) == 0) {\n shift--;\n }\n const size_t slice_count = (size_t)1U << shift;\n const size_t slice_extend = extend / slice_count;\n mi_assert_internal(slice_extend >= 1);\n mi_block_t* blocks[MI_MAX_SLICES]; // current start of the slice\n size_t counts[MI_MAX_SLICES]; // available objects in the slice\n for (size_t i = 0; i < slice_count; i++) {\n blocks[i] = mi_page_block_at(page, page_area, bsize, page->capacity + i*slice_extend);\n counts[i] = slice_extend;\n }\n counts[slice_count-1] += (extend % slice_count); // final slice holds the modulus too (todo: distribute evenly?)\n\n // and initialize the free list by randomly threading through them\n // set up first element\n const uintptr_t r = _mi_theap_random_next(theap);\n size_t current = r % slice_count;\n counts[current]--;\n mi_block_t* const free_start = blocks[current];\n // and iterate through the rest; use `random_shuffle` for performance\n uintptr_t rnd = _mi_random_shuffle(r|1); // ensure not 0\n for (size_t i = 1; i < extend; i++) {\n // call random_shuffle only every INTPTR_SIZE rounds\n const size_t round = i%MI_INTPTR_SIZE;\n if (round == 0) rnd = _mi_random_shuffle(rnd);\n // select a random next slice index\n size_t next = ((rnd >> 8*round) & (slice_count-1));\n while (counts[next]==0) { // ensure it still has space\n next++;\n if (next==slice_count) next = 0;\n }\n // and link the current block to it\n counts[next]--;\n mi_block_t* const block = blocks[current];\n blocks[current] = (mi_block_t*)((uint8_t*)block + bsize); // bump to the following block\n mi_block_set_next(page, block, blocks[next]); // and set next; note: we may have `current == next`\n current = next;\n }\n // prepend to the free list (usually NULL)\n mi_block_set_next(page, blocks[current], page->free); // end of the list\n page->free = free_start;\n}\n\nstatic mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend)\n{\n #if (MI_SECURE<3)\n mi_assert_internal(page->free == NULL);\n mi_assert_internal(page->local_free == NULL);\n #endif\n mi_assert_internal(page->capacity + extend <= page->reserved);\n mi_assert_internal(bsize == mi_page_block_size(page));\n void* const page_area = mi_page_start(page);\n\n mi_block_t* const start = mi_page_block_at(page, page_area, bsize, page->capacity);\n\n // initialize a sequential free list\n mi_block_t* const last = mi_page_block_at(page, page_area, bsize, page->capacity + extend - 1);\n mi_block_t* block = start;\n while(block <= last) {\n mi_block_t* next = (mi_block_t*)((uint8_t*)block + bsize);\n mi_block_set_next(page,block,next);\n block = next;\n }\n // prepend to free list (usually `NULL`)\n mi_block_set_next(page, last, page->free);\n page->free = start;\n}\n\n/* -----------------------------------------------------------\n Page initialize and extend the capacity\n----------------------------------------------------------- */\n\n#define MI_MAX_EXTEND_SIZE (4*1024) // heuristic, one OS page seems to work well.\n#if (MI_SECURE>=2)\n#define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many\n#else\n#define MI_MIN_EXTEND (1)\n#endif\n\n// Extend the capacity (up to reserved) by initializing a free list\n// We do at most `MI_MAX_EXTEND` to avoid touching too much memory\n// Note: we also experimented with \"bump\" allocation on the first\n// allocations but this did not speed up any benchmark (due to an\n// extra test in malloc? or cache effects?)\nstatic bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {\n mi_assert_expensive(mi_page_is_valid_init(page));\n #if (MI_SECURE<3)\n mi_assert(page->free == NULL);\n mi_assert(page->local_free == NULL);\n if (page->free != NULL) return true;\n #endif\n if (page->capacity >= page->reserved) return true;\n\n size_t page_size;\n //uint8_t* page_start =\n mi_page_area(page, &page_size);\n #if MI_STAT>0\n mi_theap_stat_counter_increase(theap, pages_extended, 1);\n #endif\n\n // calculate the extend count\n const size_t bsize = mi_page_block_size(page);\n size_t extend = (size_t)page->reserved - page->capacity;\n mi_assert_internal(extend > 0);\n\n size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE/bsize);\n if (max_extend < MI_MIN_EXTEND) { max_extend = MI_MIN_EXTEND; }\n mi_assert_internal(max_extend > 0);\n\n if (extend > max_extend) {\n // ensure we don't touch memory beyond the page to reduce page commit.\n // the `lean` benchmark tests this. Going from 1 to 8 increases rss by 50%.\n extend = max_extend;\n }\n\n mi_assert_internal(extend > 0 && extend + page->capacity <= page->reserved);\n mi_assert_internal(extend < (1UL<<16));\n\n // commit on demand?\n if (page->slice_committed > 0) {\n const size_t needed_size = (page->capacity + extend)*bsize;\n const size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), MI_PAGE_MIN_COMMIT_SIZE );\n if (needed_commit > page->slice_committed) {\n mi_assert_internal(((needed_commit - page->slice_committed) % _mi_os_page_size()) == 0);\n if (!_mi_os_commit(mi_page_slice_start(page) + page->slice_committed, needed_commit - page->slice_committed, NULL)) {\n return false;\n }\n page->slice_committed = needed_commit;\n }\n }\n\n // and append the extend the free list\n if (extend < MI_MIN_SLICES || MI_SECURE<2) { //!mi_option_is_enabled(mi_option_secure)) {\n mi_page_free_list_extend(page, bsize, extend );\n }\n else {\n mi_page_free_list_extend_secure(theap, page, bsize, extend);\n }\n // enable the new free list\n page->capacity += (uint16_t)extend;\n #if MI_STAT>0\n mi_theap_stat_increase(theap, page_committed, extend * bsize);\n #endif\n mi_assert_expensive(mi_page_is_valid_init(page));\n return true;\n}\n\n// Initialize a fresh page (that is already partially initialized)\nmi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) {\n mi_assert(page != NULL);\n mi_assert(theap!=NULL);\n page->heap = (_mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap`\n mi_page_set_theap(page, theap);\n\n size_t page_size;\n uint8_t* page_start = mi_page_area(page, &page_size); MI_UNUSED(page_start);\n mi_track_mem_noaccess(page_start,page_size);\n mi_assert_internal(page_size / mi_page_block_size(page) < (1L<<16));\n mi_assert_internal(page->reserved > 0);\n #if (MI_PADDING || MI_ENCODE_FREELIST)\n page->keys[0] = _mi_theap_random_next(theap);\n page->keys[1] = _mi_theap_random_next(theap);\n #endif\n #if MI_DEBUG>2\n if (page->memid.initially_zero) {\n mi_track_mem_defined(page->page_start, mi_page_committed(page));\n mi_assert_expensive(mi_mem_is_zero(page_start, mi_page_committed(page)));\n }\n #endif\n\n mi_assert_internal(page->theap!=NULL);\n mi_assert_internal(page->theap == mi_page_theap(page));\n mi_assert_internal(page->capacity == 0);\n mi_assert_internal(page->free == NULL);\n mi_assert_internal(page->used == 0);\n mi_assert_internal(mi_page_is_owned(page));\n mi_assert_internal(page->xthread_free == 1);\n mi_assert_internal(page->next == NULL);\n mi_assert_internal(page->prev == NULL);\n mi_assert_internal(page->retire_expire == 0);\n mi_assert_internal(!mi_page_has_interior_pointers(page));\n #if (MI_PADDING || MI_ENCODE_FREELIST)\n mi_assert_internal(page->keys[0] != 0);\n mi_assert_internal(page->keys[1] != 0);\n #endif\n mi_assert_expensive(mi_page_is_valid_init(page));\n\n // initialize an initial free list\n if (!mi_page_extend_free(theap,page)) return false;\n mi_assert(mi_page_immediate_available(page));\n return true;\n}\n\n\n/* -----------------------------------------------------------\n Find pages with free blocks\n-------------------------------------------------------------*/\n\n// Find a page with free blocks of `page->block_size`.\nstatic mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_page_queue_t* pq, bool first_try)\n{\n // search through the pages in \"next fit\" order\n size_t count = 0;\n long candidate_limit = 0; // we reset this on the first candidate to limit the search\n long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pages\n mi_page_t* page_candidate = NULL; // a page with free space\n mi_page_t* page = pq->first;\n\n while (page != NULL)\n {\n mi_page_t* next = page->next; // remember next (as this page can move to another queue)\n count++;\n candidate_limit--;\n\n // search up to N pages for a best candidate\n\n // is the local free list non-empty?\n bool immediate_available = mi_page_immediate_available(page);\n if (!immediate_available) {\n // collect freed blocks by us and other threads to we get a proper use count\n _mi_page_free_collect(page, false);\n immediate_available = mi_page_immediate_available(page);\n }\n\n // if the page is completely full, move it to the `mi_pages_full`\n // queue so we don't visit long-lived pages too often.\n if (!immediate_available && !mi_page_is_expandable(page)) {\n page_full_retain--;\n if (page_full_retain < 0) {\n mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));\n mi_page_to_full(page, pq);\n }\n }\n else {\n // the page has free space, make it a candidate\n // we prefer non-expandable pages with high usage as candidates (to reduce commit, and increase chances of free-ing up pages)\n if (page_candidate == NULL) {\n page_candidate = page;\n candidate_limit = _mi_option_get_fast(mi_option_page_max_candidates);\n }\n else if (mi_page_all_free(page_candidate)) {\n _mi_page_free(page_candidate, pq);\n page_candidate = page;\n }\n // prefer to reuse fuller pages (in the hope the less used page gets freed)\n else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) {\n page_candidate = page;\n }\n // if we find a non-expandable candidate, or searched for N pages, return with the best candidate\n if (immediate_available || candidate_limit <= 0) {\n mi_assert_internal(page_candidate!=NULL);\n break;\n }\n }\n\n #if 0\n // first-fit algorithm without candidates\n // If the page contains free blocks, we are done\n if (mi_page_immediate_available(page) || mi_page_is_expandable(page)) {\n break; // pick this one\n }\n\n // If the page is completely full, move it to the `mi_pages_full`\n // queue so we don't visit long-lived pages too often.\n mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));\n mi_page_to_full(page, pq);\n #endif\n\n page = next;\n } // for each page\n\n mi_theap_stat_counter_increase(theap, page_searches, count);\n mi_theap_stat_counter_increase(theap, page_searches_count, 1);\n\n // set the page to the best candidate\n if (page_candidate != NULL) {\n page = page_candidate;\n }\n if (page != NULL) {\n if (!mi_page_immediate_available(page)) {\n mi_assert_internal(mi_page_is_expandable(page));\n if (!mi_page_extend_free(theap, page)) {\n page = NULL; // failed to extend\n }\n }\n mi_assert_internal(page == NULL || mi_page_immediate_available(page));\n }\n\n if (page == NULL) {\n _mi_theap_collect_retired(theap, false); // perhaps make a page available\n page = mi_page_fresh(theap, pq);\n mi_assert_internal(page == NULL || mi_page_immediate_available(page));\n if (page == NULL && first_try) {\n // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again\n page = mi_page_queue_find_free_ex(theap, pq, false);\n mi_assert_internal(page == NULL || mi_page_immediate_available(page));\n }\n }\n else {\n mi_assert_internal(page == NULL || mi_page_immediate_available(page));\n // move the page to the front of the queue\n mi_page_queue_move_to_front(theap, pq, page);\n page->retire_expire = 0;\n // _mi_theap_collect_retired(theap, false); // update retire counts; note: increases rss on MemoryLoad bench so don't do this\n }\n mi_assert_internal(page == NULL || mi_page_immediate_available(page));\n\n\n return page;\n}\n\n\n\n// Find a page with free blocks of `size`.\nstatic mi_page_t* mi_find_free_page(mi_theap_t* theap, mi_page_queue_t* pq) {\n // mi_page_queue_t* pq = mi_page_queue(theap, size);\n mi_assert_internal(!mi_page_queue_is_huge(pq));\n\n // check the first page: we even do this with candidate search or otherwise we re-search every time\n mi_page_t* page = pq->first;\n if mi_likely(page != NULL && mi_page_free_quick_collect(page)) {\n #if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness\n if (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) {\n (void)mi_page_extend_free(theap, page); // ok if this fails\n mi_assert_internal(mi_page_immediate_available(page));\n }\n #endif\n page->retire_expire = 0;\n return page; // fast path\n }\n else {\n return mi_page_queue_find_free_ex(theap, pq, true);\n }\n}\n\n\n/* -----------------------------------------------------------\n Users can register a deferred free function called\n when the `free` list is empty. Since the `local_free`\n is separate this is deterministically called after\n a certain number of allocations.\n----------------------------------------------------------- */\n\n// The program should only install a single deferred free handler before doing allocation.\nstatic _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers)\nstatic _Atomic(void*) deferred_arg; \n\nvoid _mi_deferred_free(mi_theap_t* theap, bool force) {\n theap->heartbeat++;\n mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free);\n if (fun != NULL && !theap->tld->recurse) {\n theap->tld->recurse = true;\n void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg); \n fun(force, theap->heartbeat, arg);\n theap->tld->recurse = false;\n }\n}\n\nvoid mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept {\n mi_atomic_store_ptr_release(void,&deferred_arg, arg);\n mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn); \n}\n\n\n/* -----------------------------------------------------------\n General allocation\n----------------------------------------------------------- */\n\n// Huge pages contain just one block, and the segment contains just that page.\n// Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX)\n// so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`.\nstatic mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page_alignment, mi_page_queue_t* pq) {\n const size_t block_size = _mi_os_good_alloc_size(size);\n // mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0);\n #if MI_HUGE_PAGE_ABANDON\n #error todo.\n #else\n // mi_page_queue_t* pq = mi_page_queue(theap, MI_LARGE_MAX_OBJ_SIZE+1); // always in the huge queue regardless of the block size\n mi_assert_internal(mi_page_queue_is_huge(pq));\n #endif\n mi_page_t* page = mi_page_fresh_alloc(theap, pq, block_size, page_alignment);\n if (page != NULL) {\n mi_assert_internal(mi_page_block_size(page) >= size);\n mi_assert_internal(mi_page_immediate_available(page));\n mi_assert_internal(mi_page_is_huge(page));\n mi_assert_internal(mi_page_is_singleton(page));\n #if MI_HUGE_PAGE_ABANDON\n mi_assert_internal(mi_page_is_abandoned(page));\n mi_page_set_theap(page, NULL);\n #endif\n mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page));\n mi_theap_stat_counter_increase(theap, malloc_huge_count, 1);\n }\n return page;\n}\n\n\n// Allocate a page\n// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.\nstatic mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept {\n const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`\n if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) {\n _mi_error_message(EOVERFLOW, \"allocation request is too large (%zu bytes)\\n\", req_size);\n return NULL;\n }\n mi_page_queue_t* pq = mi_page_queue(theap, (huge_alignment > 0 ? MI_LARGE_MAX_OBJ_SIZE+1 : size));\n // huge allocation?\n if mi_unlikely(mi_page_queue_is_huge(pq) || req_size > MI_MAX_ALLOC_SIZE) {\n return mi_huge_page_alloc(theap,size,huge_alignment,pq);\n }\n else {\n // otherwise find a page with free blocks in our size segregated queues\n #if MI_PADDING\n mi_assert_internal(size >= MI_PADDING_SIZE);\n #endif\n return mi_find_free_page(theap, pq);\n }\n}\n\n\n// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed.\n// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.\n// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for\n// very large requested alignments in which case we use a huge singleton page.\n// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit)\n// to use 4 parameters which compiles better on msvc for the malloc fast path.\nvoid* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept\n{\n const bool zero = ((zero_huge_alignment & 1) != 0);\n const size_t huge_alignment = (zero_huge_alignment & ~1);\n\n #if !MI_THEAP_INITASNULL\n mi_assert_internal(theap != NULL);\n #endif\n\n // initialize if necessary\n if mi_unlikely(!mi_theap_is_initialized(theap)) {\n if (theap==&_mi_theap_empty_wrong) {\n // we were unable to allocate a theap for a first-class heap\n return NULL;\n }\n // otherwise we initialize the thread and its default theap\n mi_thread_init();\n theap = _mi_theap_default();\n if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; }\n mi_assert_internal(_mi_theap_default()==theap);\n }\n mi_assert_internal(mi_theap_is_initialized(theap));\n\n // do administrative tasks every N generic mallocs\n if mi_unlikely(++theap->generic_count >= 1000) {\n theap->generic_collect_count += theap->generic_count;\n theap->generic_count = 0;\n // call potential deferred free routines\n _mi_deferred_free(theap, false);\n // free retired pages\n _mi_theap_collect_retired(theap, false);\n\n // collect every once in a while (10000 by default)\n const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L);\n if (theap->generic_collect_count >= generic_collect) {\n theap->generic_collect_count = 0;\n mi_theap_collect(theap, false /* force? */);\n }\n }\n\n // find (or allocate) a page of the right size\n mi_page_t* page = mi_find_page(theap, size, huge_alignment);\n if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more\n mi_theap_collect(theap, true /* force? */);\n page = mi_find_page(theap, size, huge_alignment);\n }\n\n if mi_unlikely(page == NULL) { // out of memory\n const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`\n _mi_error_message(ENOMEM, \"unable to allocate memory (%zu bytes)\\n\", req_size);\n return NULL;\n }\n\n mi_assert_internal(mi_page_immediate_available(page));\n mi_assert_internal(mi_page_block_size(page) >= size);\n mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));\n mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);\n\n // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc)\n if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }\n void* const p = _mi_page_malloc_zero(theap,page,size,zero);\n mi_assert_internal(p != NULL);\n\n // move full pages to the full queue\n if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE && mi_page_is_full(page)) {\n mi_page_to_full(page, mi_page_queue_of(page));\n }\n return p;\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/prim/emscripten/prim.c", "sha256": "7488e9e562ca85d2d757ec48e4d334935263a2333aa05b6061d8520f427c5890", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen, Alon Zakai\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// This file is included in `src/prim/prim.c`\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\"\n\n#include // sched_yield\n#include // getentropy\n\n// Design\n// ======\n//\n// mimalloc is built on top of emmalloc. emmalloc is a minimal allocator on top\n// of sbrk. The reason for having three layers here is that we want mimalloc to\n// be able to allocate and release system memory properly, the same way it would\n// when using VirtualAlloc on Windows or mmap on POSIX, and sbrk is too limited.\n// Specifically, sbrk can only go up and down, and not \"skip\" over regions, and\n// so we end up either never freeing memory to the system, or we can get stuck\n// with holes.\n//\n// Atm wasm generally does *not* free memory back the system: once grown, we do\n// not shrink back down (https://github.com/WebAssembly/design/issues/1397).\n// However, that is expected to improve\n// (https://github.com/WebAssembly/memory-control/blob/main/proposals/memory-control/Overview.md)\n// and so we do not want to bake those limitations in here.\n//\n// Even without that issue, we want our system allocator to handle holes, that\n// is, it should merge freed regions and allow allocating new content there of\n// the full size, etc., so that we do not waste space. That means that the\n// system allocator really does need to handle the general problem of allocating\n// and freeing variable-sized chunks of memory in a random order, like malloc/\n// free do. And so it makes sense to layer mimalloc on top of such an\n// implementation.\n//\n// emmalloc makes sense for the lower level because it is small and simple while\n// still fully handling merging of holes etc. It is not the most efficient\n// allocator, but our assumption is that mimalloc needs to be fast while the\n// system allocator underneath it is called much less frequently.\n//\n\n//---------------------------------------------\n// init\n//---------------------------------------------\n\nvoid _mi_prim_mem_init( mi_os_mem_config_t* config) {\n config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB\n config->alloc_granularity = 16;\n config->has_overcommit = false;\n config->has_partial_free = false;\n config->has_virtual_reserve = false;\n}\n\nextern void emmalloc_free(void*);\n\nint _mi_prim_free(void* addr, size_t size) {\n if (size==0) return 0;\n emmalloc_free(addr);\n return 0;\n}\n\n\n//---------------------------------------------\n// Allocation\n//---------------------------------------------\n\nextern void* emmalloc_memalign(size_t alignment, size_t size);\n\n// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.\nint _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {\n MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);\n *is_large = false;\n // TODO: Track the highest address ever seen; first uses of it are zeroes.\n // That assumes no one else uses sbrk but us (they could go up,\n // scribble, and then down), but we could assert on that perhaps.\n *is_zero = false;\n // emmalloc has a minimum alignment size.\n #define MIN_EMMALLOC_ALIGN 8\n if (try_alignment < MIN_EMMALLOC_ALIGN) {\n try_alignment = MIN_EMMALLOC_ALIGN;\n }\n void* p = emmalloc_memalign(try_alignment, size);\n *addr = p;\n if (p == 0) {\n return ENOMEM;\n }\n return 0;\n}\n\n\n//---------------------------------------------\n// Commit/Reset\n//---------------------------------------------\n\nint _mi_prim_commit(void* addr, size_t size, bool* is_zero) {\n MI_UNUSED(addr); MI_UNUSED(size);\n // See TODO above.\n *is_zero = false;\n return 0;\n}\n\nint _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {\n MI_UNUSED(addr); MI_UNUSED(size);\n *needs_recommit = false;\n return 0;\n}\n\nint _mi_prim_reset(void* addr, size_t size) {\n MI_UNUSED(addr); MI_UNUSED(size);\n return 0;\n}\n\nint _mi_prim_reuse(void* addr, size_t size) {\n MI_UNUSED(addr); MI_UNUSED(size);\n return 0;\n}\n\nint _mi_prim_protect(void* addr, size_t size, bool protect) {\n MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect);\n return 0;\n}\n\n\n//---------------------------------------------\n// Huge pages and NUMA nodes\n//---------------------------------------------\n\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {\n MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);\n *is_zero = true;\n *addr = NULL;\n return ENOSYS;\n}\n\nsize_t _mi_prim_numa_node(void) {\n return 0;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n return 1;\n}\n\n\n//----------------------------------------------------------------\n// Clock\n//----------------------------------------------------------------\n\n#include \n\nmi_msecs_t _mi_prim_clock_now(void) {\n return emscripten_date_now();\n}\n\n\n//----------------------------------------------------------------\n// Process info\n//----------------------------------------------------------------\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo)\n{\n // use defaults\n MI_UNUSED(pinfo);\n}\n\n\n//----------------------------------------------------------------\n// Output\n//----------------------------------------------------------------\n\n#include \n\nvoid _mi_prim_out_stderr( const char* msg) {\n emscripten_console_error(msg);\n}\n\n\n//----------------------------------------------------------------\n// Environment\n//----------------------------------------------------------------\n\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size) {\n // For code size reasons, do not support environ customization for now.\n MI_UNUSED(name);\n MI_UNUSED(result);\n MI_UNUSED(result_size);\n return false;\n}\n\n\n//----------------------------------------------------------------\n// Random\n//----------------------------------------------------------------\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n int err = getentropy(buf, buf_len);\n return !err;\n}\n\n\n//----------------------------------------------------------------\n// Thread init/done\n//----------------------------------------------------------------\n\n#if defined(MI_USE_PTHREADS)\n\n// use pthread local storage keys to detect thread ending\n// (and used with MI_TLS_PTHREADS for the default theap)\npthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);\n\nstatic void mi_pthread_done(void* value) {\n if (value!=NULL) {\n _mi_thread_done((mi_theap_t*)value);\n }\n}\n\nvoid _mi_prim_thread_init_auto_done(void) {\n mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));\n pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);\n}\n\nvoid _mi_prim_thread_done_auto_done(void) {\n if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809\n pthread_key_delete(_mi_heap_default_key);\n }\n}\n\nvoid _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {\n if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD\n pthread_setspecific(_mi_heap_default_key, theap);\n }\n}\n\n#else\n\nvoid _mi_prim_thread_init_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_done_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {\n MI_UNUSED(theap);\n}\n#endif\n\nbool _mi_prim_thread_is_in_threadpool(void) {\n return false;\n}\n\nvoid _mi_prim_thread_yield(void) {\n sched_yield();\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/prim/osx/alloc-override-zone.c", "sha256": "ce14eb6dfd55f241095b55f70983931a2d92f7a6a622a9b360cdd70ab3b53208", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2022, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n\n#if defined(MI_MALLOC_OVERRIDE)\n\n#if !defined(__APPLE__)\n#error \"this file should only be included on macOS\"\n#endif\n\n/* ------------------------------------------------------\n Override system malloc on macOS\n This is done through the malloc zone interface.\n It seems to be most robust in combination with interposing\n though or otherwise we may get zone errors as there are could\n be allocations done by the time we take over the\n zone.\n------------------------------------------------------ */\n\n#include \n#include \n#include // memset\n#include \n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)\n// only available from OSX 10.6\nextern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import));\n#endif\n\n/* ------------------------------------------------------\n malloc zone members\n------------------------------------------------------ */\n\nstatic bool is_mimalloc_zone( malloc_zone_t* zone ); \n\nstatic size_t zone_size(malloc_zone_t* zone, const void* p) {\n if (mi_any_heap_contains(p)) { \n return mi_usable_size(p);\n }\n else if (!is_mimalloc_zone(zone)) { // can happen due to interpose\n return zone->size(zone,p);\n }\n else {\n return 0;\n }\n}\n\nstatic void* zone_malloc(malloc_zone_t* zone, size_t size) {\n MI_UNUSED(zone);\n return mi_malloc(size);\n}\n\nstatic void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) {\n MI_UNUSED(zone);\n return mi_calloc(count, size);\n}\n\nstatic void* zone_valloc(malloc_zone_t* zone, size_t size) {\n MI_UNUSED(zone);\n return mi_malloc_aligned(size, _mi_os_page_size());\n}\n\nstatic void zone_free(malloc_zone_t* zone, void* p) {\n if (mi_any_heap_contains(p)) {\n mi_free(p); // with the page_map and pagemap_commit=1 we can use the regular free\n }\n else if (!is_mimalloc_zone(zone)) { // can happen due to interpose\n zone->free(zone,p);\n }\n}\n\nstatic void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) {\n if (p == NULL || mi_any_heap_contains(p)) {\n return mi_realloc(p, newsize);\n }\n else if (!is_mimalloc_zone(zone)) { // can happen due to interpose\n return zone->realloc(zone,p,newsize);\n }\n else {\n return NULL;\n }\n}\n\nstatic void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) {\n MI_UNUSED(zone);\n return mi_malloc_aligned(size,alignment);\n}\n\nstatic void zone_destroy(malloc_zone_t* zone) {\n if (!is_mimalloc_zone(zone)) {\n zone->destroy(zone);\n }\n}\n\nstatic unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) {\n unsigned i;\n for (i = 0; i < count; i++) {\n ps[i] = zone_malloc(zone, size);\n if (ps[i] == NULL) break;\n }\n return i;\n}\n\nstatic void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) {\n for(size_t i = 0; i < count; i++) {\n zone_free(zone, ps[i]);\n ps[i] = NULL;\n }\n}\n\nstatic size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) {\n MI_UNUSED(zone); MI_UNUSED(size);\n mi_collect(false);\n return 0;\n}\n\nstatic void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) {\n MI_UNUSED(size);\n zone_free(zone,p);\n}\n\nstatic boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) {\n MI_UNUSED(zone);\n return mi_is_in_heap_region(p);\n}\n\n\n/* ------------------------------------------------------\n Introspection members\n------------------------------------------------------ */\n\nstatic kern_return_t intro_enumerator(task_t task, void* p,\n unsigned type_mask, vm_address_t zone_address,\n memory_reader_t reader,\n vm_range_recorder_t recorder)\n{\n // todo: enumerate all memory\n MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address);\n MI_UNUSED(reader); MI_UNUSED(recorder);\n return KERN_SUCCESS;\n}\n\nstatic size_t intro_good_size(malloc_zone_t* zone, size_t size) {\n MI_UNUSED(zone);\n return mi_good_size(size);\n}\n\nstatic boolean_t intro_check(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n return true;\n}\n\nstatic void intro_print(malloc_zone_t* zone, boolean_t verbose) {\n MI_UNUSED(zone); MI_UNUSED(verbose);\n mi_stats_print(NULL);\n}\n\nstatic void intro_log(malloc_zone_t* zone, void* p) {\n MI_UNUSED(zone); MI_UNUSED(p);\n // todo?\n}\n\nstatic void intro_force_lock(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n // todo?\n}\n\nstatic void intro_force_unlock(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n // todo?\n}\n\nstatic void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) {\n MI_UNUSED(zone);\n // todo...\n stats->blocks_in_use = 0;\n stats->size_in_use = 0;\n stats->max_size_in_use = 0;\n stats->size_allocated = 0;\n}\n\nstatic boolean_t intro_zone_locked(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n return false;\n}\n\n\n/* ------------------------------------------------------\n At process start, override the default allocator\n------------------------------------------------------ */\n\n#if defined(__GNUC__) && !defined(__clang__)\n#pragma GCC diagnostic ignored \"-Wmissing-field-initializers\"\n#endif\n\n#if defined(__clang__)\n#pragma clang diagnostic ignored \"-Wc99-extensions\"\n#endif\n\nstatic malloc_introspection_t mi_introspect = {\n .enumerator = &intro_enumerator,\n .good_size = &intro_good_size,\n .check = &intro_check,\n .print = &intro_print,\n .log = &intro_log,\n .force_lock = &intro_force_lock,\n .force_unlock = &intro_force_unlock,\n#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)\n .statistics = &intro_statistics,\n .zone_locked = &intro_zone_locked,\n#endif\n};\n\nstatic malloc_zone_t mi_malloc_zone = {\n // note: even with designators, the order is important for C++ compilation\n //.reserved1 = NULL,\n //.reserved2 = NULL,\n .size = &zone_size,\n .malloc = &zone_malloc,\n .calloc = &zone_calloc,\n .valloc = &zone_valloc,\n .free = &zone_free,\n .realloc = &zone_realloc,\n .destroy = &zone_destroy,\n .zone_name = \"mimalloc\",\n .batch_malloc = &zone_batch_malloc,\n .batch_free = &zone_batch_free,\n .introspect = &mi_introspect,\n#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)\n #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)\n .version = 10,\n #else\n .version = 9,\n #endif\n // switch to version 9+ on OSX 10.6 to support memalign.\n .memalign = &zone_memalign,\n .free_definite_size = &zone_free_definite_size,\n #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)\n .pressure_relief = &zone_pressure_relief,\n #endif\n #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14)\n .claimed_address = &zone_claimed_address,\n #endif\n#else\n .version = 4,\n#endif\n};\n\n#ifdef __cplusplus\n}\n#endif\n\nstatic bool is_mimalloc_zone( malloc_zone_t* zone ) {\n return (zone==NULL || zone==&mi_malloc_zone);\n}\n\n#if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT)\n\n// ------------------------------------------------------\n// Override malloc_xxx and malloc_zone_xxx api's to use only\n// our mimalloc zone. Since even the loader uses malloc\n// on macOS, this ensures that all allocations go through\n// mimalloc (as all calls are interposed).\n// The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`,\n// Here, we also override macOS specific API's like\n// `malloc_zone_calloc` etc. see \n// ------------------------------------------------------\n\nstatic inline malloc_zone_t* mi_get_default_zone(void) {\n mi_atomic_do_once {\n malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see )\n }\n return &mi_malloc_zone;\n}\n\nmi_decl_externc int malloc_jumpstart(uintptr_t cookie);\nmi_decl_externc void _malloc_fork_prepare(void);\nmi_decl_externc void _malloc_fork_parent(void);\nmi_decl_externc void _malloc_fork_child(void);\n\n\nstatic malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) {\n MI_UNUSED(size); MI_UNUSED(flags);\n return mi_get_default_zone();\n}\n\nstatic malloc_zone_t* mi_malloc_default_zone (void) {\n return mi_get_default_zone();\n}\n\nstatic malloc_zone_t* mi_malloc_default_purgeable_zone(void) {\n return mi_get_default_zone();\n}\n\nstatic void mi_malloc_destroy_zone(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n // nothing.\n}\n\nstatic kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) {\n MI_UNUSED(task); MI_UNUSED(mr);\n if (addresses != NULL) *addresses = NULL;\n if (count != NULL) *count = 0;\n return KERN_SUCCESS;\n}\n\nstatic const char* mi_malloc_get_zone_name(malloc_zone_t* zone) {\n return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name);\n}\n\nstatic void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) {\n MI_UNUSED(zone); MI_UNUSED(name);\n}\n\nstatic int mi_malloc_jumpstart(uintptr_t cookie) {\n MI_UNUSED(cookie);\n return 1; // or 0 for no error?\n}\n\nstatic void mi__malloc_fork_prepare(void) {\n // nothing\n}\nstatic void mi__malloc_fork_parent(void) {\n // nothing\n}\nstatic void mi__malloc_fork_child(void) {\n // nothing\n}\n\nstatic void mi_malloc_printf(const char* fmt, ...) {\n MI_UNUSED(fmt);\n}\n\nstatic bool zone_check(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n return true;\n}\n\nstatic malloc_zone_t* zone_from_ptr(const void* p) {\n MI_UNUSED(p);\n return (mi_any_heap_contains(p) ? mi_get_default_zone() : NULL);\n}\n\nstatic void zone_log(malloc_zone_t* zone, void* p) {\n MI_UNUSED(zone); MI_UNUSED(p);\n}\n\nstatic void zone_print(malloc_zone_t* zone, bool b) {\n MI_UNUSED(zone); MI_UNUSED(b);\n}\n\nstatic void zone_print_ptr_info(void* p) {\n MI_UNUSED(p);\n}\n\nstatic void zone_register(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n}\n\nstatic void zone_unregister(malloc_zone_t* zone) {\n MI_UNUSED(zone);\n}\n\n// use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1`\n// See: \nstruct mi_interpose_s {\n const void* replacement;\n const void* target;\n};\n#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun }\n#define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun)\n#define MI_INTERPOSE_ZONE(fun) MI_INTERPOSE_FUN(malloc_##fun,fun)\n__attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] __attribute__((section(\"__DATA, __interpose\"))) =\n{\n\n MI_INTERPOSE_MI(malloc_create_zone),\n MI_INTERPOSE_MI(malloc_default_purgeable_zone),\n MI_INTERPOSE_MI(malloc_default_zone),\n MI_INTERPOSE_MI(malloc_destroy_zone),\n MI_INTERPOSE_MI(malloc_get_all_zones),\n MI_INTERPOSE_MI(malloc_get_zone_name),\n MI_INTERPOSE_MI(malloc_jumpstart),\n MI_INTERPOSE_MI(malloc_printf),\n MI_INTERPOSE_MI(malloc_set_zone_name),\n MI_INTERPOSE_MI(_malloc_fork_child),\n MI_INTERPOSE_MI(_malloc_fork_parent),\n MI_INTERPOSE_MI(_malloc_fork_prepare),\n\n MI_INTERPOSE_ZONE(zone_batch_free),\n MI_INTERPOSE_ZONE(zone_batch_malloc),\n MI_INTERPOSE_ZONE(zone_calloc),\n MI_INTERPOSE_ZONE(zone_check),\n MI_INTERPOSE_ZONE(zone_free),\n MI_INTERPOSE_ZONE(zone_from_ptr),\n MI_INTERPOSE_ZONE(zone_log),\n MI_INTERPOSE_ZONE(zone_malloc),\n MI_INTERPOSE_ZONE(zone_memalign),\n MI_INTERPOSE_ZONE(zone_print),\n MI_INTERPOSE_ZONE(zone_print_ptr_info),\n MI_INTERPOSE_ZONE(zone_realloc),\n MI_INTERPOSE_ZONE(zone_register),\n MI_INTERPOSE_ZONE(zone_unregister),\n MI_INTERPOSE_ZONE(zone_valloc)\n};\n\n\n#else\n\n// ------------------------------------------------------\n// hook into the zone api's without interposing\n// This is the official way of adding an allocator but\n// it seems less robust than using interpose.\n// ------------------------------------------------------\n\nstatic inline malloc_zone_t* mi_get_default_zone(void)\n{\n // The first returned zone is the real default\n malloc_zone_t** zones = NULL;\n unsigned count = 0;\n kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count);\n if (ret == KERN_SUCCESS && count > 0) {\n return zones[0];\n }\n else {\n // fallback\n return malloc_default_zone();\n }\n}\n\n#if defined(__clang__)\n__attribute__((constructor(101))) // highest priority\n#else\n__attribute__((constructor)) // priority level is not supported by gcc\n#endif\n__attribute__((used))\nstatic void _mi_macos_override_malloc(void) {\n malloc_zone_t* purgeable_zone = NULL;\n\n #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)\n // force the purgeable zone to exist to avoid strange bugs\n if (malloc_default_purgeable_zone) {\n purgeable_zone = malloc_default_purgeable_zone();\n }\n #endif\n\n // Register our zone.\n // thomcc: I think this is still needed to put us in the zone list.\n malloc_zone_register(&mi_malloc_zone);\n // Unregister the default zone, this makes our zone the new default\n // as that was the last registered.\n malloc_zone_t *default_zone = mi_get_default_zone();\n // thomcc: Unsure if the next test is *always* false or just false in the\n // cases I've tried. I'm also unsure if the code inside is needed. at all\n if (default_zone != &mi_malloc_zone) {\n malloc_zone_unregister(default_zone);\n\n // Reregister the default zone so free and realloc in that zone keep working.\n malloc_zone_register(default_zone);\n }\n\n // Unregister, and re-register the purgeable_zone to avoid bugs if it occurs\n // earlier than the default zone.\n if (purgeable_zone != NULL) {\n malloc_zone_unregister(purgeable_zone);\n malloc_zone_register(purgeable_zone);\n }\n\n}\n#endif // MI_OSX_INTERPOSE\n\n#endif // MI_MALLOC_OVERRIDE\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/prim/osx/prim.c", "sha256": "247a9952465eb105be03a9962e922b085ce5d52034775551a61cd8594275be73", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// We use the unix/prim.c with the mmap API on macOSX\n#include \"../unix/prim.c\"\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/prim/prim.c", "sha256": "cc771788e5ba591efb681829a8724f8662a1f50f3db39cf5ebe7738b8ed57e44", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// Select the implementation of the primitives\n// depending on the OS.\n\n#if defined(_WIN32)\n#include \"windows/prim.c\" // VirtualAlloc (Windows)\n\n#elif defined(__APPLE__)\n#include \"osx/prim.c\" // macOSX (actually defers to mmap in unix/prim.c)\n\n#elif defined(__wasi__)\n#define MI_USE_SBRK\n#include \"wasi/prim.c\" // memory-grow or sbrk (Wasm)\n\n#elif defined(__EMSCRIPTEN__)\n#include \"emscripten/prim.c\" // emmalloc_*, + pthread support\n\n#else\n#include \"unix/prim.c\" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.)\n\n#endif\n\n// Generic process initialization\n#ifndef MI_PRIM_HAS_PROCESS_ATTACH\n#if defined(__GNUC__) || defined(__clang__)\n // gcc,clang: use the constructor/destructor attribute\n // which for both seem to run before regular constructors/destructors\n #if defined(__clang__)\n #define mi_attr_constructor __attribute__((constructor(101)))\n #define mi_attr_destructor __attribute__((destructor(101)))\n #else\n #define mi_attr_constructor __attribute__((constructor))\n #define mi_attr_destructor __attribute__((destructor))\n #endif\n static void mi_attr_constructor mi_process_attach(void) {\n _mi_auto_process_init();\n }\n static void mi_attr_destructor mi_process_detach(void) {\n _mi_auto_process_done();\n }\n#elif defined(__cplusplus)\n // C++: use static initialization to detect process start/end\n // This is not guaranteed to be first/last but the best we can generally do?\n struct mi_init_done_t {\n mi_init_done_t() {\n _mi_auto_process_init();\n }\n ~mi_init_done_t() {\n _mi_auto_process_done();\n }\n };\n static mi_init_done_t mi_init_done;\n #else\n #pragma message(\"define a way to call _mi_auto_process_init/done on your platform\")\n#endif\n#endif\n\n// Generic allocator init/done callback\n#ifndef MI_PRIM_HAS_ALLOCATOR_INIT\nbool _mi_is_redirected(void) {\n return false;\n}\nbool _mi_allocator_init(const char** message) {\n if (message != NULL) { *message = NULL; }\n return true;\n}\nvoid _mi_allocator_done(void) {\n // nothing to do\n}\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/prim/unix/prim.c", "sha256": "c1ae32eaac7ccf18c10058cca99723892f8bd027318590151f5011c929c7e3be", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// This file is included in `src/prim/prim.c`\n\n#ifndef _DEFAULT_SOURCE\n#define _DEFAULT_SOURCE // ensure mmap flags and syscall are defined\n#endif\n\n#if defined(__sun)\n// illumos provides new mman.h api when any of these are defined\n// otherwise the old api based on caddr_t which predates the void pointers one.\n// stock solaris provides only the former, chose to atomically to discard those\n// flags only here rather than project wide tough.\n#undef _XOPEN_SOURCE\n#undef _POSIX_C_SOURCE\n#endif\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\"\n\n#include // mmap\n#include // sysconf, sleep\n#include // open, close, read, access\n#include // getenv, arc4random_buf\n\n#if defined(__linux__)\n #include \n #include // THP disable, PR_SET_VMA\n #include // sysinfo\n #if defined(__GLIBC__) && !defined(PR_SET_VMA)\n #include \n #endif\n #if defined(__GLIBC__)\n #include // linux mmap flags\n #else\n #include \n #endif\n#elif defined(__APPLE__)\n #include \n #include \n #if !defined(TARGET_OS_OSX) || TARGET_OS_OSX // see issue #879, used to be (!TARGET_IOS_IPHONE && !TARGET_IOS_SIMULATOR)\n #include // VM_MAKE_TAG, VM_FLAGS_SUPERPAGE_SIZE_2MB, etc.\n #endif\n #if !defined(MAC_OS_X_VERSION_10_7)\n #define MAC_OS_X_VERSION_10_7 1070\n #endif\n #include \n#elif defined(__FreeBSD__) || defined(__DragonFly__)\n #include \n #if __FreeBSD_version >= 1200000\n #include \n #include \n #endif\n #include \n#endif\n\n#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__FreeBSD__)\n #define MI_HAS_SYSCALL_H\n #include \n#endif\n\n#if !defined(MADV_DONTNEED) && defined(POSIX_MADV_DONTNEED) // QNX\n#define MADV_DONTNEED POSIX_MADV_DONTNEED\n#endif\n#if !defined(MADV_FREE) && defined(POSIX_MADV_FREE) // QNX\n#define MADV_FREE POSIX_MADV_FREE\n#endif\n\n#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // TODO: can we query the OS for this?\n\n//------------------------------------------------------------------------------------\n// Use syscalls for some primitives to allow for libraries that override open/read/close etc.\n// and do allocation themselves; using syscalls prevents recursion when mimalloc is\n// still initializing (issue #713)\n// Declare inline to avoid unused function warnings.\n//------------------------------------------------------------------------------------\n\n#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access)\n\nstatic inline int mi_prim_open(const char* fpath, int open_flags) {\n return syscall(SYS_open,fpath,open_flags,0);\n}\nstatic inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {\n return syscall(SYS_read,fd,buf,bufsize);\n}\nstatic inline int mi_prim_close(int fd) {\n return syscall(SYS_close,fd);\n}\nstatic inline int mi_prim_access(const char *fpath, int mode) {\n return syscall(SYS_access,fpath,mode);\n}\n\n#else\n\nstatic inline int mi_prim_open(const char* fpath, int open_flags) {\n return open(fpath,open_flags);\n}\nstatic inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {\n return read(fd,buf,bufsize);\n}\nstatic inline int mi_prim_close(int fd) {\n return close(fd);\n}\nstatic inline int mi_prim_access(const char *fpath, int mode) {\n return access(fpath,mode);\n}\n\n#endif\n\n\n\n//---------------------------------------------\n// init\n//---------------------------------------------\n\nstatic bool unix_detect_overcommit(void) {\n bool os_overcommit = true;\n #if defined(__linux__)\n int fd = mi_prim_open(\"/proc/sys/vm/overcommit_memory\", O_RDONLY);\n if (fd >= 0) {\n char buf[32];\n ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));\n mi_prim_close(fd);\n // \n // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE)\n if (nread >= 1) {\n os_overcommit = (buf[0] == '0' || buf[0] == '1');\n }\n }\n #elif defined(__FreeBSD__)\n int val = 0;\n size_t olen = sizeof(val);\n if (sysctlbyname(\"vm.overcommit\", &val, &olen, NULL, 0) == 0) {\n os_overcommit = (val != 0);\n }\n #else\n // default: overcommit is true\n #endif\n return os_overcommit;\n}\n\nstatic bool unix_detect_thp(void) {\n bool thp_enabled = false;\n #if defined(__linux__)\n int fd = mi_prim_open(\"/sys/kernel/mm/transparent_hugepage/enabled\", O_RDONLY);\n if (fd >= 0) {\n char buf[32];\n ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));\n mi_prim_close(fd);\n // \n // between brackets is the current value, for example: always [madvise] never\n if (nread >= 1) {\n thp_enabled = (_mi_strnstr(buf,32,\"[never]\") == NULL);\n }\n }\n #endif\n return thp_enabled;\n}\n\n// try to detect the physical memory dynamically (if possible)\nstatic void unix_detect_physical_memory( size_t page_size, size_t* physical_memory_in_kib ) {\n #if defined(CTL_HW) && (defined(HW_PHYSMEM64) || defined(HW_MEMSIZE)) // freeBSD, macOS\n MI_UNUSED(page_size);\n int64_t physical_memory = 0;\n size_t length = sizeof(int64_t);\n #if defined(HW_PHYSMEM64)\n int mib[2] = { CTL_HW, HW_PHYSMEM64 };\n #else\n int mib[2] = { CTL_HW, HW_MEMSIZE };\n #endif\n const int err = sysctl(mib, 2, &physical_memory, &length, NULL, 0);\n if (err==0 && physical_memory > 0) {\n const int64_t phys_in_kib = physical_memory / MI_KiB;\n if (phys_in_kib > 0 && (uint64_t)phys_in_kib <= SIZE_MAX) {\n *physical_memory_in_kib = (size_t)phys_in_kib;\n }\n }\n #elif defined(__linux__)\n MI_UNUSED(page_size);\n struct sysinfo info; _mi_memzero_var(info);\n const int err = sysinfo(&info);\n if (err==0 && info.mem_unit > 0 && info.totalram <= SIZE_MAX) {\n if (info.mem_unit==MI_KiB) {\n *physical_memory_in_kib = (size_t)info.totalram;\n }\n else {\n size_t total = 0;\n if (!mi_mul_overflow((size_t)info.totalram, (size_t)info.mem_unit, &total)) {\n *physical_memory_in_kib = (total / MI_KiB);\n }\n }\n }\n #elif defined(_SC_PHYS_PAGES) // do not use by default as it might cause allocation (by using `fopen` to parse /proc/meminfo) (issue #1100)\n const long pphys = sysconf(_SC_PHYS_PAGES);\n const size_t psize_in_kib = page_size / MI_KiB;\n if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys <= SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) {\n *physical_memory_in_kib = (size_t)pphys * psize_in_kib;\n }\n #endif\n}\n\nvoid _mi_prim_mem_init( mi_os_mem_config_t* config )\n{\n long psize = sysconf(_SC_PAGESIZE);\n if (psize > 0 && (unsigned long)psize < SIZE_MAX) {\n config->page_size = (size_t)psize;\n config->alloc_granularity = (size_t)psize;\n unix_detect_physical_memory(config->page_size, &config->physical_memory_in_kib);\n }\n config->large_page_size = MI_UNIX_LARGE_PAGE_SIZE;\n config->has_overcommit = unix_detect_overcommit();\n config->has_partial_free = true; // mmap can free in parts\n config->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE)\n config->has_transparent_huge_pages = unix_detect_thp();\n\n // disable transparent huge pages for this process?\n #if (defined(__linux__) || defined(__ANDROID__)) && defined(PR_GET_THP_DISABLE)\n if (!mi_option_is_enabled(mi_option_allow_thp)) // disable THP if requested through an option\n {\n config->has_transparent_huge_pages = false;\n if (prctl(PR_GET_THP_DISABLE, 0, 0, 0, 0) == 0) { // -1 on error, 1 if already disabled\n // Most likely since distros often come with always/madvise settings.\n // Disabling only for mimalloc process rather than touching system wide settings\n (void)prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0);\n }\n }\n #endif\n}\n\n\n//---------------------------------------------\n// free\n//---------------------------------------------\n\nint _mi_prim_free(void* addr, size_t size ) {\n if (size==0) return 0;\n bool err = (munmap(addr, size) == -1);\n return (err ? errno : 0);\n}\n\n\n//---------------------------------------------\n// mmap\n//---------------------------------------------\n\n// return errno on failure\nstatic int unix_madvise(void* addr, size_t size, int advice) {\n #if defined(__sun)\n const int res = madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520)\n return (res==0 ? 0 : errno);\n #elif defined(__QNX__)\n return posix_madvise(addr, size, advice); // posix returns errno\n #else\n const int res = madvise(addr, size, advice); // linux returns -1 on failure and sets errno\n return (res==0 ? 0 : errno);\n #endif\n}\n\nstatic void* unix_mmap_prim(void* addr, size_t size, int protect_flags, int flags, int fd) {\n void* p = mmap(addr, size, protect_flags, flags, fd, 0 /* offset */);\n #if defined(__linux__) && defined(PR_SET_VMA)\n if (p!=MAP_FAILED && p!=NULL) {\n prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, p, size, \"mimalloc\");\n }\n #endif\n return p;\n}\n\nstatic void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {\n MI_UNUSED(try_alignment);\n void* p = NULL;\n #if defined(MAP_ALIGNED) // BSD\n if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {\n size_t n = 0;\n mi_bsr(try_alignment, &n);\n if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB\n p = unix_mmap_prim(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd);\n if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {\n int err = errno;\n _mi_trace_message(\"unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\\n\", err, err, size, try_alignment, addr);\n }\n if (p!=MAP_FAILED) return p;\n // fall back to regular mmap\n }\n }\n #elif defined(MAP_ALIGN) // Solaris\n if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {\n p = unix_mmap_prim((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd); // addr parameter is the required alignment\n if (p!=MAP_FAILED) return p;\n // fall back to regular mmap\n }\n #endif\n #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)\n // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations\n if (addr == NULL) {\n void* hint = _mi_os_get_aligned_hint(try_alignment, size);\n if (hint != NULL) {\n p = unix_mmap_prim(hint, size, protect_flags, flags, fd);\n if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {\n #if MI_TRACK_ENABLED // asan sometimes does not instrument errno correctly?\n int err = 0;\n #else\n int err = errno;\n #endif\n _mi_trace_message(\"unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\\n\", err, err, size, try_alignment, hint);\n }\n if (p!=MAP_FAILED) return p;\n // fall back to regular mmap\n }\n }\n #endif\n // regular mmap\n p = unix_mmap_prim(addr, size, protect_flags, flags, fd);\n if (p!=MAP_FAILED) return p;\n // failed to allocate\n return NULL;\n}\n\nstatic int unix_mmap_fd(void) {\n #if defined(VM_MAKE_TAG)\n // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)\n int os_tag = (int)mi_option_get(mi_option_os_tag);\n if (os_tag < 100 || os_tag > 255) { os_tag = 254; }\n return VM_MAKE_TAG(os_tag);\n #else\n return -1;\n #endif\n}\n\nstatic void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {\n #if !defined(MAP_ANONYMOUS)\n #define MAP_ANONYMOUS MAP_ANON\n #endif\n #if !defined(MAP_NORESERVE)\n #define MAP_NORESERVE 0\n #endif\n void* p = NULL;\n const int fd = unix_mmap_fd();\n int flags = MAP_PRIVATE | MAP_ANONYMOUS;\n if (_mi_os_has_overcommit()) {\n flags |= MAP_NORESERVE;\n }\n #if defined(PROT_MAX)\n protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD\n #endif\n // huge page allocation\n if (allow_large && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) {\n static _Atomic(size_t) large_page_try_ok; // = 0;\n size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);\n if (!large_only && try_ok > 0) {\n // If the OS is not configured for large OS pages, or the user does not have\n // enough permission, the `mmap` will always fail (but it might also fail for other reasons).\n // Therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times\n // to avoid too many failing calls to mmap.\n mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1);\n }\n else {\n int lflags = flags & ~MAP_NORESERVE; // using NORESERVE on huge pages seems to fail on Linux\n int lfd = fd;\n #ifdef MAP_ALIGNED_SUPER\n lflags |= MAP_ALIGNED_SUPER;\n #endif\n #ifdef MAP_HUGETLB\n lflags |= MAP_HUGETLB;\n #endif\n #ifdef MAP_HUGE_1GB\n static bool mi_huge_pages_available = true;\n if (large_only && (size % MI_GiB) == 0 && mi_huge_pages_available) {\n lflags |= MAP_HUGE_1GB;\n }\n else\n #endif\n {\n #ifdef MAP_HUGE_2MB\n lflags |= MAP_HUGE_2MB;\n #endif\n }\n #ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB\n lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;\n #endif\n if (large_only || lflags != flags) {\n // try large OS page allocation\n *is_large = true;\n p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);\n #ifdef MAP_HUGE_1GB\n if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) {\n mi_huge_pages_available = false; // don't try huge 1GiB pages again\n if (large_only) {\n _mi_warning_message(\"unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\\n\", errno);\n }\n lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);\n p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);\n }\n #endif\n if (large_only) return p;\n if (p == NULL) {\n mi_atomic_store_release(&large_page_try_ok, (size_t)8); // on error, don't try again for the next N allocations\n }\n }\n }\n }\n // regular allocation\n if (p == NULL) {\n *is_large = false;\n p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd);\n #if !defined(MI_NO_THP)\n if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) {\n #if defined(MADV_HUGEPAGE)\n // Many Linux systems don't allow MAP_HUGETLB but they support instead\n // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE\n // though since properly aligned allocations will already use large pages if available\n // in that case -- in particular for our large regions (in `memory.c`).\n // However, some systems only allow THP if called with explicit `madvise`, so\n // when large OS pages are enabled for mimalloc, we call `madvise` anyways.\n if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {\n // *is_large = true; // possibly\n };\n #elif defined(__sun)\n struct memcntl_mha cmd = {0};\n cmd.mha_pagesize = _mi_os_large_page_size();\n cmd.mha_cmd = MHA_MAPSIZE_VA;\n if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {\n // *is_large = true; // possibly\n }\n #endif\n }\n #endif\n }\n return p;\n}\n\n// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.\nint _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {\n mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);\n mi_assert_internal(commit || !allow_large);\n mi_assert_internal(try_alignment > 0);\n *is_zero = true;\n int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);\n *addr = unix_mmap(hint_addr, size, try_alignment, protect_flags, false, allow_large, is_large);\n return (*addr != NULL ? 0 : errno);\n}\n\n\n//---------------------------------------------\n// Commit/Reset\n//---------------------------------------------\n\nstatic void unix_mprotect_hint(int err) {\n #if defined(__linux__) && (MI_SECURE>=5) // guard page around every mimalloc page\n if (err == ENOMEM) {\n _mi_warning_message(\"The next warning may be caused by a low memory map limit.\\n\"\n \" On Linux this is controlled by the vm.max_map_count -- maybe increase it?\\n\"\n \" For example: sudo sysctl -w vm.max_map_count=262144\\n\");\n }\n #else\n MI_UNUSED(err);\n #endif\n}\n\n\n\n\n\nint _mi_prim_commit(void* start, size_t size, bool* is_zero) {\n // commit: ensure we can access the area\n // note: we may think that *is_zero can be true since the memory\n // was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but\n // we sometimes call commit on a range with still partially committed\n // memory and `mprotect` does not zero the range.\n *is_zero = false;\n int err = mprotect(start, size, (PROT_READ | PROT_WRITE));\n if (err != 0) {\n err = errno;\n unix_mprotect_hint(err);\n }\n return err;\n}\n\nint _mi_prim_reuse(void* start, size_t size) {\n MI_UNUSED(start); MI_UNUSED(size);\n #if defined(__APPLE__) && defined(MADV_FREE_REUSE)\n return unix_madvise(start, size, MADV_FREE_REUSE);\n #endif\n return 0;\n}\n\nint _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {\n int err = 0;\n #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)\n // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097)\n err = unix_madvise(start, size, MADV_FREE_REUSABLE);\n if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }\n #else\n // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)\n err = unix_madvise(start, size, MADV_DONTNEED);\n #endif\n #if !MI_DEBUG && MI_SECURE<=2\n *needs_recommit = false;\n #else\n *needs_recommit = true;\n mprotect(start, size, PROT_NONE);\n #endif\n /*\n // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss)\n *needs_recommit = true;\n const int fd = unix_mmap_fd();\n void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);\n if (p != start) { err = errno; }\n */\n return err;\n}\n\nint _mi_prim_reset(void* start, size_t size) {\n int err = 0;\n\n // on macOS can use MADV_FREE_REUSABLE (but we disable this for now as it seems slower)\n #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE)\n err = unix_madvise(start, size, MADV_FREE_REUSABLE);\n if (err==0) return 0;\n // fall through\n #endif\n\n #if defined(MADV_FREE)\n // Otherwise, we try to use `MADV_FREE` as that is the fastest. A drawback though is that it\n // will not reduce the `rss` stats in tools like `top` even though the memory is available\n // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by\n // default `MADV_DONTNEED` is used though.\n static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE);\n int oadvice = (int)mi_atomic_load_relaxed(&advice);\n while ((err = unix_madvise(start, size, oadvice)) != 0 && err == EAGAIN) { /* try again */ };\n if (err == EINVAL && oadvice == MADV_FREE) {\n // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on\n mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED);\n err = unix_madvise(start, size, MADV_DONTNEED);\n }\n #else\n err = unix_madvise(start, size, MADV_DONTNEED);\n #endif\n return err;\n}\n\nint _mi_prim_protect(void* start, size_t size, bool protect) {\n int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE));\n if (err != 0) { err = errno; }\n unix_mprotect_hint(err);\n return err;\n}\n\n\n\n//---------------------------------------------\n// Huge page allocation\n//---------------------------------------------\n\n#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__)\n\n#ifndef MPOL_PREFERRED\n#define MPOL_PREFERRED 1\n#endif\n\n#if defined(MI_HAS_SYSCALL_H) && defined(SYS_mbind)\nstatic long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {\n return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags);\n}\n#else\nstatic long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {\n MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags);\n return 0;\n}\n#endif\n\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {\n bool is_large = true;\n *is_zero = true;\n *addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large);\n if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes\n unsigned long numa_mask = (1UL << numa_node);\n // TODO: does `mbind` work correctly for huge OS pages? should we\n // use `set_mempolicy` before calling mmap instead?\n // see: \n long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0);\n if (err != 0) {\n err = errno;\n _mi_warning_message(\"failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\\n\", numa_node, err, err);\n }\n }\n return (*addr != NULL ? 0 : errno);\n}\n\n#else\n\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {\n MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);\n *is_zero = false;\n *addr = NULL;\n return ENOMEM;\n}\n\n#endif\n\n//---------------------------------------------\n// NUMA nodes\n//---------------------------------------------\n\n#if defined(__linux__)\n\nsize_t _mi_prim_numa_node(void) {\n #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu)\n unsigned long node = 0;\n unsigned long ncpu = 0;\n long err = syscall(SYS_getcpu, &ncpu, &node, NULL);\n if (err != 0) return 0;\n return node;\n #else\n return 0;\n #endif\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n char buf[128];\n unsigned node = 0;\n for(node = 0; node < 256; node++) {\n // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)\n _mi_snprintf(buf, 127, \"/sys/devices/system/node/node%u\", node + 1);\n if (mi_prim_access(buf,R_OK) != 0) break;\n }\n return (node+1);\n}\n\n#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000\n\nsize_t _mi_prim_numa_node(void) {\n domainset_t dom;\n size_t node;\n int policy;\n if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul;\n for (node = 0; node < MAXMEMDOM; node++) {\n if (DOMAINSET_ISSET(node, &dom)) return node;\n }\n return 0ul;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n size_t ndomains = 0;\n size_t len = sizeof(ndomains);\n if (sysctlbyname(\"vm.ndomains\", &ndomains, &len, NULL, 0) == -1) return 0ul;\n return ndomains;\n}\n\n#elif defined(__DragonFly__)\n\nsize_t _mi_prim_numa_node(void) {\n // TODO: DragonFly does not seem to provide any userland means to get this information.\n return 0ul;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n size_t ncpus = 0, nvirtcoresperphys = 0;\n size_t len = sizeof(size_t);\n if (sysctlbyname(\"hw.ncpu\", &ncpus, &len, NULL, 0) == -1) return 0ul;\n if (sysctlbyname(\"hw.cpu_topology_ht_ids\", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul;\n return nvirtcoresperphys * ncpus;\n}\n\n#else\n\nsize_t _mi_prim_numa_node(void) {\n return 0;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n return 1;\n}\n\n#endif\n\n// ----------------------------------------------------------------\n// Clock\n// ----------------------------------------------------------------\n\n#include \n\n#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)\n\nmi_msecs_t _mi_prim_clock_now(void) {\n struct timespec t;\n #ifdef CLOCK_MONOTONIC\n clock_gettime(CLOCK_MONOTONIC, &t);\n #else\n clock_gettime(CLOCK_REALTIME, &t);\n #endif\n return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);\n}\n\n#else\n\n// low resolution timer\nmi_msecs_t _mi_prim_clock_now(void) {\n #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)\n return (mi_msecs_t)clock();\n #elif (CLOCKS_PER_SEC < 1000)\n return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);\n #else\n return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);\n #endif\n}\n\n#endif\n\n\n\n\n//----------------------------------------------------------------\n// Process info\n//----------------------------------------------------------------\n\n#if defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__)\n#include \n#include \n#include \n\n#if defined(__APPLE__)\n#include \n#endif\n\n#if defined(__HAIKU__)\n#include \n#endif\n\nstatic mi_msecs_t timeval_secs(const struct timeval* tv) {\n return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L);\n}\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo)\n{\n struct rusage rusage;\n getrusage(RUSAGE_SELF, &rusage);\n pinfo->utime = timeval_secs(&rusage.ru_utime);\n pinfo->stime = timeval_secs(&rusage.ru_stime);\n#if !defined(__HAIKU__)\n pinfo->page_faults = rusage.ru_majflt;\n#endif\n#if defined(__HAIKU__)\n // Haiku does not have (yet?) a way to\n // get these stats per process\n thread_info tid;\n area_info mem;\n ssize_t c;\n get_thread_info(find_thread(0), &tid);\n while (get_next_area_info(tid.team, &c, &mem) == B_OK) {\n pinfo->peak_rss += mem.ram_size;\n }\n pinfo->page_faults = 0;\n#elif defined(__APPLE__)\n pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes\n #ifdef MACH_TASK_BASIC_INFO\n struct mach_task_basic_info info;\n mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;\n if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {\n pinfo->current_rss = (size_t)info.resident_size;\n }\n #else\n struct task_basic_info info;\n mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT;\n if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {\n pinfo->current_rss = (size_t)info.resident_size;\n }\n #endif\n#else\n pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB\n#endif\n // use defaults for commit\n}\n\n#else\n\n#ifndef __wasi__\n// WebAssembly instances are not processes\n#pragma message(\"define a way to get process info\")\n#endif\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo)\n{\n // use defaults\n MI_UNUSED(pinfo);\n}\n\n#endif\n\n\n//----------------------------------------------------------------\n// Output\n//----------------------------------------------------------------\n\nvoid _mi_prim_out_stderr( const char* msg ) {\n fputs(msg,stderr);\n}\n\n\n//----------------------------------------------------------------\n// Environment\n//----------------------------------------------------------------\n\n#if !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0)\n// On Posix systemsr use `environ` to access environment variables\n// even before the C runtime is initialized.\n#if defined(__APPLE__) && defined(__has_include) && __has_include()\n#include \nstatic char** mi_get_environ(void) {\n return (*_NSGetEnviron());\n}\n#else\nextern char** environ;\nstatic char** mi_get_environ(void) {\n return environ;\n}\n#endif\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size) {\n if (name==NULL) return false;\n const size_t len = _mi_strlen(name);\n if (len == 0) return false;\n char** env = mi_get_environ();\n if (env == NULL) return false;\n // compare up to 10000 entries\n for (int i = 0; i < 10000 && env[i] != NULL; i++) {\n const char* s = env[i];\n if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive\n // found it\n _mi_strlcpy(result, s + len + 1, result_size);\n return true;\n }\n }\n return false;\n}\n#else\n// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size) {\n // cannot call getenv() when still initializing the C runtime.\n if (_mi_preloading()) return false;\n const char* s = getenv(name);\n if (s == NULL) {\n // we check the upper case name too.\n char buf[64+1];\n size_t len = _mi_strnlen(name,sizeof(buf)-1);\n for (size_t i = 0; i < len; i++) {\n buf[i] = _mi_toupper(name[i]);\n }\n buf[len] = 0;\n s = getenv(buf);\n }\n if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false;\n _mi_strlcpy(result, s, result_size);\n return true;\n}\n#endif // !MI_USE_ENVIRON\n\n\n//----------------------------------------------------------------\n// Random\n//----------------------------------------------------------------\n\n#if defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_15)\n#include \n#include \n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n // We prefer CCRandomGenerateBytes as it returns an error code while arc4random_buf\n // may fail silently on macOS. See PR #390, and \n return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess);\n}\n\n#elif defined(__ANDROID__) || defined(__DragonFly__) || \\\n defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \\\n defined(__sun) || \\\n (defined(__APPLE__) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_7))\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n arc4random_buf(buf, buf_len);\n return true;\n}\n\n#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) // also for old apple versions < 10.7 (issue #829)\n\n#include \n#include \n#include \n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n // Modern Linux provides `getrandom` but different distributions either use `sys/random.h` or `linux/random.h`\n // and for the latter the actual `getrandom` call is not always defined.\n // (see )\n // We therefore use a syscall directly and fall back dynamically to /dev/urandom when needed.\n #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getrandom)\n #ifndef GRND_NONBLOCK\n #define GRND_NONBLOCK (1)\n #endif\n static _Atomic(uintptr_t) no_getrandom; // = 0\n if (mi_atomic_load_acquire(&no_getrandom)==0) {\n ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK);\n if (ret >= 0) return (buf_len == (size_t)ret);\n if (errno != ENOSYS) return false;\n mi_atomic_store_release(&no_getrandom, (uintptr_t)1); // don't call again, and fall back to /dev/urandom\n }\n #endif\n int flags = O_RDONLY;\n #if defined(O_CLOEXEC)\n flags |= O_CLOEXEC;\n #endif\n int fd = mi_prim_open(\"/dev/urandom\", flags);\n if (fd < 0) return false;\n size_t count = 0;\n while(count < buf_len) {\n ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count);\n if (ret<=0) {\n if (errno!=EAGAIN && errno!=EINTR) break;\n }\n else {\n count += ret;\n }\n }\n mi_prim_close(fd);\n return (count==buf_len);\n}\n\n#else\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n return false;\n}\n\n#endif\n\n\n//----------------------------------------------------------------\n// Thread init/done\n//----------------------------------------------------------------\n\n#if defined(MI_USE_PTHREADS)\n\n// use pthread local storage keys to detect thread ending\n// (and used with MI_TLS_PTHREADS for the default theap)\npthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);\n\nstatic void mi_pthread_done(void* value) {\n if (value!=NULL) {\n _mi_thread_done((mi_theap_t*)value);\n }\n}\n\nvoid _mi_prim_thread_init_auto_done(void) {\n mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));\n pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);\n}\n\nvoid _mi_prim_thread_done_auto_done(void) {\n if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809\n pthread_key_delete(_mi_heap_default_key);\n }\n}\n\nvoid _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {\n if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD\n pthread_setspecific(_mi_heap_default_key, theap);\n }\n}\n\n#else\n\nvoid _mi_prim_thread_init_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_done_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {\n MI_UNUSED(theap);\n}\n\n#endif\n\nbool _mi_prim_thread_is_in_threadpool(void) {\n return false;\n}\n\nvoid _mi_prim_thread_yield(void) {\n sleep(0);\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/prim/wasi/prim.c", "sha256": "bee9c655a17e1e93bf4908e0a902fd3b566628447b1446fcbfa92bca2e76ca15", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// This file is included in `src/prim/prim.c`\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\"\n\n#include // fputs\n#include // getenv\n#include // sbrk, sleep\n\n//---------------------------------------------\n// Initialize\n//---------------------------------------------\n\nvoid _mi_prim_mem_init( mi_os_mem_config_t* config ) {\n config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB\n config->alloc_granularity = 16;\n config->has_overcommit = false;\n config->has_partial_free = false;\n config->has_virtual_reserve = false;\n}\n\n//---------------------------------------------\n// Free\n//---------------------------------------------\n\nint _mi_prim_free(void* addr, size_t size ) {\n MI_UNUSED(addr); MI_UNUSED(size);\n // wasi theap cannot be shrunk\n return 0;\n}\n\n\n//---------------------------------------------\n// Allocation: sbrk or memory_grow\n//---------------------------------------------\n\n#if defined(MI_USE_SBRK)\n static void* mi_memory_grow( size_t size ) {\n void* p = sbrk(size);\n if (p == (void*)(-1)) return NULL;\n #if !defined(__wasi__) // on wasi this is always zero initialized already (?)\n memset(p,0,size);\n #endif\n return p;\n }\n#elif defined(__wasi__)\n static void* mi_memory_grow( size_t size ) {\n size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size()))\n : __builtin_wasm_memory_size(0));\n if (base == SIZE_MAX) return NULL;\n return (void*)(base * _mi_os_page_size());\n }\n#endif\n\n#if defined(MI_USE_PTHREADS)\nstatic pthread_mutex_t mi_theap_grow_mutex = PTHREAD_MUTEX_INITIALIZER;\n#endif\n\nstatic void* mi_prim_mem_grow(size_t size, size_t try_alignment) {\n void* p = NULL;\n if (try_alignment <= 1) {\n // `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now)\n #if defined(MI_USE_PTHREADS)\n pthread_mutex_lock(&mi_theap_grow_mutex);\n #endif\n p = mi_memory_grow(size);\n #if defined(MI_USE_PTHREADS)\n pthread_mutex_unlock(&mi_theap_grow_mutex);\n #endif\n }\n else {\n void* base = NULL;\n size_t alloc_size = 0;\n // to allocate aligned use a lock to try to avoid thread interaction\n // between getting the current size and actual allocation\n // (also, `sbrk` is not thread safe in general)\n #if defined(MI_USE_PTHREADS)\n pthread_mutex_lock(&mi_theap_grow_mutex);\n #endif\n {\n void* current = mi_memory_grow(0); // get current size\n if (current != NULL) {\n void* aligned_current = _mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space\n alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size());\n base = mi_memory_grow(alloc_size);\n }\n }\n #if defined(MI_USE_PTHREADS)\n pthread_mutex_unlock(&mi_theap_grow_mutex);\n #endif\n if (base != NULL) {\n p = _mi_align_up_ptr(base, try_alignment);\n if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) {\n // another thread used wasm_memory_grow/sbrk in-between and we do not have enough\n // space after alignment. Give up (and waste the space as we cannot shrink :-( )\n // (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align)\n p = NULL;\n }\n }\n }\n /*\n if (p == NULL) {\n _mi_warning_message(\"unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\\n\", size, try_alignment);\n errno = ENOMEM;\n return NULL;\n }\n */\n mi_assert_internal( p == NULL || try_alignment == 0 || (uintptr_t)p % try_alignment == 0 );\n return p;\n}\n\n// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.\nint _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {\n MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);\n *is_large = false;\n *is_zero = false;\n *addr = mi_prim_mem_grow(size, try_alignment);\n return (*addr != NULL ? 0 : ENOMEM);\n}\n\n\n//---------------------------------------------\n// Commit/Reset/Protect\n//---------------------------------------------\n\nint _mi_prim_commit(void* addr, size_t size, bool* is_zero) {\n MI_UNUSED(addr); MI_UNUSED(size);\n *is_zero = false;\n return 0;\n}\n\nint _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {\n MI_UNUSED(addr); MI_UNUSED(size);\n *needs_recommit = false;\n return 0;\n}\n\nint _mi_prim_reset(void* addr, size_t size) {\n MI_UNUSED(addr); MI_UNUSED(size);\n return 0;\n}\n\nint _mi_prim_reuse(void* addr, size_t size) {\n MI_UNUSED(addr); MI_UNUSED(size);\n return 0;\n}\n\nint _mi_prim_protect(void* addr, size_t size, bool protect) {\n MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect);\n return 0;\n}\n\n\n//---------------------------------------------\n// Huge pages and NUMA nodes\n//---------------------------------------------\n\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {\n MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);\n *is_zero = true;\n *addr = NULL;\n return ENOSYS;\n}\n\nsize_t _mi_prim_numa_node(void) {\n return 0;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n return 1;\n}\n\n\n//----------------------------------------------------------------\n// Clock\n//----------------------------------------------------------------\n\n#include \n\n#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)\n\nmi_msecs_t _mi_prim_clock_now(void) {\n struct timespec t;\n #ifdef CLOCK_MONOTONIC\n clock_gettime(CLOCK_MONOTONIC, &t);\n #else\n clock_gettime(CLOCK_REALTIME, &t);\n #endif\n return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);\n}\n\n#else\n\n// low resolution timer\nmi_msecs_t _mi_prim_clock_now(void) {\n #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)\n return (mi_msecs_t)clock();\n #elif (CLOCKS_PER_SEC < 1000)\n return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);\n #else\n return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);\n #endif\n}\n\n#endif\n\n\n//----------------------------------------------------------------\n// Process info\n//----------------------------------------------------------------\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo)\n{\n // use defaults\n MI_UNUSED(pinfo);\n}\n\n\n//----------------------------------------------------------------\n// Output\n//----------------------------------------------------------------\n\nvoid _mi_prim_out_stderr( const char* msg ) {\n fputs(msg,stderr);\n}\n\n\n//----------------------------------------------------------------\n// Environment\n//----------------------------------------------------------------\n\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size) {\n // cannot call getenv() when still initializing the C runtime.\n if (_mi_preloading()) return false;\n const char* s = getenv(name);\n if (s == NULL) {\n // we check the upper case name too.\n char buf[64+1];\n size_t len = _mi_strnlen(name,sizeof(buf)-1);\n for (size_t i = 0; i < len; i++) {\n buf[i] = _mi_toupper(name[i]);\n }\n buf[len] = 0;\n s = getenv(buf);\n }\n if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false;\n _mi_strlcpy(result, s, result_size);\n return true;\n}\n\n\n//----------------------------------------------------------------\n// Random\n//----------------------------------------------------------------\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n return false;\n}\n\n\n//----------------------------------------------------------------\n// Thread init/done\n//----------------------------------------------------------------\n\nvoid _mi_prim_thread_init_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_done_auto_done(void) {\n // nothing\n}\n\nvoid _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {\n MI_UNUSED(theap);\n}\n\nbool _mi_prim_thread_is_in_threadpool(void) {\n return false;\n}\n\nvoid _mi_prim_thread_yield(void) {\n sleep(0);\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/prim/windows/etw.h", "sha256": "4f3110ef2054c95cb275be96cb279224d3d46a728c5117bd1435425f382de778", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "//**********************************************************************`\n//* This is an include file generated by Message Compiler. *`\n//* *`\n//* Copyright (c) Microsoft Corporation. All Rights Reserved. *`\n//**********************************************************************`\n#pragma once\n\n//*****************************************************************************\n//\n// Notes on the ETW event code generated by MC:\n//\n// - Structures and arrays of structures are treated as an opaque binary blob.\n// The caller is responsible for packing the data for the structure into a\n// single region of memory, with no padding between values. The macro will\n// have an extra parameter for the length of the blob.\n// - Arrays of nul-terminated strings must be packed by the caller into a\n// single binary blob containing the correct number of strings, with a nul\n// after each string. The size of the blob is specified in characters, and\n// includes the final nul.\n// - Arrays of SID are treated as a single binary blob. The caller is\n// responsible for packing the SID values into a single region of memory with\n// no padding.\n// - The length attribute on the data element in the manifest is significant\n// for values with intype win:UnicodeString, win:AnsiString, or win:Binary.\n// The length attribute must be specified for win:Binary, and is optional for\n// win:UnicodeString and win:AnsiString (if no length is given, the strings\n// are assumed to be nul-terminated). For win:UnicodeString, the length is\n// measured in characters, not bytes.\n// - For an array of win:UnicodeString, win:AnsiString, or win:Binary, the\n// length attribute applies to every value in the array, so every value in\n// the array must have the same length. The values in the array are provided\n// to the macro via a single pointer -- the caller is responsible for packing\n// all of the values into a single region of memory with no padding between\n// values.\n// - Values of type win:CountedUnicodeString, win:CountedAnsiString, and\n// win:CountedBinary can be generated and collected on Vista or later.\n// However, they may not decode properly without the Windows 10 2018 Fall\n// Update.\n// - Arrays of type win:CountedUnicodeString, win:CountedAnsiString, and\n// win:CountedBinary must be packed by the caller into a single region of\n// memory. The format for each item is a UINT16 byte-count followed by that\n// many bytes of data. When providing the array to the generated macro, you\n// must provide the total size of the packed array data, including the UINT16\n// sizes for each item. In the case of win:CountedUnicodeString, the data\n// size is specified in WCHAR (16-bit) units. In the case of\n// win:CountedAnsiString and win:CountedBinary, the data size is specified in\n// bytes.\n//\n//*****************************************************************************\n\n#include \n#include \n#include \n\n#ifndef ETW_INLINE\n #ifdef _ETW_KM_\n // In kernel mode, save stack space by never inlining templates.\n #define ETW_INLINE DECLSPEC_NOINLINE __inline\n #else\n // In user mode, save code size by inlining templates as appropriate.\n #define ETW_INLINE __inline\n #endif\n#endif // ETW_INLINE\n\n#if defined(__cplusplus)\nextern \"C\" {\n#endif\n\n//\n// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:\n// Define this macro to have the compiler skip the generated functions in this\n// header.\n//\n#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n//\n// MCGEN_USE_KERNEL_MODE_APIS macro:\n// Controls whether the generated code uses kernel-mode or user-mode APIs.\n// - Set to 0 to use Windows user-mode APIs such as EventRegister.\n// - Set to 1 to use Windows kernel-mode APIs such as EtwRegister.\n// Default is based on whether the _ETW_KM_ macro is defined (i.e. by wdm.h).\n// Note that the APIs can also be overridden directly, e.g. by setting the\n// MCGEN_EVENTWRITETRANSFER or MCGEN_EVENTREGISTER macros.\n//\n#ifndef MCGEN_USE_KERNEL_MODE_APIS\n #ifdef _ETW_KM_\n #define MCGEN_USE_KERNEL_MODE_APIS 1\n #else\n #define MCGEN_USE_KERNEL_MODE_APIS 0\n #endif\n#endif // MCGEN_USE_KERNEL_MODE_APIS\n\n//\n// MCGEN_HAVE_EVENTSETINFORMATION macro:\n// Controls how McGenEventSetInformation uses the EventSetInformation API.\n// - Set to 0 to disable the use of EventSetInformation\n// (McGenEventSetInformation will always return an error).\n// - Set to 1 to directly invoke MCGEN_EVENTSETINFORMATION.\n// - Set to 2 to to locate EventSetInformation at runtime via GetProcAddress\n// (user-mode) or MmGetSystemRoutineAddress (kernel-mode).\n// Default is determined as follows:\n// - If MCGEN_EVENTSETINFORMATION has been customized, set to 1\n// (i.e. use MCGEN_EVENTSETINFORMATION).\n// - Else if the target OS version has EventSetInformation, set to 1\n// (i.e. use MCGEN_EVENTSETINFORMATION).\n// - Else set to 2 (i.e. try to dynamically locate EventSetInformation).\n// Note that an McGenEventSetInformation function will only be generated if one\n// or more provider in a manifest has provider traits.\n//\n#ifndef MCGEN_HAVE_EVENTSETINFORMATION\n #ifdef MCGEN_EVENTSETINFORMATION // if MCGEN_EVENTSETINFORMATION has been customized,\n #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...).\n #elif MCGEN_USE_KERNEL_MODE_APIS // else if using kernel-mode APIs,\n #if NTDDI_VERSION >= 0x06040000 // if target OS is Windows 10 or later,\n #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...).\n #else // else\n #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find \"EtwSetInformation\" via MmGetSystemRoutineAddress.\n #endif // else (using user-mode APIs)\n #else // if target OS and SDK is Windows 8 or later,\n #if WINVER >= 0x0602 && defined(EVENT_FILTER_TYPE_SCHEMATIZED)\n #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...).\n #else // else\n #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find \"EventSetInformation\" via GetModuleHandleExW/GetProcAddress.\n #endif\n #endif\n#endif // MCGEN_HAVE_EVENTSETINFORMATION\n\n//\n// MCGEN Override Macros\n//\n// The following override macros may be defined before including this header\n// to control the APIs used by this header:\n//\n// - MCGEN_EVENTREGISTER\n// - MCGEN_EVENTUNREGISTER\n// - MCGEN_EVENTSETINFORMATION\n// - MCGEN_EVENTWRITETRANSFER\n//\n// If the the macro is undefined, the MC implementation will default to the\n// corresponding ETW APIs. For example, if the MCGEN_EVENTREGISTER macro is\n// undefined, the EventRegister[MyProviderName] macro will use EventRegister\n// in user mode and will use EtwRegister in kernel mode.\n//\n// To prevent issues from conflicting definitions of these macros, the value\n// of the override macro will be used as a suffix in certain internal function\n// names. Because of this, the override macros must follow certain rules:\n//\n// - The macro must be defined before any MC-generated header is included and\n// must not be undefined or redefined after any MC-generated header is\n// included. Different translation units (i.e. different .c or .cpp files)\n// may set the macros to different values, but within a translation unit\n// (within a single .c or .cpp file), the macro must be set once and not\n// changed.\n// - The override must be an object-like macro, not a function-like macro\n// (i.e. the override macro must not have a parameter list).\n// - The override macro's value must be a simple identifier, i.e. must be\n// something that starts with a letter or '_' and contains only letters,\n// numbers, and '_' characters.\n// - If the override macro's value is the name of a second object-like macro,\n// the second object-like macro must follow the same rules. (The override\n// macro's value can also be the name of a function-like macro, in which\n// case the function-like macro does not need to follow the same rules.)\n//\n// For example, the following will cause compile errors:\n//\n// #define MCGEN_EVENTWRITETRANSFER MyNamespace::MyClass::MyFunction // Value has non-identifier characters (colon).\n// #define MCGEN_EVENTWRITETRANSFER GetEventWriteFunctionPointer(7) // Value has non-identifier characters (parentheses).\n// #define MCGEN_EVENTWRITETRANSFER(h,e,a,r,c,d) EventWrite(h,e,c,d) // Override is defined as a function-like macro.\n// #define MY_OBJECT_LIKE_MACRO MyNamespace::MyClass::MyEventWriteFunction\n// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // Evaluates to something with non-identifier characters (colon).\n//\n// The following would be ok:\n//\n// #define MCGEN_EVENTWRITETRANSFER MyEventWriteFunction1 // OK, suffix will be \"MyEventWriteFunction1\".\n// #define MY_OBJECT_LIKE_MACRO MyEventWriteFunction2\n// #define MCGEN_EVENTWRITETRANSFER MY_OBJECT_LIKE_MACRO // OK, suffix will be \"MyEventWriteFunction2\".\n// #define MY_FUNCTION_LIKE_MACRO(h,e,a,r,c,d) MyNamespace::MyClass::MyEventWriteFunction3(h,e,c,d)\n// #define MCGEN_EVENTWRITETRANSFER MY_FUNCTION_LIKE_MACRO // OK, suffix will be \"MY_FUNCTION_LIKE_MACRO\".\n//\n#ifndef MCGEN_EVENTREGISTER\n #if MCGEN_USE_KERNEL_MODE_APIS\n #define MCGEN_EVENTREGISTER EtwRegister\n #else\n #define MCGEN_EVENTREGISTER EventRegister\n #endif\n#endif // MCGEN_EVENTREGISTER\n#ifndef MCGEN_EVENTUNREGISTER\n #if MCGEN_USE_KERNEL_MODE_APIS\n #define MCGEN_EVENTUNREGISTER EtwUnregister\n #else\n #define MCGEN_EVENTUNREGISTER EventUnregister\n #endif\n#endif // MCGEN_EVENTUNREGISTER\n#ifndef MCGEN_EVENTSETINFORMATION\n #if MCGEN_USE_KERNEL_MODE_APIS\n #define MCGEN_EVENTSETINFORMATION EtwSetInformation\n #else\n #define MCGEN_EVENTSETINFORMATION EventSetInformation\n #endif\n#endif // MCGEN_EVENTSETINFORMATION\n#ifndef MCGEN_EVENTWRITETRANSFER\n #if MCGEN_USE_KERNEL_MODE_APIS\n #define MCGEN_EVENTWRITETRANSFER EtwWriteTransfer\n #else\n #define MCGEN_EVENTWRITETRANSFER EventWriteTransfer\n #endif\n#endif // MCGEN_EVENTWRITETRANSFER\n\n//\n// MCGEN_EVENT_ENABLED macro:\n// Override to control how the EventWrite[EventName] macros determine whether\n// an event is enabled. The default behavior is for EventWrite[EventName] to\n// use the EventEnabled[EventName] macros.\n//\n#ifndef MCGEN_EVENT_ENABLED\n#define MCGEN_EVENT_ENABLED(EventName) EventEnabled##EventName()\n#endif\n\n//\n// MCGEN_EVENT_ENABLED_FORCONTEXT macro:\n// Override to control how the EventWrite[EventName]_ForContext macros\n// determine whether an event is enabled. The default behavior is for\n// EventWrite[EventName]_ForContext to use the\n// EventEnabled[EventName]_ForContext macros.\n//\n#ifndef MCGEN_EVENT_ENABLED_FORCONTEXT\n#define MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, EventName) EventEnabled##EventName##_ForContext(pContext)\n#endif\n\n//\n// MCGEN_ENABLE_CHECK macro:\n// Determines whether the specified event would be considered as enabled\n// based on the state of the specified context. Slightly faster than calling\n// McGenEventEnabled directly.\n//\n#ifndef MCGEN_ENABLE_CHECK\n#define MCGEN_ENABLE_CHECK(Context, Descriptor) (Context.IsEnabled && McGenEventEnabled(&Context, &Descriptor))\n#endif\n\n#if !defined(MCGEN_TRACE_CONTEXT_DEF)\n#define MCGEN_TRACE_CONTEXT_DEF\n// This structure is for use by MC-generated code and should not be used directly.\ntypedef struct _MCGEN_TRACE_CONTEXT\n{\n TRACEHANDLE RegistrationHandle;\n TRACEHANDLE Logger; // Used as pointer to provider traits.\n ULONGLONG MatchAnyKeyword;\n ULONGLONG MatchAllKeyword;\n ULONG Flags;\n ULONG IsEnabled;\n UCHAR Level;\n UCHAR Reserve;\n USHORT EnableBitsCount;\n PULONG EnableBitMask;\n const ULONGLONG* EnableKeyWords;\n const UCHAR* EnableLevel;\n} MCGEN_TRACE_CONTEXT, *PMCGEN_TRACE_CONTEXT;\n#endif // MCGEN_TRACE_CONTEXT_DEF\n\n#if !defined(MCGEN_LEVEL_KEYWORD_ENABLED_DEF)\n#define MCGEN_LEVEL_KEYWORD_ENABLED_DEF\n//\n// Determines whether an event with a given Level and Keyword would be\n// considered as enabled based on the state of the specified context.\n// Note that you may want to use MCGEN_ENABLE_CHECK instead of calling this\n// function directly.\n//\nFORCEINLINE\nBOOLEAN\nMcGenLevelKeywordEnabled(\n _In_ PMCGEN_TRACE_CONTEXT EnableInfo,\n _In_ UCHAR Level,\n _In_ ULONGLONG Keyword\n )\n{\n //\n // Check if the event Level is lower than the level at which\n // the channel is enabled.\n // If the event Level is 0 or the channel is enabled at level 0,\n // all levels are enabled.\n //\n\n if ((Level <= EnableInfo->Level) || // This also covers the case of Level == 0.\n (EnableInfo->Level == 0)) {\n\n //\n // Check if Keyword is enabled\n //\n\n if ((Keyword == (ULONGLONG)0) ||\n ((Keyword & EnableInfo->MatchAnyKeyword) &&\n ((Keyword & EnableInfo->MatchAllKeyword) == EnableInfo->MatchAllKeyword))) {\n return TRUE;\n }\n }\n\n return FALSE;\n}\n#endif // MCGEN_LEVEL_KEYWORD_ENABLED_DEF\n\n#if !defined(MCGEN_EVENT_ENABLED_DEF)\n#define MCGEN_EVENT_ENABLED_DEF\n//\n// Determines whether the specified event would be considered as enabled based\n// on the state of the specified context. Note that you may want to use\n// MCGEN_ENABLE_CHECK instead of calling this function directly.\n//\nFORCEINLINE\nBOOLEAN\nMcGenEventEnabled(\n _In_ PMCGEN_TRACE_CONTEXT EnableInfo,\n _In_ PCEVENT_DESCRIPTOR EventDescriptor\n )\n{\n return McGenLevelKeywordEnabled(EnableInfo, EventDescriptor->Level, EventDescriptor->Keyword);\n}\n#endif // MCGEN_EVENT_ENABLED_DEF\n\n#if !defined(MCGEN_CONTROL_CALLBACK)\n#define MCGEN_CONTROL_CALLBACK\n\n// This function is for use by MC-generated code and should not be used directly.\nDECLSPEC_NOINLINE __inline\nVOID\n__stdcall\nMcGenControlCallbackV2(\n _In_ LPCGUID SourceId,\n _In_ ULONG ControlCode,\n _In_ UCHAR Level,\n _In_ ULONGLONG MatchAnyKeyword,\n _In_ ULONGLONG MatchAllKeyword,\n _In_opt_ PEVENT_FILTER_DESCRIPTOR FilterData,\n _Inout_opt_ PVOID CallbackContext\n )\n/*++\n\nRoutine Description:\n\n This is the notification callback for Windows Vista and later.\n\nArguments:\n\n SourceId - The GUID that identifies the session that enabled the provider.\n\n ControlCode - The parameter indicates whether the provider\n is being enabled or disabled.\n\n Level - The level at which the event is enabled.\n\n MatchAnyKeyword - The bitmask of keywords that the provider uses to\n determine the category of events that it writes.\n\n MatchAllKeyword - This bitmask additionally restricts the category\n of events that the provider writes.\n\n FilterData - The provider-defined data.\n\n CallbackContext - The context of the callback that is defined when the provider\n called EtwRegister to register itself.\n\nRemarks:\n\n ETW calls this function to notify provider of enable/disable\n\n--*/\n{\n PMCGEN_TRACE_CONTEXT Ctx = (PMCGEN_TRACE_CONTEXT)CallbackContext;\n ULONG Ix;\n#ifndef MCGEN_PRIVATE_ENABLE_CALLBACK_V2\n UNREFERENCED_PARAMETER(SourceId);\n UNREFERENCED_PARAMETER(FilterData);\n#endif\n\n if (Ctx == NULL) {\n return;\n }\n\n switch (ControlCode) {\n\n case EVENT_CONTROL_CODE_ENABLE_PROVIDER:\n Ctx->Level = Level;\n Ctx->MatchAnyKeyword = MatchAnyKeyword;\n Ctx->MatchAllKeyword = MatchAllKeyword;\n Ctx->IsEnabled = EVENT_CONTROL_CODE_ENABLE_PROVIDER;\n\n for (Ix = 0; Ix < Ctx->EnableBitsCount; Ix += 1) {\n if (McGenLevelKeywordEnabled(Ctx, Ctx->EnableLevel[Ix], Ctx->EnableKeyWords[Ix]) != FALSE) {\n Ctx->EnableBitMask[Ix >> 5] |= (1 << (Ix % 32));\n } else {\n Ctx->EnableBitMask[Ix >> 5] &= ~(1 << (Ix % 32));\n }\n }\n break;\n\n case EVENT_CONTROL_CODE_DISABLE_PROVIDER:\n Ctx->IsEnabled = EVENT_CONTROL_CODE_DISABLE_PROVIDER;\n Ctx->Level = 0;\n Ctx->MatchAnyKeyword = 0;\n Ctx->MatchAllKeyword = 0;\n if (Ctx->EnableBitsCount > 0) {\n#pragma warning(suppress: 26451) // Arithmetic overflow cannot occur, no matter the value of EnableBitCount\n RtlZeroMemory(Ctx->EnableBitMask, (((Ctx->EnableBitsCount - 1) / 32) + 1) * sizeof(ULONG));\n }\n break;\n\n default:\n break;\n }\n\n#ifdef MCGEN_PRIVATE_ENABLE_CALLBACK_V2\n //\n // Call user defined callback\n //\n MCGEN_PRIVATE_ENABLE_CALLBACK_V2(\n SourceId,\n ControlCode,\n Level,\n MatchAnyKeyword,\n MatchAllKeyword,\n FilterData,\n CallbackContext\n );\n#endif // MCGEN_PRIVATE_ENABLE_CALLBACK_V2\n\n return;\n}\n\n#endif // MCGEN_CONTROL_CALLBACK\n\n#ifndef _mcgen_PENABLECALLBACK\n #if MCGEN_USE_KERNEL_MODE_APIS\n #define _mcgen_PENABLECALLBACK PETWENABLECALLBACK\n #else\n #define _mcgen_PENABLECALLBACK PENABLECALLBACK\n #endif\n#endif // _mcgen_PENABLECALLBACK\n\n#if !defined(_mcgen_PASTE2)\n// This macro is for use by MC-generated code and should not be used directly.\n#define _mcgen_PASTE2(a, b) _mcgen_PASTE2_imp(a, b)\n#define _mcgen_PASTE2_imp(a, b) a##b\n#endif // _mcgen_PASTE2\n\n#if !defined(_mcgen_PASTE3)\n// This macro is for use by MC-generated code and should not be used directly.\n#define _mcgen_PASTE3(a, b, c) _mcgen_PASTE3_imp(a, b, c)\n#define _mcgen_PASTE3_imp(a, b, c) a##b##_##c\n#endif // _mcgen_PASTE3\n\n//\n// Macro validation\n//\n\n// Validate MCGEN_EVENTREGISTER:\n\n// Trigger an error if MCGEN_EVENTREGISTER is not an unqualified (simple) identifier:\nstruct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER);\n\n// Trigger an error if MCGEN_EVENTREGISTER is redefined:\ntypedef struct _mcgen_PASTE2(MCGEN_EVENTREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTREGISTER)\n MCGEN_EVENTREGISTER_must_not_be_redefined_between_headers;\n\n// Trigger an error if MCGEN_EVENTREGISTER is defined as a function-like macro:\ntypedef void MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTREGISTER;\ntypedef int _mcgen_PASTE2(MCGEN_EVENTREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTREGISTER);\n\n// Validate MCGEN_EVENTUNREGISTER:\n\n// Trigger an error if MCGEN_EVENTUNREGISTER is not an unqualified (simple) identifier:\nstruct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER);\n\n// Trigger an error if MCGEN_EVENTUNREGISTER is redefined:\ntypedef struct _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTUNREGISTER)\n MCGEN_EVENTUNREGISTER_must_not_be_redefined_between_headers;\n\n// Trigger an error if MCGEN_EVENTUNREGISTER is defined as a function-like macro:\ntypedef void MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_MCGEN_EVENTUNREGISTER;\ntypedef int _mcgen_PASTE2(MCGEN_EVENTUNREGISTER_must_not_be_a_functionLike_macro_, MCGEN_EVENTUNREGISTER);\n\n// Validate MCGEN_EVENTSETINFORMATION:\n\n// Trigger an error if MCGEN_EVENTSETINFORMATION is not an unqualified (simple) identifier:\nstruct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION);\n\n// Trigger an error if MCGEN_EVENTSETINFORMATION is redefined:\ntypedef struct _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTSETINFORMATION)\n MCGEN_EVENTSETINFORMATION_must_not_be_redefined_between_headers;\n\n// Trigger an error if MCGEN_EVENTSETINFORMATION is defined as a function-like macro:\ntypedef void MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_MCGEN_EVENTSETINFORMATION;\ntypedef int _mcgen_PASTE2(MCGEN_EVENTSETINFORMATION_must_not_be_a_functionLike_macro_, MCGEN_EVENTSETINFORMATION);\n\n// Validate MCGEN_EVENTWRITETRANSFER:\n\n// Trigger an error if MCGEN_EVENTWRITETRANSFER is not an unqualified (simple) identifier:\nstruct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER);\n\n// Trigger an error if MCGEN_EVENTWRITETRANSFER is redefined:\ntypedef struct _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_definition_must_be_an_unqualified_identifier_, MCGEN_EVENTWRITETRANSFER)\n MCGEN_EVENTWRITETRANSFER_must_not_be_redefined_between_headers;;\n\n// Trigger an error if MCGEN_EVENTWRITETRANSFER is defined as a function-like macro:\ntypedef void MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_MCGEN_EVENTWRITETRANSFER;\ntypedef int _mcgen_PASTE2(MCGEN_EVENTWRITETRANSFER_must_not_be_a_functionLike_macro_, MCGEN_EVENTWRITETRANSFER);\n\n#ifndef McGenEventWrite_def\n#define McGenEventWrite_def\n\n// This macro is for use by MC-generated code and should not be used directly.\n#define McGenEventWrite _mcgen_PASTE2(McGenEventWrite_, MCGEN_EVENTWRITETRANSFER)\n\n// This function is for use by MC-generated code and should not be used directly.\nDECLSPEC_NOINLINE __inline\nULONG __stdcall\nMcGenEventWrite(\n _In_ PMCGEN_TRACE_CONTEXT Context,\n _In_ PCEVENT_DESCRIPTOR Descriptor,\n _In_opt_ LPCGUID ActivityId,\n _In_range_(1, 128) ULONG EventDataCount,\n _Pre_cap_(EventDataCount) EVENT_DATA_DESCRIPTOR* EventData\n )\n{\n const USHORT UNALIGNED* Traits;\n\n // Some customized MCGEN_EVENTWRITETRANSFER macros might ignore ActivityId.\n UNREFERENCED_PARAMETER(ActivityId);\n\n Traits = (const USHORT UNALIGNED*)(UINT_PTR)Context->Logger;\n\n if (Traits == NULL) {\n EventData[0].Ptr = 0;\n EventData[0].Size = 0;\n EventData[0].Reserved = 0;\n } else {\n EventData[0].Ptr = (ULONG_PTR)Traits;\n EventData[0].Size = *Traits;\n EventData[0].Reserved = 2; // EVENT_DATA_DESCRIPTOR_TYPE_PROVIDER_METADATA\n }\n\n return MCGEN_EVENTWRITETRANSFER(\n Context->RegistrationHandle,\n Descriptor,\n ActivityId,\n NULL,\n EventDataCount,\n EventData);\n}\n#endif // McGenEventWrite_def\n\n#if !defined(McGenEventRegisterUnregister)\n#define McGenEventRegisterUnregister\n\n// This macro is for use by MC-generated code and should not be used directly.\n#define McGenEventRegister _mcgen_PASTE2(McGenEventRegister_, MCGEN_EVENTREGISTER)\n\n#pragma warning(push)\n#pragma warning(disable:6103)\n// This function is for use by MC-generated code and should not be used directly.\nDECLSPEC_NOINLINE __inline\nULONG __stdcall\nMcGenEventRegister(\n _In_ LPCGUID ProviderId,\n _In_opt_ _mcgen_PENABLECALLBACK EnableCallback,\n _In_opt_ PVOID CallbackContext,\n _Inout_ PREGHANDLE RegHandle\n )\n/*++\n\nRoutine Description:\n\n This function registers the provider with ETW.\n\nArguments:\n\n ProviderId - Provider ID to register with ETW.\n\n EnableCallback - Callback to be used.\n\n CallbackContext - Context for the callback.\n\n RegHandle - Pointer to registration handle.\n\nRemarks:\n\n Should not be called if the provider is already registered (i.e. should not\n be called if *RegHandle != 0). Repeatedly registering a provider is a bug\n and may indicate a race condition. However, for compatibility with previous\n behavior, this function will return SUCCESS in this case.\n\n--*/\n{\n ULONG Error;\n\n if (*RegHandle != 0)\n {\n Error = 0; // ERROR_SUCCESS\n }\n else\n {\n Error = MCGEN_EVENTREGISTER(ProviderId, EnableCallback, CallbackContext, RegHandle);\n }\n\n return Error;\n}\n#pragma warning(pop)\n\n// This macro is for use by MC-generated code and should not be used directly.\n#define McGenEventUnregister _mcgen_PASTE2(McGenEventUnregister_, MCGEN_EVENTUNREGISTER)\n\n// This function is for use by MC-generated code and should not be used directly.\nDECLSPEC_NOINLINE __inline\nULONG __stdcall\nMcGenEventUnregister(_Inout_ PREGHANDLE RegHandle)\n/*++\n\nRoutine Description:\n\n Unregister from ETW and set *RegHandle = 0.\n\nArguments:\n\n RegHandle - the pointer to the provider registration handle\n\nRemarks:\n\n If provider has not been registered (i.e. if *RegHandle == 0),\n return SUCCESS. It is safe to call McGenEventUnregister even if the\n call to McGenEventRegister returned an error.\n\n--*/\n{\n ULONG Error;\n\n if(*RegHandle == 0)\n {\n Error = 0; // ERROR_SUCCESS\n }\n else\n {\n Error = MCGEN_EVENTUNREGISTER(*RegHandle);\n *RegHandle = (REGHANDLE)0;\n }\n\n return Error;\n}\n\n#endif // McGenEventRegisterUnregister\n\n#ifndef _mcgen_EVENT_BIT_SET\n #if defined(_M_IX86) || defined(_M_X64)\n // This macro is for use by MC-generated code and should not be used directly.\n #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((((const unsigned char*)EnableBits)[BitPosition >> 3] & (1u << (BitPosition & 7))) != 0)\n #else // CPU type\n // This macro is for use by MC-generated code and should not be used directly.\n #define _mcgen_EVENT_BIT_SET(EnableBits, BitPosition) ((EnableBits[BitPosition >> 5] & (1u << (BitPosition & 31))) != 0)\n #endif // CPU type\n#endif // _mcgen_EVENT_BIT_SET\n\n#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n// Provider \"microsoft-windows-mimalloc\" event count 2\n//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n// Provider GUID = 138f4dbb-ee04-4899-aa0a-572ad4475779\nEXTERN_C __declspec(selectany) const GUID ETW_MI_Provider = {0x138f4dbb, 0xee04, 0x4899, {0xaa, 0x0a, 0x57, 0x2a, 0xd4, 0x47, 0x57, 0x79}};\n\n#ifndef ETW_MI_Provider_Traits\n#define ETW_MI_Provider_Traits NULL\n#endif // ETW_MI_Provider_Traits\n\n//\n// Event Descriptors\n//\nEXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_ALLOC = {0x64, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0};\n#define ETW_MI_ALLOC_value 0x64\nEXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR ETW_MI_FREE = {0x65, 0x1, 0x0, 0x4, 0x0, 0x0, 0x0};\n#define ETW_MI_FREE_value 0x65\n\n//\n// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:\n// Define this macro to have the compiler skip the generated functions in this\n// header.\n//\n#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n//\n// Event Enablement Bits\n// These variables are for use by MC-generated code and should not be used directly.\n//\nEXTERN_C __declspec(selectany) DECLSPEC_CACHEALIGN ULONG microsoft_windows_mimallocEnableBits[1];\nEXTERN_C __declspec(selectany) const ULONGLONG microsoft_windows_mimallocKeywords[1] = {0x0};\nEXTERN_C __declspec(selectany) const unsigned char microsoft_windows_mimallocLevels[1] = {4};\n\n//\n// Provider context\n//\nEXTERN_C __declspec(selectany) MCGEN_TRACE_CONTEXT ETW_MI_Provider_Context = {0, (ULONG_PTR)ETW_MI_Provider_Traits, 0, 0, 0, 0, 0, 0, 1, microsoft_windows_mimallocEnableBits, microsoft_windows_mimallocKeywords, microsoft_windows_mimallocLevels};\n\n//\n// Provider REGHANDLE\n//\n#define microsoft_windows_mimallocHandle (ETW_MI_Provider_Context.RegistrationHandle)\n\n//\n// This macro is set to 0, indicating that the EventWrite[Name] macros do not\n// have an Activity parameter. This is controlled by the -km and -um options.\n//\n#define ETW_MI_Provider_EventWriteActivity 0\n\n//\n// Register with ETW using the control GUID specified in the manifest.\n// Invoke this macro during module initialization (i.e. program startup,\n// DLL process attach, or driver load) to initialize the provider.\n// Note that if this function returns an error, the error means that\n// will not work, but no action needs to be taken -- even if EventRegister\n// returns an error, it is generally safe to use EventWrite and\n// EventUnregister macros (they will be no-ops if EventRegister failed).\n//\n#ifndef EventRegistermicrosoft_windows_mimalloc\n#define EventRegistermicrosoft_windows_mimalloc() McGenEventRegister(&ETW_MI_Provider, McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle)\n#endif\n\n//\n// Register with ETW using a specific control GUID (i.e. a GUID other than what\n// is specified in the manifest). Advanced scenarios only.\n//\n#ifndef EventRegisterByGuidmicrosoft_windows_mimalloc\n#define EventRegisterByGuidmicrosoft_windows_mimalloc(Guid) McGenEventRegister(&(Guid), McGenControlCallbackV2, &ETW_MI_Provider_Context, µsoft_windows_mimallocHandle)\n#endif\n\n//\n// Unregister with ETW and close the provider.\n// Invoke this macro during module shutdown (i.e. program exit, DLL process\n// detach, or driver unload) to unregister the provider.\n// Note that you MUST call EventUnregister before DLL or driver unload\n// (not optional): failure to unregister a provider before DLL or driver unload\n// will result in crashes.\n//\n#ifndef EventUnregistermicrosoft_windows_mimalloc\n#define EventUnregistermicrosoft_windows_mimalloc() McGenEventUnregister(µsoft_windows_mimallocHandle)\n#endif\n\n//\n// MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION macro:\n// Define this macro to enable support for caller-allocated provider context.\n//\n#ifdef MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION\n\n//\n// Advanced scenarios: Caller-allocated provider context.\n// Use when multiple differently-configured provider handles are needed,\n// e.g. for container-aware drivers, one context per container.\n//\n// Usage:\n//\n// - Caller enables the feature before including this header, e.g.\n// #define MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION 1\n// - Caller allocates memory, e.g. pContext = malloc(sizeof(McGenContext_microsoft_windows_mimalloc));\n// - Caller registers the provider, e.g. EventRegistermicrosoft_windows_mimalloc_ForContext(pContext);\n// - Caller writes events, e.g. EventWriteMyEvent_ForContext(pContext, ...);\n// - Caller unregisters, e.g. EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext);\n// - Caller frees memory, e.g. free(pContext);\n//\n\ntypedef struct tagMcGenContext_microsoft_windows_mimalloc {\n // The fields of this structure are subject to change and should\n // not be accessed directly. To access the provider's REGHANDLE,\n // use microsoft_windows_mimallocHandle_ForContext(pContext).\n MCGEN_TRACE_CONTEXT Context;\n ULONG EnableBits[1];\n} McGenContext_microsoft_windows_mimalloc;\n\n#define EventRegistermicrosoft_windows_mimalloc_ForContext(pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&ETW_MI_Provider, pContext)\n#define EventRegisterByGuidmicrosoft_windows_mimalloc_ForContext(Guid, pContext) _mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(&(Guid), pContext)\n#define EventUnregistermicrosoft_windows_mimalloc_ForContext(pContext) McGenEventUnregister(&(pContext)->Context.RegistrationHandle)\n\n//\n// Provider REGHANDLE for caller-allocated context.\n//\n#define microsoft_windows_mimallocHandle_ForContext(pContext) ((pContext)->Context.RegistrationHandle)\n\n// This function is for use by MC-generated code and should not be used directly.\n// Initialize and register the caller-allocated context.\n__inline\nULONG __stdcall\n_mcgen_PASTE2(_mcgen_RegisterForContext_microsoft_windows_mimalloc_, MCGEN_EVENTREGISTER)(\n _In_ LPCGUID pProviderId,\n _Out_ McGenContext_microsoft_windows_mimalloc* pContext)\n{\n RtlZeroMemory(pContext, sizeof(*pContext));\n pContext->Context.Logger = (ULONG_PTR)ETW_MI_Provider_Traits;\n pContext->Context.EnableBitsCount = 1;\n pContext->Context.EnableBitMask = pContext->EnableBits;\n pContext->Context.EnableKeyWords = microsoft_windows_mimallocKeywords;\n pContext->Context.EnableLevel = microsoft_windows_mimallocLevels;\n return McGenEventRegister(\n pProviderId,\n McGenControlCallbackV2,\n &pContext->Context,\n &pContext->Context.RegistrationHandle);\n}\n\n// This function is for use by MC-generated code and should not be used directly.\n// Trigger a compile error if called with the wrong parameter type.\nFORCEINLINE\n_Ret_ McGenContext_microsoft_windows_mimalloc*\n_mcgen_CheckContextType_microsoft_windows_mimalloc(_In_ McGenContext_microsoft_windows_mimalloc* pContext)\n{\n return pContext;\n}\n\n#endif // MCGEN_ENABLE_FORCONTEXT_CODE_GENERATION\n\n//\n// Enablement check macro for event \"ETW_MI_ALLOC\"\n//\n#define EventEnabledETW_MI_ALLOC() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0)\n#define EventEnabledETW_MI_ALLOC_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0)\n\n//\n// Event write macros for event \"ETW_MI_ALLOC\"\n//\n#define EventWriteETW_MI_ALLOC(Address, Size) \\\n MCGEN_EVENT_ENABLED(ETW_MI_ALLOC) \\\n ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size) : 0\n#define EventWriteETW_MI_ALLOC_AssumeEnabled(Address, Size) \\\n _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&ETW_MI_Provider_Context, &ETW_MI_ALLOC, Address, Size)\n#define EventWriteETW_MI_ALLOC_ForContext(pContext, Address, Size) \\\n MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_ALLOC) \\\n ? _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&(pContext)->Context, &ETW_MI_ALLOC, Address, Size) : 0\n#define EventWriteETW_MI_ALLOC_ForContextAssumeEnabled(pContext, Address, Size) \\\n _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_ALLOC, Address, Size)\n\n// This macro is for use by MC-generated code and should not be used directly.\n#define _mcgen_TEMPLATE_FOR_ETW_MI_ALLOC _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)\n\n//\n// Enablement check macro for event \"ETW_MI_FREE\"\n//\n#define EventEnabledETW_MI_FREE() _mcgen_EVENT_BIT_SET(microsoft_windows_mimallocEnableBits, 0)\n#define EventEnabledETW_MI_FREE_ForContext(pContext) _mcgen_EVENT_BIT_SET(_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->EnableBits, 0)\n\n//\n// Event write macros for event \"ETW_MI_FREE\"\n//\n#define EventWriteETW_MI_FREE(Address, Size) \\\n MCGEN_EVENT_ENABLED(ETW_MI_FREE) \\\n ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size) : 0\n#define EventWriteETW_MI_FREE_AssumeEnabled(Address, Size) \\\n _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&ETW_MI_Provider_Context, &ETW_MI_FREE, Address, Size)\n#define EventWriteETW_MI_FREE_ForContext(pContext, Address, Size) \\\n MCGEN_EVENT_ENABLED_FORCONTEXT(pContext, ETW_MI_FREE) \\\n ? _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&(pContext)->Context, &ETW_MI_FREE, Address, Size) : 0\n#define EventWriteETW_MI_FREE_ForContextAssumeEnabled(pContext, Address, Size) \\\n _mcgen_TEMPLATE_FOR_ETW_MI_FREE(&_mcgen_CheckContextType_microsoft_windows_mimalloc(pContext)->Context, &ETW_MI_FREE, Address, Size)\n\n// This macro is for use by MC-generated code and should not be used directly.\n#define _mcgen_TEMPLATE_FOR_ETW_MI_FREE _mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)\n\n#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n//\n// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro:\n// Define this macro to have the compiler skip the generated functions in this\n// header.\n//\n#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n//\n// Template Functions\n//\n\n//\n// Function for template \"ETW_CUSTOM_HEAP_ALLOC_DATA\" (and possibly others).\n// This function is for use by MC-generated code and should not be used directly.\n//\n#ifndef McTemplateU0xx_def\n#define McTemplateU0xx_def\nETW_INLINE\nULONG\n_mcgen_PASTE2(McTemplateU0xx_, MCGEN_EVENTWRITETRANSFER)(\n _In_ PMCGEN_TRACE_CONTEXT Context,\n _In_ PCEVENT_DESCRIPTOR Descriptor,\n _In_ const unsigned __int64 _Arg0,\n _In_ const unsigned __int64 _Arg1\n )\n{\n#define McTemplateU0xx_ARGCOUNT 2\n\n EVENT_DATA_DESCRIPTOR EventData[McTemplateU0xx_ARGCOUNT + 1];\n\n EventDataDescCreate(&EventData[1],&_Arg0, sizeof(const unsigned __int64) );\n\n EventDataDescCreate(&EventData[2],&_Arg1, sizeof(const unsigned __int64) );\n\n return McGenEventWrite(Context, Descriptor, NULL, McTemplateU0xx_ARGCOUNT + 1, EventData);\n}\n#endif // McTemplateU0xx_def\n\n#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION\n\n#if defined(__cplusplus)\n}\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/prim/windows/prim.c", "sha256": "5cde60fcef94cd79ab816030be00f8c1bdf18fab343563508c681ec6e762d1f1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2023, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n// This file is included in `src/prim/prim.c`\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\"\n#include // fputs, stderr\n#include // atexit\n\n// xbox has no console IO\n#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP | WINAPI_PARTITION_SYSTEM)\n#define MI_HAS_CONSOLE_IO\n#endif\n\n//---------------------------------------------\n// Dynamically bind Windows API points for portability\n//---------------------------------------------\n\n#if defined(_MSC_VER)\n#pragma warning(disable:4996) // don't use GetVersionExW\n#endif\n\nstatic DWORD win_major_version = 6;\nstatic DWORD win_minor_version = 0;\n\n// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.\n// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)\n// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB)\n// We define a minimal MEM_EXTENDED_PARAMETER ourselves in order to be able to compile with older SDK's.\ntypedef enum MI_MEM_EXTENDED_PARAMETER_TYPE_E {\n MiMemExtendedParameterInvalidType = 0,\n MiMemExtendedParameterAddressRequirements,\n MiMemExtendedParameterNumaNode,\n MiMemExtendedParameterPartitionHandle,\n MiMemExtendedParameterUserPhysicalHandle,\n MiMemExtendedParameterAttributeFlags,\n MiMemExtendedParameterMax\n} MI_MEM_EXTENDED_PARAMETER_TYPE;\n\ntypedef struct DECLSPEC_ALIGN(8) MI_MEM_EXTENDED_PARAMETER_S {\n struct { DWORD64 Type : 8; DWORD64 Reserved : 56; } Type;\n union { DWORD64 ULong64; PVOID Pointer; SIZE_T Size; HANDLE Handle; DWORD ULong; } Arg;\n} MI_MEM_EXTENDED_PARAMETER;\n\ntypedef struct MI_MEM_ADDRESS_REQUIREMENTS_S {\n PVOID LowestStartingAddress;\n PVOID HighestEndingAddress;\n SIZE_T Alignment;\n} MI_MEM_ADDRESS_REQUIREMENTS;\n\n#define MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE 0x00000010\n\n#include \ntypedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG);\ntypedef LONG (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG); // avoid NTSTATUS as it is not defined on xbox (pr #1084)\nstatic PVirtualAlloc2 pVirtualAlloc2 = NULL;\nstatic PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL;\n\n// Similarly, GetNumaProcessorNodeEx is only supported since Windows 7 (and GetNumaNodeProcessorMask is not supported on xbox)\ntypedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER;\n\ntypedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber);\ntypedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(MI_PROCESSOR_NUMBER* Processor, PUSHORT NodeNumber);\ntypedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINITY ProcessorMask);\ntypedef BOOL (__stdcall *PGetNumaProcessorNode)(UCHAR Processor, PUCHAR NodeNumber);\ntypedef BOOL (__stdcall* PGetNumaNodeProcessorMask)(UCHAR Node, PULONGLONG ProcessorMask);\ntypedef BOOL (__stdcall* PGetNumaHighestNodeNumber)(PULONG Node);\nstatic PGetCurrentProcessorNumberEx pGetCurrentProcessorNumberEx = NULL;\nstatic PGetNumaProcessorNodeEx pGetNumaProcessorNodeEx = NULL;\nstatic PGetNumaNodeProcessorMaskEx pGetNumaNodeProcessorMaskEx = NULL;\nstatic PGetNumaProcessorNode pGetNumaProcessorNode = NULL;\nstatic PGetNumaNodeProcessorMask pGetNumaNodeProcessorMask = NULL;\nstatic PGetNumaHighestNodeNumber pGetNumaHighestNodeNumber = NULL;\n\n// Not available on xbox\ntypedef SIZE_T(__stdcall* PGetLargePageMinimum)(VOID);\nstatic PGetLargePageMinimum pGetLargePageMinimum = NULL;\n\n// Available after Windows XP\ntypedef BOOL (__stdcall *PGetPhysicallyInstalledSystemMemory)( PULONGLONG TotalMemoryInKilobytes );\ntypedef BOOL (__stdcall* PGetVersionExW)(LPOSVERSIONINFOW lpVersionInformation);\n\n\n\n//---------------------------------------------\n// Enable large page support dynamically (if possible)\n//---------------------------------------------\n\nstatic bool win_enable_large_os_pages_once(size_t* large_page_size)\n{\n if (pGetLargePageMinimum==NULL) return false; // no large page support (xbox etc.)\n\n // Try to see if large OS pages are supported\n // To use large pages on Windows, we first need access permission\n // Set \"Lock pages in memory\" permission in the group policy editor\n // \n unsigned long err = 0;\n HANDLE token = NULL;\n BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);\n if (ok) {\n TOKEN_PRIVILEGES tp;\n ok = LookupPrivilegeValue(NULL, TEXT(\"SeLockMemoryPrivilege\"), &tp.Privileges[0].Luid);\n if (ok) {\n tp.PrivilegeCount = 1;\n tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;\n ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0);\n if (ok) {\n err = GetLastError();\n ok = (err == ERROR_SUCCESS);\n if (ok && large_page_size != NULL && pGetLargePageMinimum != NULL) {\n *large_page_size = (*pGetLargePageMinimum)();\n }\n }\n }\n CloseHandle(token);\n }\n if (!ok) {\n if (err == 0) err = GetLastError();\n _mi_warning_message(\"cannot enable large OS page support, error %lu\\n\", err);\n }\n return (ok!=0);\n}\n\nstatic bool win_enable_large_os_pages(size_t* large_page_size) {\n mi_atomic_do_once {\n win_enable_large_os_pages_once(large_page_size);\n }\n return (_mi_os_large_page_size() > 0);\n}\n\n\n//---------------------------------------------\n// Initialize\n//---------------------------------------------\n\nstatic DWORD win_allocation_granularity = 64*MI_KiB;\n\nvoid _mi_prim_mem_init( mi_os_mem_config_t* config )\n{\n config->has_overcommit = false;\n config->has_partial_free = false;\n config->has_virtual_reserve = true;\n\n // get the page size\n SYSTEM_INFO si; _mi_memzero_var(si);\n GetSystemInfo(&si);\n if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; }\n if (si.dwAllocationGranularity > 0) {\n config->alloc_granularity = si.dwAllocationGranularity;\n win_allocation_granularity = si.dwAllocationGranularity;\n }\n // get virtual address bits\n if ((uintptr_t)si.lpMaximumApplicationAddress > 0) {\n const size_t vbits = MI_SIZE_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress);\n config->virtual_address_bits = vbits;\n }\n\n // get the VirtualAlloc2 function\n HINSTANCE hDll = LoadLibrary(TEXT(\"kernelbase.dll\"));\n if (hDll != NULL) {\n // use VirtualAlloc2FromApp if possible as it is available to Windows store apps\n pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, \"VirtualAlloc2FromApp\");\n if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, \"VirtualAlloc2\");\n FreeLibrary(hDll);\n }\n // NtAllocateVirtualMemoryEx is used for huge page allocation\n hDll = LoadLibrary(TEXT(\"ntdll.dll\"));\n if (hDll != NULL) {\n pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, \"NtAllocateVirtualMemoryEx\");\n FreeLibrary(hDll);\n }\n // Try to use Win7+ numa API\n hDll = LoadLibrary(TEXT(\"kernel32.dll\"));\n if (hDll != NULL) {\n pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, \"GetCurrentProcessorNumberEx\");\n pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, \"GetNumaProcessorNodeEx\");\n pGetNumaNodeProcessorMaskEx = (PGetNumaNodeProcessorMaskEx)(void (*)(void))GetProcAddress(hDll, \"GetNumaNodeProcessorMaskEx\");\n pGetNumaProcessorNode = (PGetNumaProcessorNode)(void (*)(void))GetProcAddress(hDll, \"GetNumaProcessorNode\");\n pGetNumaNodeProcessorMask = (PGetNumaNodeProcessorMask)(void (*)(void))GetProcAddress(hDll, \"GetNumaNodeProcessorMask\");\n pGetNumaHighestNodeNumber = (PGetNumaHighestNodeNumber)(void (*)(void))GetProcAddress(hDll, \"GetNumaHighestNodeNumber\");\n pGetLargePageMinimum = (PGetLargePageMinimum)(void (*)(void))GetProcAddress(hDll, \"GetLargePageMinimum\");\n // Get physical memory (not available on XP, so check dynamically)\n PGetPhysicallyInstalledSystemMemory pGetPhysicallyInstalledSystemMemory = (PGetPhysicallyInstalledSystemMemory)(void (*)(void))GetProcAddress(hDll,\"GetPhysicallyInstalledSystemMemory\");\n if (pGetPhysicallyInstalledSystemMemory != NULL) {\n ULONGLONG memInKiB = 0;\n if ((*pGetPhysicallyInstalledSystemMemory)(&memInKiB)) {\n if (memInKiB > 0 && memInKiB <= SIZE_MAX) {\n config->physical_memory_in_kib = (size_t)memInKiB;\n }\n }\n }\n // Get Windows version\n PGetVersionExW pGetVersionExW = (PGetVersionExW)(void (*)(void))GetProcAddress(hDll, \"GetVersionExW\");\n if (pGetVersionExW != NULL) {\n OSVERSIONINFOW version; _mi_memzero_var(version);\n version.dwOSVersionInfoSize = sizeof(version);\n if ((*pGetVersionExW)(&version)) {\n win_major_version = version.dwMajorVersion;\n win_minor_version = version.dwMinorVersion;\n }\n }\n FreeLibrary(hDll);\n }\n // Enable large/huge OS page support?\n if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {\n win_enable_large_os_pages(&config->large_page_size);\n }\n}\n\n\n//---------------------------------------------\n// Free\n//---------------------------------------------\n\nint _mi_prim_free(void* addr, size_t size ) {\n MI_UNUSED(size);\n DWORD errcode = 0;\n bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0);\n if (err) { errcode = GetLastError(); }\n if (errcode == ERROR_INVALID_ADDRESS) {\n // In mi_os_mem_alloc_aligned the fallback path may have returned a pointer inside\n // the memory region returned by VirtualAlloc; in that case we need to free using\n // the start of the region.\n MEMORY_BASIC_INFORMATION info; _mi_memzero_var(info);\n VirtualQuery(addr, &info, sizeof(info));\n if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) {\n errcode = 0;\n err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0);\n if (err) { errcode = GetLastError(); }\n }\n }\n return (int)errcode;\n}\n\n\n//---------------------------------------------\n// VirtualAlloc\n//---------------------------------------------\n\nstatic void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) {\n #if (MI_INTPTR_SIZE >= 8)\n // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations\n if (addr == NULL) {\n void* hint = _mi_os_get_aligned_hint(try_alignment,size);\n if (hint != NULL) {\n void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE);\n if (p != NULL) return p;\n _mi_verbose_message(\"warning: unable to allocate hinted aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\\n\", size, GetLastError(), hint, try_alignment, flags);\n // fall through on error\n }\n }\n #endif\n // on modern Windows try use VirtualAlloc2 for aligned allocation\n if (addr == NULL && try_alignment > win_allocation_granularity && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {\n MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 };\n reqs.Alignment = try_alignment;\n MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} };\n param.Type.Type = MiMemExtendedParameterAddressRequirements;\n param.Arg.Pointer = &reqs;\n void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1);\n if (p != NULL) return p;\n _mi_warning_message(\"unable to allocate aligned OS memory (0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\\n\", size, GetLastError(), addr, try_alignment, flags);\n // fall through on error\n }\n // last resort\n return VirtualAlloc(addr, size, flags, PAGE_READWRITE);\n}\n\nstatic bool win_is_out_of_memory_error(DWORD err) {\n switch (err) {\n case ERROR_COMMITMENT_MINIMUM:\n case ERROR_COMMITMENT_LIMIT:\n case ERROR_PAGEFILE_QUOTA:\n case ERROR_NOT_ENOUGH_MEMORY:\n return true;\n default:\n return false;\n }\n}\n\nstatic void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) {\n long max_retry_msecs = mi_option_get_clamp(mi_option_retry_on_oom, 0, 2000); // at most 2 seconds\n if (max_retry_msecs == 1) { max_retry_msecs = 100; } // if one sets the option to \"true\"\n for (long tries = 1; tries <= 10; tries++) { // try at most 10 times (=2200ms)\n void* p = win_virtual_alloc_prim_once(addr, size, try_alignment, flags);\n if (p != NULL) {\n // success, return the address\n return p;\n }\n else if (max_retry_msecs > 0 && (try_alignment <= 8*MI_MiB) &&\n (flags&MEM_COMMIT) != 0 && (flags&MEM_LARGE_PAGES) == 0 &&\n win_is_out_of_memory_error(GetLastError())) {\n // if committing regular memory and being out-of-memory,\n // keep trying for a bit in case memory frees up after all. See issue #894\n _mi_warning_message(\"out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\\n\", tries, size, GetLastError(), addr, try_alignment, flags);\n long sleep_msecs = tries*40; // increasing waits\n if (sleep_msecs > max_retry_msecs) { sleep_msecs = max_retry_msecs; }\n max_retry_msecs -= sleep_msecs;\n Sleep(sleep_msecs);\n }\n else {\n // otherwise return with an error\n break;\n }\n }\n return NULL;\n}\n\nstatic void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) {\n mi_assert_internal(!(large_only && !allow_large));\n static _Atomic(size_t) large_page_try_ok; // = 0;\n void* p = NULL;\n // Try to allocate large OS pages (2MiB) if allowed or required.\n if ((large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))\n && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0)\n {\n size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);\n if (!large_only && try_ok > 0) {\n // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive.\n // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times.\n mi_atomic_cas_strong_acq_rel(&large_page_try_ok, &try_ok, try_ok - 1);\n }\n else {\n // large OS pages must always reserve and commit.\n *is_large = true;\n p = win_virtual_alloc_prim(addr, size, try_alignment, flags | MEM_LARGE_PAGES);\n if (large_only) return p;\n // fall back to non-large page allocation on error (`p == NULL`).\n if (p == NULL) {\n mi_atomic_store_release(&large_page_try_ok,10UL); // on error, don't try again for the next N allocations\n }\n }\n }\n // Fall back to regular page allocation\n if (p == NULL) {\n *is_large = ((flags&MEM_LARGE_PAGES) != 0);\n p = win_virtual_alloc_prim(addr, size, try_alignment, flags);\n }\n //if (p == NULL) { _mi_warning_message(\"unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\\n\", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); }\n return p;\n}\n\nint _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {\n mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);\n mi_assert_internal(commit || !allow_large);\n mi_assert_internal(try_alignment > 0);\n *is_zero = true;\n int flags = MEM_RESERVE;\n if (commit) { flags |= MEM_COMMIT; }\n *addr = win_virtual_alloc(hint_addr, size, try_alignment, flags, false, allow_large, is_large);\n return (*addr != NULL ? 0 : (int)GetLastError());\n}\n\n\n//---------------------------------------------\n// Commit/Reset/Protect\n//---------------------------------------------\n#ifdef _MSC_VER\n#pragma warning(disable:6250) // suppress warning calling VirtualFree without MEM_RELEASE (for decommit)\n#endif\n\nint _mi_prim_commit(void* addr, size_t size, bool* is_zero) {\n *is_zero = false;\n /*\n // zero'ing only happens on an initial commit... but checking upfront seems expensive..\n _MEMORY_BASIC_INFORMATION meminfo; _mi_memzero_var(meminfo);\n if (VirtualQuery(addr, &meminfo, size) > 0) {\n if ((meminfo.State & MEM_COMMIT) == 0) {\n *is_zero = true;\n }\n }\n */\n // commit\n void* p = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);\n if (p == NULL) return (int)GetLastError();\n return 0;\n}\n\nint _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {\n BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT);\n *needs_recommit = true; // for safety, assume always decommitted even in the case of an error.\n return (ok ? 0 : (int)GetLastError());\n}\n\nint _mi_prim_reset(void* addr, size_t size) {\n void* p = VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);\n mi_assert_internal(p == addr);\n #if 0\n if (p != NULL) {\n VirtualUnlock(addr,size); // VirtualUnlock after MEM_RESET removes the memory directly from the working set\n }\n #endif\n return (p != NULL ? 0 : (int)GetLastError());\n}\n\nint _mi_prim_reuse(void* addr, size_t size) {\n MI_UNUSED(addr); MI_UNUSED(size);\n return 0;\n}\n\nint _mi_prim_protect(void* addr, size_t size, bool protect) {\n DWORD oldprotect = 0;\n BOOL ok = VirtualProtect(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect);\n return (ok ? 0 : (int)GetLastError());\n}\n\n\n//---------------------------------------------\n// Huge page allocation\n//---------------------------------------------\n\nstatic void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int numa_node)\n{\n const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE;\n\n win_enable_large_os_pages(NULL);\n\n MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} };\n // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages\n static _Atomic(size_t) mi_huge_pages_available = MI_ATOMIC_VAR_INIT(1);\n if (pNtAllocateVirtualMemoryEx != NULL && mi_atomic_load_acquire(&mi_huge_pages_available) != 0) {\n params[0].Type.Type = MiMemExtendedParameterAttributeFlags;\n params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;\n ULONG param_count = 1;\n if (numa_node >= 0) {\n param_count++;\n params[1].Type.Type = MiMemExtendedParameterNumaNode;\n params[1].Arg.ULong = (unsigned)numa_node;\n }\n SIZE_T psize = size;\n void* base = hint_addr;\n LONG err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count);\n if (err == 0 && base != NULL) {\n return base;\n }\n else {\n // fall back to regular large pages\n mi_atomic_store_release(&mi_huge_pages_available,0); // don't try further huge pages\n _mi_warning_message(\"unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\\n\", err);\n }\n }\n // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation\n if (pVirtualAlloc2 != NULL && numa_node >= 0) {\n params[0].Type.Type = MiMemExtendedParameterNumaNode;\n params[0].Arg.ULong = (unsigned)numa_node;\n return (*pVirtualAlloc2)(GetCurrentProcess(), hint_addr, size, flags, PAGE_READWRITE, params, 1);\n }\n\n // otherwise use regular virtual alloc on older windows\n return VirtualAlloc(hint_addr, size, flags, PAGE_READWRITE);\n}\n\nint _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {\n *is_zero = true;\n *addr = _mi_prim_alloc_huge_os_pagesx(hint_addr,size,numa_node);\n return (*addr != NULL ? 0 : (int)GetLastError());\n}\n\n\n//---------------------------------------------\n// Numa nodes\n//---------------------------------------------\n\nsize_t _mi_prim_numa_node(void) {\n USHORT numa_node = 0;\n if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) {\n // Extended API is supported\n MI_PROCESSOR_NUMBER pnum;\n (*pGetCurrentProcessorNumberEx)(&pnum);\n USHORT nnode = 0;\n BOOL ok = (*pGetNumaProcessorNodeEx)(&pnum, &nnode);\n if (ok) { numa_node = nnode; }\n }\n else if (pGetNumaProcessorNode != NULL) {\n // Vista or earlier, use older API that is limited to 64 processors. Issue #277\n DWORD pnum = GetCurrentProcessorNumber();\n UCHAR nnode = 0;\n BOOL ok = pGetNumaProcessorNode((UCHAR)pnum, &nnode);\n if (ok) { numa_node = nnode; }\n }\n return numa_node;\n}\n\nsize_t _mi_prim_numa_node_count(void) {\n ULONG numa_max = 0;\n if (pGetNumaHighestNodeNumber!=NULL) {\n (*pGetNumaHighestNodeNumber)(&numa_max);\n }\n // find the highest node number that has actual processors assigned to it. Issue #282\n while (numa_max > 0) {\n if (pGetNumaNodeProcessorMaskEx != NULL) {\n // Extended API is supported\n GROUP_AFFINITY affinity;\n if ((*pGetNumaNodeProcessorMaskEx)((USHORT)numa_max, &affinity)) {\n if (affinity.Mask != 0) break; // found the maximum non-empty node\n }\n }\n else {\n // Vista or earlier, use older API that is limited to 64 processors.\n ULONGLONG mask;\n if (pGetNumaNodeProcessorMask != NULL) {\n if ((*pGetNumaNodeProcessorMask)((UCHAR)numa_max, &mask)) {\n if (mask != 0) break; // found the maximum non-empty node\n }\n };\n }\n // max node was invalid or had no processor assigned, try again\n numa_max--;\n }\n return ((size_t)numa_max + 1);\n}\n\n\n//----------------------------------------------------------------\n// Clock\n//----------------------------------------------------------------\n\nstatic mi_msecs_t mi_to_msecs(LARGE_INTEGER t) {\n static LARGE_INTEGER mfreq; // = 0\n if (mfreq.QuadPart == 0LL) {\n LARGE_INTEGER f;\n QueryPerformanceFrequency(&f);\n mfreq.QuadPart = f.QuadPart/1000LL;\n if (mfreq.QuadPart == 0) mfreq.QuadPart = 1;\n }\n return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart);\n}\n\nmi_msecs_t _mi_prim_clock_now(void) {\n LARGE_INTEGER t;\n QueryPerformanceCounter(&t);\n return mi_to_msecs(t);\n}\n\n\n//----------------------------------------------------------------\n// Process Info\n//----------------------------------------------------------------\n\n#include \n\nstatic mi_msecs_t filetime_msecs(const FILETIME* ftime) {\n ULARGE_INTEGER i;\n i.LowPart = ftime->dwLowDateTime;\n i.HighPart = ftime->dwHighDateTime;\n mi_msecs_t msecs = (i.QuadPart / 10000); // FILETIME is in 100 nano seconds\n return msecs;\n}\n\ntypedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD);\nstatic PGetProcessMemoryInfo pGetProcessMemoryInfo = NULL;\n\nvoid _mi_prim_process_info(mi_process_info_t* pinfo)\n{\n FILETIME ct;\n FILETIME ut;\n FILETIME st;\n FILETIME et;\n GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut);\n pinfo->utime = filetime_msecs(&ut);\n pinfo->stime = filetime_msecs(&st);\n\n // load psapi on demand\n mi_atomic_do_once {\n HINSTANCE hDll = LoadLibrary(TEXT(\"psapi.dll\"));\n if (hDll != NULL) {\n pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, \"GetProcessMemoryInfo\");\n // FreeLibrary(hDll); // don't free\n }\n }\n\n // get process info\n PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info);\n if (pGetProcessMemoryInfo != NULL) {\n pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));\n }\n pinfo->current_rss = (size_t)info.WorkingSetSize;\n pinfo->peak_rss = (size_t)info.PeakWorkingSetSize;\n pinfo->current_commit = (size_t)info.PagefileUsage;\n pinfo->peak_commit = (size_t)info.PeakPagefileUsage;\n pinfo->page_faults = (size_t)info.PageFaultCount;\n}\n\n//----------------------------------------------------------------\n// Output\n//----------------------------------------------------------------\n\nvoid _mi_prim_out_stderr( const char* msg )\n{\n // on windows with redirection, the C runtime cannot handle locale dependent output\n // after the main thread closes so we use direct console output.\n if (!_mi_preloading()) {\n // _cputs(msg); // _cputs cannot be used as it aborts when failing to lock the console\n static HANDLE hcon = INVALID_HANDLE_VALUE;\n static bool hconIsConsole = false;\n if (hcon == INVALID_HANDLE_VALUE) {\n hcon = GetStdHandle(STD_ERROR_HANDLE);\n #ifdef MI_HAS_CONSOLE_IO\n CONSOLE_SCREEN_BUFFER_INFO sbi;\n hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi));\n #endif\n }\n const size_t len = _mi_strlen(msg);\n if (len > 0 && len < UINT32_MAX) {\n DWORD written = 0;\n if (hconIsConsole) {\n #ifdef MI_HAS_CONSOLE_IO\n WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL);\n #endif\n }\n else if (hcon != INVALID_HANDLE_VALUE) {\n // use direct write if stderr was redirected\n WriteFile(hcon, msg, (DWORD)len, &written, NULL);\n }\n else {\n // finally fall back to fputs after all\n fputs(msg, stderr);\n }\n }\n }\n}\n\n\n//----------------------------------------------------------------\n// Environment\n//----------------------------------------------------------------\n\n// On Windows use GetEnvironmentVariable instead of getenv to work\n// reliably even when this is invoked before the C runtime is initialized.\n// i.e. when `_mi_preloading() == true`.\n// Note: on windows, environment names are not case sensitive.\nbool _mi_prim_getenv(const char* name, char* result, size_t result_size) {\n result[0] = 0;\n const size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size);\n return (len > 0 && len < result_size);\n}\n\n\n//----------------------------------------------------------------\n// Random\n//----------------------------------------------------------------\n\n#if defined(MI_USE_RTLGENRANDOM) // || defined(__cplusplus)\n// We prefer to use BCryptGenRandom instead of (the unofficial) RtlGenRandom but when using\n// dynamic overriding, we observed it can raise an exception when compiled with C++, and\n// sometimes deadlocks when also running under the VS debugger.\n// In contrast, issue #623 implies that on Windows Server 2019 we need to use BCryptGenRandom.\n// To be continued..\n#pragma comment (lib,\"advapi32.lib\")\n#define RtlGenRandom SystemFunction036\nmi_decl_externc BOOLEAN NTAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength);\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n return (RtlGenRandom(buf, (ULONG)buf_len) != 0);\n}\n\n#else\n\n#ifndef BCRYPT_USE_SYSTEM_PREFERRED_RNG\n#define BCRYPT_USE_SYSTEM_PREFERRED_RNG 0x00000002\n#endif\n\ntypedef LONG (NTAPI *PBCryptGenRandom)(HANDLE, PUCHAR, ULONG, ULONG);\nstatic PBCryptGenRandom pBCryptGenRandom = NULL;\n\nbool _mi_prim_random_buf(void* buf, size_t buf_len) {\n mi_assert(buf_len <= ULONG_MAX);\n if (buf_len > ULONG_MAX) return false;\n mi_atomic_do_once {\n HINSTANCE hDll = LoadLibrary(TEXT(\"bcrypt.dll\"));\n if (hDll != NULL) {\n pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, \"BCryptGenRandom\");\n // FreeLibrary(hDll); // don't free\n }\n }\n if (pBCryptGenRandom == NULL) return false;\n return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0);\n}\n\n#endif // MI_USE_RTLGENRANDOM\n\n\n//----------------------------------------------------------------\n// Thread pool?\n//----------------------------------------------------------------\n\nbool _mi_prim_thread_is_in_threadpool(void) {\n#if (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64)\n if (win_major_version >= 6) {\n // check if this thread belongs to a windows threadpool\n // see: \n struct _TEB* const teb = NtCurrentTeb();\n void* const pool_data = *((void**)((uint8_t*)teb + (MI_SIZE_BITS == 32 ? 0x0F90 : 0x1778)));\n return (pool_data != NULL);\n }\n#endif\n return false;\n}\n\nvoid _mi_prim_thread_yield(void) {\n SwitchToThread();\n}\n\n//----------------------------------------------------------------\n// Process & Thread Init/Done\n//----------------------------------------------------------------\n\n//static void mi_debug_out(const char* s) {\n// HANDLE h = GetStdHandle(STD_ERROR_HANDLE);\n// WriteConsole(h, s, (DWORD)_mi_strlen(s), NULL, NULL);\n//}\n\nstatic void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {\n MI_UNUSED(reserved);\n MI_UNUSED(module);\n if (reason==DLL_PROCESS_ATTACH) {\n _mi_auto_process_init();\n }\n else if (reason==DLL_PROCESS_DETACH) {\n _mi_auto_process_done();\n }\n else if (reason==DLL_THREAD_DETACH && !_mi_is_redirected()) {\n _mi_thread_done(NULL);\n }\n}\n\n\n/* -----------------------------------------------------------------------\n Auto initialize and finalize mimalloc on process and thread start/end.\n By default we use a combination of CRT init and TLS sections for\n both static and dynamic linkage (`MI_WIN_INIT_USE_CRT_TLS`).\n------------------------------------------------------------------------- */\n#if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS)\n #if !defined(__INTEL_LLVM_COMPILER) && !defined(__INTEL_COMPILER)\n #define MI_WIN_INIT_USE_CRT_TLS 1 \n #else\n #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* default for Intel ICX, see issue #1268 */ \n #endif\n#endif\n\n#if defined(MI_WIN_INIT_USE_CRT_TLS)\n #define MI_PRIM_HAS_PROCESS_ATTACH 1\n // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.\n void _mi_prim_thread_init_auto_done(void) {}\n void _mi_prim_thread_done_auto_done(void) {}\n void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {\n MI_UNUSED(theap);\n }\n\n static bool mi_module_is_dll(PVOID mod) {\n if (mod==NULL) return false;\n PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod;\n PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew);\n return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL);\n }\n\n static bool mi_current_module_is_dll(void) {\n HMODULE mod = NULL;\n const BOOL ok = GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCSTR)mi_current_module_is_dll, &mod);\n return (ok && mi_module_is_dll(mod));\n }\n\n // Hook into CRT initialization and finalization.\n static void mi_cdecl mi_crt_done(void) {\n // mi_debug_out(mi_current_module_is_dll() ? \"crt dll done\\n\" : \"crt exe done\\n\");\n mi_win_main(NULL, DLL_PROCESS_DETACH, 0);\n }\n\n static int mi_cdecl mi_crt_init(void) {\n // mi_debug_out(mi_current_module_is_dll() ? \"crt dll init\\n\" : \"crt exe init\\n\");\n if (mi_current_module_is_dll()) {\n // in a dll, atexit (crt_done) is called after tls process detach\n atexit(&mi_crt_done);\n }\n return 0;\n }\n\n // We also hook into the Windows loader TLS initialization and finalization.\n static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_THREAD_ATTACH) {\n //mi_debug_out(\"tls thread attach\\n\");\n mi_win_main(module, reason, reserved);\n }\n else if (reason == DLL_PROCESS_ATTACH) {\n // mi_debug_out(mi_module_is_dll(module) ? \"tls dll process attach\\n\" : \"tls exe process attach\\n\");\n // tls process attach is always called before crt init\n mi_win_main(module, reason, reserved);\n }\n }\n\n static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_THREAD_DETACH) {\n //mi_debug_out(\"tls thread detach\\n\");\n mi_win_main(module, reason, reserved);\n }\n else if (reason == DLL_PROCESS_DETACH) {\n // mi_debug_out(mi_module_is_dll(module) ? \"tls dll process detach\\n\" : \"tls exe process detach\\n\");\n if (!mi_module_is_dll(module)) {\n // in an exe, tls process detach is called after atexit (mi_crt_done)\n mi_win_main(module, reason, reserved);\n }\n }\n }\n\n // Set up TLS callbacks in a statically linked library by using special data sections.\n // See \n // We use 2 entries to ensure we call attach events before constructors\n // are called, and detach events after destructors are called.\n // We also set up a CRT initialization callback.\n #if defined(__cplusplus)\n extern \"C\" {\n #endif\n\n typedef int (mi_cdecl* mi_crt_callback_t)(void);\n #if defined(_WIN64)\n #pragma comment(linker, \"/INCLUDE:_tls_used\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_post\")\n #pragma const_seg(\".CRT$XLB\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };\n #pragma const_seg()\n #pragma const_seg(\".CRT$XLY\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };\n #pragma const_seg()\n #pragma const_seg(\".CRT$XIB\")\n extern const mi_crt_callback_t _mi_crt_callback_init[];\n const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };\n #pragma const_seg()\n #else\n #pragma comment(linker, \"/INCLUDE:__tls_used\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_post\")\n #pragma data_seg(\".CRT$XLB\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };\n #pragma data_seg()\n #pragma data_seg(\".CRT$XLY\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };\n #pragma data_seg()\n #pragma data_seg(\".CRT$XIB\")\n mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };\n #pragma data_seg()\n #endif\n\n #if defined(__cplusplus)\n }\n #endif\n\n/* ----------------------------------------------------------------\n Here we use a combination of the raw DLL main and TLS sections for\n both static and dynamic linkage. This is perhaps the most robust as\n the raw DLL main ensure we can close mimalloc latest possible.\n However, it can lead to link errors with on `_pRawDllMain` if other\n libraries also try to hook into this (like Boost).\n*/\n#elif defined(MI_WIN_INIT_USE_RAW_DLLMAIN)\n #define MI_PRIM_HAS_PROCESS_ATTACH 1\n // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.\n void _mi_prim_thread_init_auto_done(void) {}\n void _mi_prim_thread_done_auto_done(void) {}\n void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {\n MI_UNUSED(theap);\n }\n\n // If linked into a DLL module, this raw entry is called before the CRT attach and\n // after the CRT detach through the CRT _pRawDllMain pointer.\n static BOOL NTAPI mi_dll_main_raw(PVOID module, DWORD reason, LPVOID reserved) {\n mi_win_main(module, reason, reserved);\n return TRUE;\n }\n\n // Set the value of the CRT _pRawDllMain pointer\n #if defined(__cplusplus)\n extern \"C\"\n #endif\n PVOID _pRawDllMain = (PVOID)&mi_dll_main_raw;\n\n // We also hook into the Windows loader TLS initialization and finalization.\n // If we are linked into an EXE module we rely on these as `mi_dll_main_raw`\n // is not called (and otherwise we ignore the TLS callbacks by checking if we are in a DLL).\n static bool mi_module_is_dll(PVOID mod) {\n if (mod==NULL) return false;\n PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod;\n PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew);\n return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL);\n }\n\n static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {\n if (!mi_module_is_dll(module)) {\n mi_win_main(module, reason, reserved);\n }\n }\n }\n\n static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {\n if (!mi_module_is_dll(module)) {\n mi_win_main(module, reason, reserved);\n }\n }\n }\n\n // Set up TLS callbacks in a statically linked library by using special data sections.\n // See \n // We use 2 entries to ensure we call attach events before constructors\n // are called, and detach events after destructors are called.\n #if defined(__cplusplus)\n extern \"C\" {\n #endif\n\n #if defined(_WIN64)\n #pragma comment(linker, \"/INCLUDE:_tls_used\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_post\")\n #pragma const_seg(\".CRT$XLB\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };\n #pragma const_seg()\n #pragma const_seg(\".CRT$XLY\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };\n #pragma const_seg()\n #else\n #pragma comment(linker, \"/INCLUDE:__tls_used\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_post\")\n #pragma data_seg(\".CRT$XLB\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };\n #pragma data_seg()\n #pragma data_seg(\".CRT$XLY\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };\n #pragma data_seg()\n #endif\n\n #if defined(__cplusplus)\n }\n #endif\n\n/* --------------------------------------------------------------------\n Legacy options: DllMain, TLS, and FLS\n --------------------------------------------------------------------*/\n#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && defined(MI_SHARED_LIB)\n #define MI_PRIM_HAS_PROCESS_ATTACH 1\n\n // Windows DLL: easy to hook into process_init and thread_done\n BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) {\n mi_win_main((PVOID)inst,reason,reserved);\n return TRUE;\n }\n\n // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.\n void _mi_prim_thread_init_auto_done(void) { }\n void _mi_prim_thread_done_auto_done(void) { }\n void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {\n MI_UNUSED(theap);\n }\n\n#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN)\n #define MI_PRIM_HAS_PROCESS_ATTACH 1\n\n static void NTAPI mi_win_main_attach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {\n mi_win_main(module, reason, reserved);\n }\n }\n static void NTAPI mi_win_main_detach(PVOID module, DWORD reason, LPVOID reserved) {\n if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {\n mi_win_main(module, reason, reserved);\n }\n }\n\n // Set up TLS callbacks in a statically linked library by using special data sections.\n // See \n // We use 2 entries to ensure we call attach events before constructors\n // are called, and detach events after destructors are called.\n #if defined(__cplusplus)\n extern \"C\" {\n #endif\n\n #if defined(_WIN64)\n #pragma comment(linker, \"/INCLUDE:_tls_used\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback_post\")\n #pragma const_seg(\".CRT$XLB\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };\n #pragma const_seg()\n #pragma const_seg(\".CRT$XLY\")\n extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];\n const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };\n #pragma const_seg()\n #else\n #pragma comment(linker, \"/INCLUDE:__tls_used\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_pre\")\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback_post\")\n #pragma data_seg(\".CRT$XLB\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };\n #pragma data_seg()\n #pragma data_seg(\".CRT$XIY\")\n PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };\n #pragma data_seg()\n #endif\n\n #if defined(__cplusplus)\n }\n #endif\n\n // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.\n void _mi_prim_thread_init_auto_done(void) { }\n void _mi_prim_thread_done_auto_done(void) { }\n void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {\n MI_UNUSED(theap);\n }\n\n#elif defined(MI_WIN_INIT_USE_FLS) // deprecated: statically linked, use fiber api\n\n #if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`)\n // MSVC: use data section magic for static libraries\n // See \n #define MI_PRIM_HAS_PROCESS_ATTACH 1\n\n static int mi_process_attach(void) {\n mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL);\n atexit(&_mi_auto_process_done);\n return 0;\n }\n typedef int(*mi_crt_callback_t)(void);\n #if defined(_WIN64)\n #pragma comment(linker, \"/INCLUDE:_mi_tls_callback\")\n #pragma section(\".CRT$XIU\", long, read)\n #else\n #pragma comment(linker, \"/INCLUDE:__mi_tls_callback\")\n #endif\n #pragma data_seg(\".CRT$XIU\")\n mi_decl_externc mi_crt_callback_t _mi_tls_callback[] = { &mi_process_attach };\n #pragma data_seg()\n #endif\n\n // use the fiber api for calling `_mi_thread_done`.\n #include \n #if (_WIN32_WINNT < 0x600) // before Windows Vista\n WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback );\n WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex );\n WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData );\n WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex);\n #endif\n\n static DWORD mi_fls_key = (DWORD)(-1);\n\n static void NTAPI mi_fls_done(PVOID value) {\n mi_theap_t* theap = (mi_theap_t*)value;\n if (theap != NULL) {\n _mi_thread_done(theap);\n FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main theap, issue #672\n }\n }\n\n void _mi_prim_thread_init_auto_done(void) {\n mi_fls_key = FlsAlloc(&mi_fls_done);\n }\n\n void _mi_prim_thread_done_auto_done(void) {\n // call thread-done on all threads (except the main thread) to prevent\n // dangling callback pointer if statically linked with a DLL; Issue #208\n FlsFree(mi_fls_key);\n }\n\n void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {\n mi_assert_internal(mi_fls_key != (DWORD)(-1));\n FlsSetValue(mi_fls_key, theap);\n }\n#else\n#error \"define windows process and thread auto initialization\"\n#endif\n\n// ----------------------------------------------------\n// Communicate with the redirection module on Windows\n// ----------------------------------------------------\n#if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT)\n #define MI_PRIM_HAS_ALLOCATOR_INIT 1\n\n static bool mi_redirected = false; // true if malloc redirects to mi_malloc\n\n bool _mi_is_redirected(void) {\n return mi_redirected;\n }\n\n #ifdef __cplusplus\n extern \"C\" {\n #endif\n mi_decl_export void _mi_redirect_entry(DWORD reason) {\n // called on redirection; careful as this may be called before DllMain\n if (reason == DLL_PROCESS_ATTACH) {\n mi_redirected = true;\n }\n else if (reason == DLL_PROCESS_DETACH) {\n mi_redirected = false;\n }\n else if (reason == DLL_THREAD_DETACH) {\n // mi_debug_out(\"redirect thread detach\\n\");\n _mi_thread_done(NULL);\n }\n }\n __declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message);\n __declspec(dllimport) void mi_cdecl mi_allocator_done(void);\n #ifdef __cplusplus\n }\n #endif\n bool _mi_allocator_init(const char** message) {\n return mi_allocator_init(message);\n }\n void _mi_allocator_done(void) {\n mi_allocator_done();\n }\n#endif\n\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/random.c", "sha256": "a96a1acc75fd6595811b28420cf0495b87fac98c4dbc73aa0d742b6824f607c8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2021, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\" // _mi_prim_random_buf\n\n/* ----------------------------------------------------------------------------\nWe use our own PRNG to keep predictable performance of random number generation\nand to avoid implementations that use a lock. We only use the OS provided\nrandom source to initialize the initial seeds. Since we do not need ultimate\nperformance but we do rely on the security (for secret cookies in secure mode)\nwe use a cryptographically secure generator (chacha20).\n-----------------------------------------------------------------------------*/\n\n#define MI_CHACHA_ROUNDS (20) // perhaps use 12 for better performance?\n\n\n/* ----------------------------------------------------------------------------\nChacha20 implementation as the original algorithm with a 64-bit nonce\nand counter: https://en.wikipedia.org/wiki/Salsa20\nThe input matrix has sixteen 32-bit values:\nPosition 0 to 3: constant key\nPosition 4 to 11: the key\nPosition 12 to 13: the counter.\nPosition 14 to 15: the nonce.\n\nThe implementation uses regular C code which compiles very well on modern compilers.\n(gcc x64 has no register spills, and clang 6+ uses SSE instructions)\n-----------------------------------------------------------------------------*/\n\nstatic inline void qround(uint32_t x[16], size_t a, size_t b, size_t c, size_t d) {\n x[a] += x[b]; x[d] = mi_rotl32(x[d] ^ x[a], 16);\n x[c] += x[d]; x[b] = mi_rotl32(x[b] ^ x[c], 12);\n x[a] += x[b]; x[d] = mi_rotl32(x[d] ^ x[a], 8);\n x[c] += x[d]; x[b] = mi_rotl32(x[b] ^ x[c], 7);\n}\n\nstatic void chacha_block(mi_random_ctx_t* ctx)\n{\n // scramble into `x`\n uint32_t x[16];\n for (size_t i = 0; i < 16; i++) {\n x[i] = ctx->input[i];\n }\n for (size_t i = 0; i < MI_CHACHA_ROUNDS; i += 2) {\n qround(x, 0, 4, 8, 12);\n qround(x, 1, 5, 9, 13);\n qround(x, 2, 6, 10, 14);\n qround(x, 3, 7, 11, 15);\n qround(x, 0, 5, 10, 15);\n qround(x, 1, 6, 11, 12);\n qround(x, 2, 7, 8, 13);\n qround(x, 3, 4, 9, 14);\n }\n\n // add scrambled data to the initial state\n for (size_t i = 0; i < 16; i++) {\n ctx->output[i] = x[i] + ctx->input[i];\n }\n ctx->output_available = 16;\n\n // increment the counter for the next round\n ctx->input[12] += 1;\n if (ctx->input[12] == 0) {\n ctx->input[13] += 1;\n if (ctx->input[13] == 0) { // and keep increasing into the nonce\n ctx->input[14] += 1;\n }\n }\n}\n\nstatic uint32_t chacha_next32(mi_random_ctx_t* ctx) {\n if (ctx->output_available <= 0) {\n chacha_block(ctx);\n ctx->output_available = 16; // (assign again to suppress static analysis warning)\n }\n const uint32_t x = ctx->output[16 - ctx->output_available];\n ctx->output[16 - ctx->output_available] = 0; // reset once the data is handed out\n ctx->output_available--;\n return x;\n}\n\nstatic inline uint32_t read32(const uint8_t* p, size_t idx32) {\n const size_t i = 4*idx32;\n return ((uint32_t)p[i+0] | (uint32_t)p[i+1] << 8 | (uint32_t)p[i+2] << 16 | (uint32_t)p[i+3] << 24);\n}\n\nstatic void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t nonce)\n{\n // since we only use chacha for randomness (and not encryption) we\n // do not _need_ to read 32-bit values as little endian but we do anyways\n // just for being compatible :-)\n ctx->output_available = 0;\n _mi_memzero(ctx->output,sizeof(ctx->output));\n for (size_t i = 0; i < 4; i++) {\n const uint8_t* sigma = (uint8_t*)\"expand 32-byte k\";\n ctx->input[i] = read32(sigma,i);\n }\n for (size_t i = 0; i < 8; i++) {\n ctx->input[i + 4] = read32(key,i);\n }\n ctx->input[12] = 0;\n ctx->input[13] = 0;\n ctx->input[14] = (uint32_t)nonce;\n ctx->input[15] = (uint32_t)(nonce >> 32);\n}\n\nstatic void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) {\n _mi_memzero(ctx_new, sizeof(*ctx_new));\n ctx_new->weak = ctx->weak;\n _mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input));\n ctx_new->input[12] = 0;\n ctx_new->input[13] = 0;\n ctx_new->input[14] = (uint32_t)nonce;\n ctx_new->input[15] = (uint32_t)(nonce >> 32);\n mi_assert_internal(ctx->input[14] != ctx_new->input[14] || ctx->input[15] != ctx_new->input[15]); // do not reuse nonces!\n chacha_block(ctx_new);\n}\n\n\n/* ----------------------------------------------------------------------------\nRandom interface\n-----------------------------------------------------------------------------*/\n\n#if MI_DEBUG>1\nstatic bool mi_random_is_initialized(mi_random_ctx_t* ctx) {\n return (ctx != NULL && ctx->input[0] != 0);\n}\n#endif\n\nvoid _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) {\n mi_assert_internal(mi_random_is_initialized(ctx));\n mi_assert_internal(ctx != ctx_new);\n chacha_split(ctx, (uintptr_t)ctx_new /*nonce*/, ctx_new);\n}\n\nuintptr_t _mi_random_next(mi_random_ctx_t* ctx) {\n mi_assert_internal(mi_random_is_initialized(ctx));\n uintptr_t r;\n do {\n #if MI_INTPTR_SIZE <= 4\n r = chacha_next32(ctx);\n #elif MI_INTPTR_SIZE == 8\n r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx));\n #else\n # error \"define mi_random_next for this platform\"\n #endif\n } while (r==0);\n return r;\n}\n\n\n/* ----------------------------------------------------------------------------\nTo initialize a fresh random context.\nIf we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR.\n-----------------------------------------------------------------------------*/\n\nuintptr_t _mi_os_random_weak(uintptr_t extra_seed) {\n uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random\n x ^= _mi_prim_clock_now();\n // and do a few randomization steps\n uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1;\n for (uintptr_t i = 0; i < max || x==0; i++, x++) {\n x = _mi_random_shuffle(x);\n }\n mi_assert_internal(x != 0);\n return x;\n}\n\nstatic void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) {\n uint8_t key[32];\n if (use_weak || !_mi_prim_random_buf(key, sizeof(key))) {\n // if we fail to get random data from the OS, we fall back to a\n // weak random source based on the current time\n #if !defined(__wasi__)\n if (!use_weak) { _mi_warning_message(\"unable to use secure randomness\\n\"); }\n #endif\n uintptr_t x = _mi_os_random_weak(0);\n for (size_t i = 0; i < 32; i+=4, x++) { \n x = _mi_random_shuffle(x);\n key[i] = (uint8_t)(x);\n key[i+1] = (uint8_t)(x>>8);\n key[i+2] = (uint8_t)(x>>16);\n key[i+3] = (uint8_t)(x>>24); \n }\n ctx->weak = true;\n }\n else {\n ctx->weak = false;\n }\n chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ );\n}\n\nvoid _mi_random_init(mi_random_ctx_t* ctx) {\n mi_random_init_ex(ctx, false);\n}\n\nvoid _mi_random_init_weak(mi_random_ctx_t * ctx) {\n mi_random_init_ex(ctx, true);\n}\n\nvoid _mi_random_reinit_if_weak(mi_random_ctx_t * ctx) {\n if (ctx->weak) {\n _mi_random_init(ctx);\n }\n}\n\n/* --------------------------------------------------------\ntest vectors from \n----------------------------------------------------------- */\n/*\nstatic bool array_equals(uint32_t* x, uint32_t* y, size_t n) {\n for (size_t i = 0; i < n; i++) {\n if (x[i] != y[i]) return false;\n }\n return true;\n}\nstatic void chacha_test(void)\n{\n uint32_t x[4] = { 0x11111111, 0x01020304, 0x9b8d6f43, 0x01234567 };\n uint32_t x_out[4] = { 0xea2a92f4, 0xcb1cf8ce, 0x4581472e, 0x5881c4bb };\n qround(x, 0, 1, 2, 3);\n mi_assert_internal(array_equals(x, x_out, 4));\n\n uint32_t y[16] = {\n 0x879531e0, 0xc5ecf37d, 0x516461b1, 0xc9a62f8a,\n 0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0x2a5f714c,\n 0x53372767, 0xb00a5631, 0x974c541a, 0x359e9963,\n 0x5c971061, 0x3d631689, 0x2098d9d6, 0x91dbd320 };\n uint32_t y_out[16] = {\n 0x879531e0, 0xc5ecf37d, 0xbdb886dc, 0xc9a62f8a,\n 0x44c20ef3, 0x3390af7f, 0xd9fc690b, 0xcfacafd2,\n 0xe46bea80, 0xb00a5631, 0x974c541a, 0x359e9963,\n 0x5c971061, 0xccc07c79, 0x2098d9d6, 0x91dbd320 };\n qround(y, 2, 7, 8, 13);\n mi_assert_internal(array_equals(y, y_out, 16));\n\n mi_random_ctx_t r = {\n { 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574,\n 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c,\n 0x13121110, 0x17161514, 0x1b1a1918, 0x1f1e1d1c,\n 0x00000001, 0x09000000, 0x4a000000, 0x00000000 },\n {0},\n 0\n };\n uint32_t r_out[16] = {\n 0xe4e7f110, 0x15593bd1, 0x1fdd0f50, 0xc47120a3,\n 0xc7f4d1c7, 0x0368c033, 0x9aaa2204, 0x4e6cd4c3,\n 0x466482d2, 0x09aa9f07, 0x05d7c214, 0xa2028bd9,\n 0xd19c12b5, 0xb94e16de, 0xe883d0cb, 0x4e3c50a2 };\n chacha_block(&r);\n mi_assert_internal(array_equals(r.output, r_out, 16));\n}\n*/\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/static.c", "sha256": "4059be2a6676693be3433f5aedc711f56301772e65f3573654c236d6d1cfd074", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2020, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#ifndef _DEFAULT_SOURCE\n#define _DEFAULT_SOURCE\n#endif\n#if defined(__sun)\n// same remarks as os.c for the static's context.\n#undef _XOPEN_SOURCE\n#undef _POSIX_C_SOURCE\n#endif\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n\n// For a static override we create a single object file\n// containing the whole library. If it is linked first\n// it will override all the standard library allocation\n// functions (on Unix's).\n#include \"alloc.c\" // includes alloc-override.c and free.c\n#include \"alloc-aligned.c\"\n#include \"alloc-posix.c\"\n#include \"arena.c\"\n#include \"arena-meta.c\"\n#include \"bitmap.c\"\n#include \"heap.c\"\n#include \"init.c\"\n#include \"libc.c\"\n#include \"options.c\"\n#include \"os.c\"\n#include \"page.c\" // includes page-queue.c\n#include \"page-map.c\"\n#include \"random.c\"\n#include \"stats.c\"\n#include \"theap.c\"\n#include \"threadlocal.c\"\n#include \"prim/prim.c\"\n#if MI_OSX_ZONE\n#include \"prim/osx/alloc-override-zone.c\"\n#endif\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/stats.c", "sha256": "16825bbc252de422982b5728df7eda181d952824f5e3ce958c7fc85feae64b8e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n#include \"mimalloc.h\"\n#include \"mimalloc-stats.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/atomic.h\"\n#include \"mimalloc/prim.h\"\n\n#include // memset\n\n#if defined(_MSC_VER) && (_MSC_VER < 1920)\n#pragma warning(disable:4204) // non-constant aggregate initializer\n#endif\n\n/* -----------------------------------------------------------\n Statistics operations\n----------------------------------------------------------- */\n\nstatic void mi_stat_update_mt(mi_stat_count_t* stat, int64_t amount) {\n if (amount == 0) return;\n // add atomically\n int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount);\n mi_atomic_maxi64_relaxed(&stat->peak, current + amount);\n if (amount > 0) {\n mi_atomic_addi64_relaxed(&stat->total, amount);\n }\n}\n\nstatic void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {\n if (amount == 0) return;\n // add thread local\n stat->current += amount;\n if (stat->current > stat->peak) { stat->peak = stat->current; }\n if (amount > 0) { stat->total += amount; }\n}\n\n\nvoid __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount) {\n mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount);\n}\n\nvoid __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {\n stat->total += amount;\n}\n\nvoid __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) {\n mi_stat_update_mt(stat, (int64_t)amount);\n}\nvoid __mi_stat_increase(mi_stat_count_t* stat, size_t amount) {\n mi_stat_update(stat, (int64_t)amount);\n}\n\nvoid __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount) {\n mi_stat_update_mt(stat, -((int64_t)amount));\n}\nvoid __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {\n mi_stat_update(stat, -((int64_t)amount));\n}\n\n\n// Adjust stats to compensate; for example before committing a range,\n// first adjust downwards with parts that were already committed so\n// we avoid double counting.\nstatic void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) {\n if (amount == 0) return;\n // adjust atomically\n mi_atomic_addi64_relaxed(&stat->current, amount);\n mi_atomic_addi64_relaxed(&stat->total, amount);\n}\n\nstatic void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) {\n if (amount == 0) return;\n stat->current += amount;\n stat->total += amount;\n}\n\nvoid __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) {\n mi_stat_adjust_mt(stat, (int64_t)amount);\n}\nvoid __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) {\n mi_stat_adjust(stat, (int64_t)amount);\n}\nvoid __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount) {\n mi_stat_adjust_mt(stat, -((int64_t)amount));\n}\nvoid __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) {\n mi_stat_adjust(stat, -((int64_t)amount));\n}\n\n\n// must be thread safe as it is called from stats_merge\nstatic void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) {\n if (stat==src) return;\n mi_atomic_void_addi64_relaxed(&stat->total, &src->total);\n const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src->current);\n\n // Global current plus thread peak approximates new global peak\n // note: peak scores do really not work across threads.\n // we used to just add them together but that often overestimates in practice.\n // similarly, max does not seem to work well. The current approach\n // by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112\n // for a longer description.\n mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src->peak);\n}\n\nstatic void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) {\n if (stat==src) return;\n mi_atomic_void_addi64_relaxed(&stat->total, &src->total);\n}\n\n#define MI_STAT_COUNT(stat) mi_stat_count_add_mt(&stats->stat, &src->stat);\n#define MI_STAT_COUNTER(stat) mi_stat_counter_add_mt(&stats->stat, &src->stat);\n\n// must be thread safe as it is called from stats_merge\nstatic void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {\n if (stats==NULL || src==NULL || stats==src) return;\n\n // copy all fields\n MI_STAT_FIELDS()\n\n #if MI_STAT>1\n for (size_t i = 0; i <= MI_BIN_HUGE; i++) {\n mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]);\n }\n #endif\n for (size_t i = 0; i <= MI_BIN_HUGE; i++) {\n mi_stat_count_add_mt(&stats->page_bins[i], &src->page_bins[i]);\n }\n}\n\n#undef MI_STAT_COUNT\n#undef MI_STAT_COUNTER\n\n/* -----------------------------------------------------------\n Display statistics\n----------------------------------------------------------- */\n\n// unit > 0 : size in binary bytes\n// unit == 0: count as decimal\n// unit < 0 : count in binary\nstatic void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) {\n char buf[32]; _mi_memzero_var(buf);\n int len = 32;\n const char* suffix = (unit <= 0 ? \" \" : \"B\");\n const int64_t base = (unit == 0 ? 1000 : 1024);\n if (unit>0) n *= unit;\n\n const int64_t pos = (n < 0 ? -n : n);\n if (pos < base) {\n if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column\n _mi_snprintf(buf, len, \"%lld %-3s\", (long long)n, (n==0 ? \"\" : suffix));\n }\n }\n else {\n int64_t divider = base;\n const char* magnitude = \"K\";\n if (pos >= divider*base) { divider *= base; magnitude = \"M\"; }\n if (pos >= divider*base) { divider *= base; magnitude = \"G\"; }\n const int64_t tens = (n / (divider/10));\n const long whole = (long)(tens/10);\n const long frac1 = (long)(tens%10);\n char unitdesc[8];\n _mi_snprintf(unitdesc, 8, \"%s%s%s\", magnitude, (base==1024 ? \"i\" : \"\"), suffix);\n _mi_snprintf(buf, len, \"%ld.%ld %-3s\", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc);\n }\n _mi_fprintf(out, arg, (fmt==NULL ? \"%12s\" : fmt), buf);\n}\n\n\nstatic void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {\n mi_printf_amount(n,unit,out,arg,NULL);\n}\n\nstatic void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {\n if (unit==1) _mi_fprintf(out, arg, \"%12s\",\" \");\n else mi_print_amount(n,0,out,arg);\n}\n\nstatic void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) {\n _mi_fprintf(out, arg,\" %-10s:\", msg);\n if (unit != 0) {\n if (unit > 0) {\n mi_print_amount(stat->peak, unit, out, arg);\n mi_print_amount(stat->total, unit, out, arg);\n // mi_print_amount(stat->freed, unit, out, arg);\n mi_print_amount(stat->current, unit, out, arg);\n mi_print_amount(unit, 1, out, arg);\n mi_print_count(stat->total, unit, out, arg);\n }\n else {\n mi_print_amount(stat->peak, -1, out, arg);\n mi_print_amount(stat->total, -1, out, arg);\n // mi_print_amount(stat->freed, -1, out, arg);\n mi_print_amount(stat->current, -1, out, arg);\n if (unit == -1) {\n _mi_fprintf(out, arg, \"%24s\", \"\");\n }\n else {\n mi_print_amount(-unit, 1, out, arg);\n mi_print_count((stat->total / -unit), 0, out, arg);\n }\n }\n if (stat->current != 0) {\n _mi_fprintf(out, arg, \" \");\n _mi_fprintf(out, arg, (notok == NULL ? \"not all freed\" : notok));\n _mi_fprintf(out, arg, \"\\n\");\n }\n else {\n _mi_fprintf(out, arg, \" ok\\n\");\n }\n }\n else {\n mi_print_amount(stat->peak, 0, out, arg);\n mi_print_amount(stat->total, 0, out, arg);\n mi_print_amount(stat->current, 0, out, arg);\n _mi_fprintf(out, arg, \"\\n\");\n }\n}\n\nstatic void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {\n mi_stat_print_ex(stat, msg, unit, out, arg, NULL);\n}\n\n#if MI_STAT>1\nstatic void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {\n _mi_fprintf(out, arg, \" %-10s:\", msg);\n _mi_fprintf(out, arg, \"%12s\", \" \"); // no peak\n mi_print_amount(stat->total, unit, out, arg);\n _mi_fprintf(out, arg, \"\\n\");\n}\n#endif\n\nstatic void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {\n _mi_fprintf(out, arg, \" %-10s:\", msg);\n mi_print_amount(stat->total, 0, out, arg);\n _mi_fprintf(out, arg, \"\\n\");\n}\n\nstatic void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {\n _mi_fprintf(out, arg, \" %-10s:\", msg);\n mi_print_amount(stat->total, 1, out, arg);\n _mi_fprintf(out, arg, \"\\n\");\n}\n\nstatic void mi_stat_average_print(int64_t count, int64_t total, const char* msg, mi_output_fun* out, void* arg) {\n const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count));\n const int64_t avg_whole = avg_tens/10;\n const int64_t avg_frac1 = avg_tens%10;\n _mi_fprintf(out, arg, \" %-10s: %5lld.%lld avg\\n\", msg, avg_whole, avg_frac1);\n}\n\n\nstatic void mi_print_header(const char* name,mi_output_fun* out, void* arg ) {\n _mi_fprintf(out, arg, \" %-11s %11s %11s %11s %11s %11s\\n\",\n name, \"peak \", \"total \", \"current \", \"block \", \"total# \");\n}\n\n#if MI_STAT>1\nstatic bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) {\n bool found = false;\n char buf[64];\n for (size_t i = 0; i <= max; i++) {\n if (bins[i].total > 0) {\n found = true;\n const size_t unit = _mi_bin_size((uint8_t)i);\n const char* pagekind = (unit <= MI_SMALL_MAX_OBJ_SIZE ? \"S\" :\n (unit <= MI_MEDIUM_MAX_OBJ_SIZE ? \"M\" :\n (unit <= MI_LARGE_MAX_OBJ_SIZE ? \"L\" : \"H\")));\n _mi_snprintf(buf, 64, \"bin%2s %3lu\", pagekind, (long)i);\n mi_stat_print(&bins[i], buf, (int64_t)unit, out, arg);\n }\n }\n if (found) {\n _mi_fprintf(out, arg, \"\\n\");\n }\n return found;\n}\n#endif\n\n\n\n//------------------------------------------------------------\n// Use an output wrapper for line-buffered output\n// (which is nice when using loggers etc.)\n//------------------------------------------------------------\ntypedef struct buffered_s {\n mi_output_fun* out; // original output function\n void* arg; // and state\n char* buf; // local buffer of at least size `count+1`\n size_t used; // currently used chars `used <= count`\n size_t count; // total chars available for output\n} buffered_t;\n\nstatic void mi_buffered_flush(buffered_t* buf) {\n buf->buf[buf->used] = 0;\n _mi_fputs(buf->out, buf->arg, NULL, buf->buf);\n buf->used = 0;\n}\n\nstatic void mi_cdecl mi_buffered_out(const char* msg, void* arg) {\n buffered_t* buf = (buffered_t*)arg;\n if (msg==NULL || buf==NULL) return;\n for (const char* src = msg; *src != 0; src++) {\n char c = *src;\n if (buf->used >= buf->count) mi_buffered_flush(buf);\n mi_assert_internal(buf->used < buf->count);\n buf->buf[buf->used++] = c;\n if (c == '\\n') mi_buffered_flush(buf);\n }\n}\n\n//------------------------------------------------------------\n// Print statistics\n//------------------------------------------------------------\n\nmi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept\n{\n size_t elapsed;\n size_t user_time;\n size_t sys_time;\n size_t current_rss;\n size_t peak_rss;\n size_t current_commit;\n size_t peak_commit;\n size_t page_faults;\n mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults);\n _mi_fprintf(out, arg, \" %-10s: %5zu.%03zu s\\n\", \"elapsed\", elapsed/1000, elapsed%1000);\n _mi_fprintf(out, arg, \" %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: \", \"process\",\n user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults);\n mi_printf_amount((int64_t)peak_rss, 1, out, arg, \"%s\");\n if (peak_commit > 0) {\n _mi_fprintf(out, arg, \", peak commit: \");\n mi_printf_amount((int64_t)peak_commit, 1, out, arg, \"%s\");\n }\n _mi_fprintf(out, arg, \"\\n\");\n}\n\nvoid _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept {\n // wrap the output function to be line buffered\n char buf[256]; _mi_memzero_var(buf);\n buffered_t buffer = { out0, arg0, NULL, 0, 255 };\n buffer.buf = buf;\n mi_output_fun* out = &mi_buffered_out;\n void* arg = &buffer;\n\n // and print using that\n _mi_fprintf(out, arg, \"%s %zu\\n\", name, id);\n\n if (stats->malloc_normal.total + stats->malloc_huge.total != 0) {\n #if MI_STAT>1\n mi_print_header(\"blocks\", out, arg);\n mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg);\n #endif\n #if MI_STAT\n mi_stat_print(&stats->malloc_normal, \"binned\", (stats->malloc_normal_count.total == 0 ? -1 : 1), out, arg);\n mi_stat_print(&stats->malloc_huge, \"huge\", (stats->malloc_huge_count.total == 0 ? -1 : 1), out, arg);\n mi_stat_count_t total = { 0,0,0 };\n mi_stat_count_add_mt(&total, &stats->malloc_normal);\n mi_stat_count_add_mt(&total, &stats->malloc_huge);\n mi_stat_print_ex(&total, \"total\", 1, out, arg, \"\");\n #if MI_STAT>1\n mi_stat_total_print(&stats->malloc_requested, \"malloc req\", 1, out, arg);\n #endif\n _mi_fprintf(out, arg, \"\\n\");\n #endif\n }\n\n if (stats->pages.total != 0) {\n mi_print_header(\"pages\", out, arg);\n mi_stat_print_ex(&stats->page_committed, \"touched\", 1, out, arg, \"\");\n // mi_stat_print(&stats->segments, \"segments\", -1, out, arg);\n // mi_stat_print(&stats->segments_abandoned, \"-abandoned\", -1, out, arg);\n // mi_stat_print(&stats->segments_cache, \"-cached\", -1, out, arg);\n mi_stat_print(&stats->pages, \"pages\", 0, out, arg);\n mi_stat_print(&stats->pages_abandoned, \"abandoned\", 0, out, arg);\n mi_stat_counter_print(&stats->pages_reclaim_on_alloc, \"reclaima\", out, arg);\n mi_stat_counter_print(&stats->pages_reclaim_on_free, \"reclaimf\", out, arg);\n mi_stat_counter_print(&stats->pages_reabandon_full, \"reabandon\", out, arg);\n mi_stat_counter_print(&stats->pages_unabandon_busy_wait, \"waits\", out, arg);\n mi_stat_counter_print(&stats->pages_extended, \"extended\", out, arg);\n mi_stat_counter_print(&stats->pages_retire, \"retire\", out, arg);\n mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, \"searches\", out, arg);\n _mi_fprintf(out, arg, \"\\n\");\n }\n\n if (stats->arena_count.total > 0) {\n mi_print_header(\"arenas\", out, arg);\n mi_stat_print_ex(&stats->reserved, \"reserved\", 1, out, arg, \"\");\n mi_stat_print_ex(&stats->committed, \"committed\", 1, out, arg, \"\");\n mi_stat_counter_print_size(&stats->reset, \"reset\", out, arg);\n mi_stat_counter_print_size(&stats->purged, \"purged\", out, arg);\n\n mi_stat_counter_print(&stats->arena_count, \"arenas\", out, arg);\n mi_stat_counter_print(&stats->arena_rollback_count, \"rollback\", out, arg);\n mi_stat_counter_print(&stats->mmap_calls, \"mmaps\", out, arg);\n mi_stat_counter_print(&stats->commit_calls, \"commits\", out, arg);\n mi_stat_counter_print(&stats->reset_calls, \"resets\", out, arg);\n mi_stat_counter_print(&stats->purge_calls, \"purges\", out, arg);\n mi_stat_counter_print(&stats->malloc_guarded_count, \"guarded\", out, arg);\n mi_stat_print_ex(&stats->theaps, \"theaps\", 0, out, arg, \"\");\n mi_stat_print_ex(&stats->heaps, \"heaps\", 0, out, arg, \"\");\n mi_stat_counter_print(&stats->heaps_delete_wait, \"heap waits\", out, arg);\n _mi_fprintf(out, arg, \"\\n\");\n\n mi_print_header(\"process\", out, arg);\n mi_stat_print_ex(&stats->threads, \"threads\", 0, out, arg, \"\");\n _mi_fprintf(out, arg, \" %-10s: %5i\\n\", \"numa nodes\", _mi_os_numa_node_count());\n mi_process_info_print_out(out, arg);\n }\n _mi_fprintf(out, arg, \"\\n\");\n}\n\n\nstatic mi_msecs_t mi_process_start; // = 0\n\n// called on process init\nvoid _mi_stats_init(void) {\n if (mi_process_start == 0) { mi_process_start = _mi_clock_start(); };\n}\n\nstatic void mi_stats_add_into(mi_stats_t* to, const mi_stats_t* from) {\n mi_assert_internal(to != NULL && from != NULL);\n if (to == from) return;\n mi_stats_add(to, from);\n}\n\nvoid _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from) {\n mi_assert_internal(to != NULL && from != NULL);\n if (to == from) return;\n mi_stats_add(to, from);\n _mi_memzero(from, sizeof(mi_stats_t));\n}\n\nstatic const mi_stats_t* mi_stats_merge_theap_to_heap(mi_theap_t* theap) mi_attr_noexcept {\n mi_stats_t* stats = &theap->stats;\n mi_stats_t* heap_stats = &_mi_theap_heap(theap)->stats;\n _mi_stats_merge_into( heap_stats, stats );\n return heap_stats;\n}\n\nstatic const mi_stats_t* mi_heap_get_stats(mi_heap_t* heap) {\n if (heap==NULL) { heap = mi_heap_main(); }\n mi_theap_t* theap = _mi_heap_theap_peek(heap);\n if (theap==NULL) return &heap->stats;\n else return mi_stats_merge_theap_to_heap(theap);\n}\n\n// deprecated\nvoid mi_stats_reset(void) mi_attr_noexcept {\n if (!mi_theap_is_initialized(_mi_theap_default())) return;\n mi_heap_get_stats(mi_heap_main());\n mi_heap_stats_merge_to_subproc(mi_heap_main());\n}\n\n\nvoid mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept {\n if (heap==NULL) { heap = mi_heap_main(); }\n _mi_stats_print(\"heap\", heap->heap_seq, mi_heap_get_stats(heap), out, arg);\n}\n\ntypedef struct mi_heap_print_visit_info_s {\n mi_output_fun* out;\n void* out_arg;\n} mi_heap_print_visit_info_t;\n\nstatic bool mi_cdecl mi_heap_print_visitor(mi_heap_t* heap, void* arg) {\n mi_heap_print_visit_info_t* vinfo = (mi_heap_print_visit_info_t*)(arg);\n mi_heap_stats_print_out(heap, vinfo->out, vinfo->out_arg);\n return true;\n}\n\n\n// show each heap and then the subproc\nvoid mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept {\n mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);\n if (subproc==NULL) return;\n mi_heap_print_visit_info_t vinfo = { out, arg };\n mi_subproc_visit_heaps(subproc_id, &mi_heap_print_visitor, &vinfo);\n _mi_stats_print(\"subproc\", subproc->subproc_seq, &subproc->stats, out, arg);\n}\n\n\n// aggregate all stats from the heaps and subproc and print those\nvoid mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg) mi_attr_noexcept {\n mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);\n if (subproc==NULL) return;\n mi_stats_t_decl(stats);\n if (mi_subproc_stats_get(subproc_id, &stats)) {\n _mi_stats_print(\"subproc\", subproc->subproc_seq, &stats, out, arg);\n }\n}\n\nvoid mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {\n mi_subproc_stats_print_out(mi_subproc_current(),out, arg);\n}\n\n// deprecated\nvoid mi_stats_print(void* out) mi_attr_noexcept {\n // for compatibility there is an `out` parameter (which can be `stdout` or `stderr`)\n mi_stats_print_out((mi_output_fun*)out, NULL);\n}\n\n// deprecated\nvoid mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {\n mi_theap_t* theap = _mi_theap_default();\n if (theap==NULL || !mi_theap_is_initialized(theap)) return;\n _mi_stats_print(\"heap\", _mi_theap_heap(theap)->heap_seq, &theap->stats, out, arg);\n mi_stats_merge_theap_to_heap(_mi_theap_default());\n}\n\n\n// ----------------------------------------------------------------\n// Basic timer for convenience; use milli-seconds to avoid doubles\n// ----------------------------------------------------------------\n\nstatic mi_msecs_t mi_clock_diff;\n\nmi_msecs_t _mi_clock_now(void) {\n return _mi_prim_clock_now();\n}\n\nmi_msecs_t _mi_clock_start(void) {\n if (mi_clock_diff == 0.0) {\n mi_msecs_t t0 = _mi_clock_now();\n mi_clock_diff = _mi_clock_now() - t0;\n }\n return _mi_clock_now();\n}\n\nmi_msecs_t _mi_clock_end(mi_msecs_t start) {\n mi_msecs_t end = _mi_clock_now();\n return (end - start - mi_clock_diff);\n}\n\n\n// --------------------------------------------------------\n// Basic process statistics\n// --------------------------------------------------------\n\nmi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept\n{\n mi_process_info_t pinfo;\n _mi_memzero_var(pinfo);\n pinfo.elapsed = _mi_clock_end(mi_process_start);\n { const mi_subproc_t* subproc = _mi_subproc_main();\n if (subproc!=NULL) {\n pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)));\n pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)));\n } \n }\n pinfo.current_rss = pinfo.current_commit;\n pinfo.peak_rss = pinfo.peak_commit;\n pinfo.utime = 0;\n pinfo.stime = 0;\n pinfo.page_faults = 0;\n\n _mi_prim_process_info(&pinfo);\n\n if (elapsed_msecs!=NULL) *elapsed_msecs = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX));\n if (user_msecs!=NULL) *user_msecs = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX));\n if (system_msecs!=NULL) *system_msecs = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX));\n if (current_rss!=NULL) *current_rss = pinfo.current_rss;\n if (peak_rss!=NULL) *peak_rss = pinfo.peak_rss;\n if (current_commit!=NULL) *current_commit = pinfo.current_commit;\n if (peak_commit!=NULL) *peak_commit = pinfo.peak_commit;\n if (page_faults!=NULL) *page_faults = pinfo.page_faults;\n}\n\nmi_decl_export void mi_process_info_print(void) mi_attr_noexcept {\n mi_process_info_print_out(NULL, NULL);\n}\n\n\n// --------------------------------------------------------\n// Return statistics\n// --------------------------------------------------------\n\nsize_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept {\n if (bin > MI_BIN_HUGE) return 0;\n return _mi_bin_size(bin);\n}\n\nstatic bool mi_stats_copy(mi_stats_t* stats_to, const mi_stats_t* stats_from) mi_attr_noexcept {\n if (stats_to == NULL || stats_to->size != sizeof(mi_stats_t) || stats_to->version != MI_STAT_VERSION) return false;\n if (stats_from == NULL || stats_from->size != stats_to->size) return false;\n _mi_memcpy(stats_to, stats_from, stats_to->size);\n return true;\n}\n\nbool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept {\n const mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);\n if (subproc==NULL) return false;\n return mi_stats_copy(stats,&subproc->stats);\n}\n\nbool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept {\n return mi_stats_copy(stats, mi_heap_get_stats(heap));\n}\n\n\nstatic bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) {\n mi_stats_t* stats = (mi_stats_t*)arg;\n mi_stats_add_into(stats, mi_heap_get_stats(heap));\n return true;\n}\n\nbool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept {\n if (stats==NULL) return false;\n mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);\n if (subproc == NULL) return false;\n if (!mi_stats_copy(stats, &subproc->stats)) return false;\n mi_subproc_visit_heaps(subproc_id, &mi_heap_aggregate_visitor, stats);\n return true;\n}\n\nbool mi_stats_get(mi_stats_t* stats) mi_attr_noexcept {\n return mi_subproc_stats_get(mi_subproc_current(), stats);\n}\n\n\n// --------------------------------------------------------\n// Statics in json format\n// --------------------------------------------------------\n\ntypedef struct mi_json_buf_s {\n char* buf;\n size_t size;\n size_t used;\n bool can_realloc;\n} mi_json_buf_t;\n\nstatic bool mi_json_buf_expand(mi_json_buf_t* hbuf) {\n if (hbuf==NULL) return false;\n if (hbuf->buf != NULL && hbuf->size>0) {\n hbuf->buf[hbuf->size-1] = 0;\n }\n if (hbuf->size > SIZE_MAX/2 || !hbuf->can_realloc) return false;\n const size_t newsize = (hbuf->size == 0 ? mi_good_size(12*MI_KiB) : 2*hbuf->size);\n char* const newbuf = (char*)mi_rezalloc(hbuf->buf, newsize);\n if (newbuf == NULL) return false;\n hbuf->buf = newbuf;\n hbuf->size = newsize;\n return true;\n}\n\nstatic void mi_json_buf_print(mi_json_buf_t* hbuf, const char* msg) {\n if (msg==NULL || hbuf==NULL) return;\n if (hbuf->used + 1 >= hbuf->size && !hbuf->can_realloc) return;\n for (const char* src = msg; *src != 0; src++) {\n char c = *src;\n if (hbuf->used + 1 >= hbuf->size) {\n if (!mi_json_buf_expand(hbuf)) return;\n }\n mi_assert_internal(hbuf->used < hbuf->size);\n hbuf->buf[hbuf->used++] = c;\n }\n mi_assert_internal(hbuf->used < hbuf->size);\n hbuf->buf[hbuf->used] = 0;\n}\n\nstatic void mi_json_buf_print_count_bin(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, size_t bin, bool add_comma) {\n const size_t binsize = mi_stats_get_bin_size(bin);\n const size_t pagesize = (binsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_SMALL_PAGE_SIZE :\n (binsize <= MI_MEDIUM_MAX_OBJ_SIZE ? MI_MEDIUM_PAGE_SIZE :\n (binsize <= MI_LARGE_MAX_OBJ_SIZE ? MI_LARGE_PAGE_SIZE : 0)));\n char buf[128];\n _mi_snprintf(buf, 128, \"%s{ \\\"total\\\": %lld, \\\"peak\\\": %lld, \\\"current\\\": %lld, \\\"block_size\\\": %zu, \\\"page_size\\\": %zu }%s\\n\", prefix, stat->total, stat->peak, stat->current, binsize, pagesize, (add_comma ? \",\" : \"\"));\n buf[127] = 0;\n mi_json_buf_print(hbuf, buf);\n}\n\nstatic void mi_json_buf_print_count_cbin(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, mi_chunkbin_t bin, bool add_comma) {\n const char* cbin = \" \";\n switch(bin) {\n case MI_CBIN_SMALL: cbin = \"S\"; break;\n case MI_CBIN_MEDIUM: cbin = \"M\"; break;\n case MI_CBIN_LARGE: cbin = \"L\"; break;\n case MI_CBIN_HUGE: cbin = \"H\"; break;\n case MI_CBIN_OTHER: cbin = \"X\"; break;\n default: cbin = \" \"; break;\n }\n char buf[128];\n _mi_snprintf(buf, 128, \"%s{ \\\"total\\\": %lld, \\\"peak\\\": %lld, \\\"current\\\": %lld, \\\"bin\\\": \\\"%s\\\" }%s\\n\", prefix, stat->total, stat->peak, stat->current, cbin, (add_comma ? \",\" : \"\"));\n buf[127] = 0;\n mi_json_buf_print(hbuf, buf);\n}\n\nstatic void mi_json_buf_print_count(mi_json_buf_t* hbuf, const char* prefix, const mi_stat_count_t* stat, bool add_comma) {\n char buf[128];\n _mi_snprintf(buf, 128, \"%s{ \\\"total\\\": %lld, \\\"peak\\\": %lld, \\\"current\\\": %lld }%s\\n\", prefix, stat->total, stat->peak, stat->current, (add_comma ? \",\" : \"\"));\n buf[127] = 0;\n mi_json_buf_print(hbuf, buf);\n}\n\nstatic void mi_json_buf_print_count_value(mi_json_buf_t* hbuf, const char* name, const mi_stat_count_t* stat) {\n char buf[128];\n _mi_snprintf(buf, 128, \" \\\"%s\\\": \", name);\n buf[127] = 0;\n mi_json_buf_print(hbuf, buf);\n mi_json_buf_print_count(hbuf, \"\", stat, true);\n}\n\nstatic void mi_json_buf_print_value(mi_json_buf_t* hbuf, const char* name, int64_t val) {\n char buf[128];\n _mi_snprintf(buf, 128, \" \\\"%s\\\": %lld,\\n\", name, val);\n buf[127] = 0;\n mi_json_buf_print(hbuf, buf);\n}\n\nstatic void mi_json_buf_print_size(mi_json_buf_t* hbuf, const char* name, size_t val, bool add_comma) {\n char buf[128];\n _mi_snprintf(buf, 128, \" \\\"%s\\\": %zu%s\\n\", name, val, (add_comma ? \",\" : \"\"));\n buf[127] = 0;\n mi_json_buf_print(hbuf, buf);\n}\n\nstatic void mi_json_buf_print_counter_value(mi_json_buf_t* hbuf, const char* name, const mi_stat_counter_t* stat) {\n mi_json_buf_print_value(hbuf, name, stat->total);\n}\n\n#define MI_STAT_COUNT(stat) mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat);\n#define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat);\n\nstatic char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, char* output_buf) mi_attr_noexcept {\n if (stats==NULL || stats->size!=sizeof(mi_stats_t) || stats->version!=MI_STAT_VERSION) return NULL;\n mi_json_buf_t hbuf = { NULL, 0, 0, true };\n if (output_size > 0 && output_buf != NULL) {\n _mi_memzero(output_buf, output_size);\n hbuf.buf = output_buf;\n hbuf.size = output_size;\n hbuf.can_realloc = false;\n }\n else {\n if (!mi_json_buf_expand(&hbuf)) return NULL;\n }\n mi_json_buf_print(&hbuf, \"{\\n\");\n mi_json_buf_print_value(&hbuf, \"stat_version\", MI_STAT_VERSION);\n mi_json_buf_print_value(&hbuf, \"mimalloc_version\", MI_MALLOC_VERSION);\n\n // process info\n mi_json_buf_print(&hbuf, \" \\\"process\\\": {\\n\");\n size_t elapsed;\n size_t user_time;\n size_t sys_time;\n size_t current_rss;\n size_t peak_rss;\n size_t current_commit;\n size_t peak_commit;\n size_t page_faults;\n mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults);\n mi_json_buf_print_size(&hbuf, \"elapsed_msecs\", elapsed, true);\n mi_json_buf_print_size(&hbuf, \"user_msecs\", user_time, true);\n mi_json_buf_print_size(&hbuf, \"system_msecs\", sys_time, true);\n mi_json_buf_print_size(&hbuf, \"page_faults\", page_faults, true);\n mi_json_buf_print_size(&hbuf, \"rss_current\", current_rss, true);\n mi_json_buf_print_size(&hbuf, \"rss_peak\", peak_rss, true);\n mi_json_buf_print_size(&hbuf, \"commit_current\", current_commit, true);\n mi_json_buf_print_size(&hbuf, \"commit_peak\", peak_commit, false);\n mi_json_buf_print(&hbuf, \" },\\n\");\n\n // statistics\n MI_STAT_FIELDS()\n\n // size bins\n mi_json_buf_print(&hbuf, \" \\\"malloc_bins\\\": [\\n\");\n for (size_t i = 0; i <= MI_BIN_HUGE; i++) {\n mi_json_buf_print_count_bin(&hbuf, \" \", &stats->malloc_bins[i], i, i!=MI_BIN_HUGE);\n }\n mi_json_buf_print(&hbuf, \" ],\\n\");\n mi_json_buf_print(&hbuf, \" \\\"page_bins\\\": [\\n\");\n for (size_t i = 0; i <= MI_BIN_HUGE; i++) {\n mi_json_buf_print_count_bin(&hbuf, \" \", &stats->page_bins[i], i, i!=MI_BIN_HUGE);\n }\n mi_json_buf_print(&hbuf, \" ],\\n\");\n mi_json_buf_print(&hbuf, \" \\\"chunk_bins\\\": [\\n\");\n for (size_t i = 0; i < MI_CBIN_COUNT; i++) {\n mi_json_buf_print_count_cbin(&hbuf, \" \", &stats->chunk_bins[i], (mi_chunkbin_t)i, i!=MI_CBIN_COUNT-1);\n }\n mi_json_buf_print(&hbuf, \" ]\\n\");\n mi_json_buf_print(&hbuf, \"}\\n\");\n if (hbuf.used >= hbuf.size) {\n // failed\n if (hbuf.can_realloc) { mi_free(hbuf.buf); }\n return NULL;\n }\n else {\n return hbuf.buf;\n }\n}\n\nchar* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept {\n mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);\n if (subproc==NULL) return NULL;\n mi_stats_t_decl(stats);\n if (!mi_subproc_stats_get(subproc_id,&stats)) return NULL;\n return mi_stats_get_json_from(&stats, buf_size, buf);\n}\n\nchar* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept {\n return mi_stats_get_json_from(mi_heap_get_stats(heap), buf_size, buf);\n}\n\nchar* mi_stats_get_json(size_t buf_size, char* buf) mi_attr_noexcept {\n return mi_subproc_stats_get_json(mi_subproc_current(), buf_size, buf);\n}\n\nchar* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept {\n return mi_stats_get_json_from(stats, buf_size, buf);\n}\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/theap.c", "sha256": "7f76bcace4b0578f2063477752138adabd99f7b10a276998e296a312c47a7cfe", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/*----------------------------------------------------------------------------\nCopyright (c) 2018-2025, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\" // _mi_theap_default\n\n#if defined(_MSC_VER) && (_MSC_VER < 1920)\n#pragma warning(disable:4204) // non-constant aggregate initializer\n#endif\n\n/* -----------------------------------------------------------\n Helpers\n----------------------------------------------------------- */\n\n// return `true` if ok, `false` to break\ntypedef bool (theap_page_visitor_fun)(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2);\n\n// Visit all pages in a theap; returns `false` if break was called.\nstatic bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn, bool include_full, void* arg1, void* arg2)\n{\n if (theap==NULL || theap->page_count==0) return 0;\n\n // visit all pages\n #if MI_DEBUG>1\n size_t total = theap->page_count;\n size_t count = 0;\n #endif\n\n const size_t max_bin = (include_full ? MI_BIN_FULL : MI_BIN_FULL - 1);\n for (size_t i = 0; i <= max_bin; i++) {\n mi_page_queue_t* pq = &theap->pages[i];\n mi_page_t* page = pq->first;\n while(page != NULL) {\n mi_page_t* next = page->next; // save next in case the page gets removed from the queue\n mi_assert_internal(mi_page_theap(page) == theap);\n #if MI_DEBUG>1\n count++;\n #endif\n if (!fn(theap, pq, page, arg1, arg2)) return false;\n page = next; // and continue\n }\n }\n mi_assert_internal(!include_full || count == total);\n return true;\n}\n\n\n#if MI_DEBUG>=2\nstatic bool mi_theap_page_is_valid(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {\n MI_UNUSED(arg1);\n MI_UNUSED(arg2);\n MI_UNUSED(pq);\n mi_assert_internal(mi_page_theap(page) == theap);\n mi_assert_expensive(_mi_page_is_valid(page));\n return true;\n}\n#endif\n#if MI_DEBUG>=3\nstatic bool mi_theap_is_valid(mi_theap_t* theap) {\n mi_assert_internal(theap!=NULL);\n mi_theap_visit_pages(theap, &mi_theap_page_is_valid, true, NULL, NULL);\n for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {\n mi_assert_internal(_mi_page_queue_is_valid(theap, &theap->pages[bin]));\n }\n return true;\n}\n#endif\n\n\n\n\n/* -----------------------------------------------------------\n \"Collect\" pages by migrating `local_free` and `thread_free`\n lists and freeing empty pages. This is done when a thread\n stops (and in that case abandons pages if there are still\n blocks alive)\n----------------------------------------------------------- */\n\ntypedef enum mi_collect_e {\n MI_NORMAL,\n MI_FORCE,\n MI_ABANDON\n} mi_collect_t;\n\n\nstatic bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) {\n MI_UNUSED(arg2);\n MI_UNUSED(theap);\n mi_assert_internal(mi_theap_page_is_valid(theap, pq, page, NULL, NULL));\n mi_collect_t collect = *((mi_collect_t*)arg_collect);\n _mi_page_free_collect(page, collect >= MI_FORCE);\n if (mi_page_all_free(page)) {\n // no more used blocks, possibly free the page.\n if (collect >= MI_FORCE || page->retire_expire == 0) { // either forced/abandon, or not already retired\n // note: this will potentially free retired pages as well.\n _mi_page_free(page, pq);\n }\n }\n else if (collect == MI_ABANDON) {\n // still used blocks but the thread is done; abandon the page\n _mi_page_abandon(page, pq);\n }\n return true; // don't break\n}\n\nstatic void mi_theap_merge_stats(mi_theap_t* theap) {\n mi_assert_internal(mi_theap_is_initialized(theap));\n _mi_stats_merge_into(&_mi_theap_heap(theap)->stats, &theap->stats);\n}\n\nstatic void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect)\n{\n if (theap==NULL || !mi_theap_is_initialized(theap)) return;\n mi_assert_expensive(mi_theap_is_valid(theap));\n\n const bool force = (collect >= MI_FORCE);\n _mi_deferred_free(theap, force);\n\n // python/cpython#112532: we may be called from a thread that is not the owner of the theap\n // const bool is_main_thread = (_mi_is_main_thread() && theap->thread_id == _mi_thread_id());\n\n // collect retired pages\n _mi_theap_collect_retired(theap, force);\n\n // collect all pages owned by this thread\n mi_theap_visit_pages(theap, &mi_theap_page_collect, (collect!=MI_NORMAL), &collect, NULL); // dont normally visit full pages, see issue #1220\n\n // collect arenas (this is program wide so don't force purges on abandonment of threads)\n //mi_atomic_storei64_release(&theap->tld->subproc->purge_expire, 1);\n _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld);\n\n // merge statistics\n mi_theap_merge_stats(theap);\n}\n\nvoid _mi_theap_collect_abandon(mi_theap_t* theap) {\n mi_theap_collect_ex(theap, MI_ABANDON);\n}\n\nvoid mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept {\n mi_theap_collect_ex(theap, (force ? MI_FORCE : MI_NORMAL));\n}\n\nvoid mi_collect(bool force) mi_attr_noexcept {\n // cannot really collect process wide, just a theap..\n mi_theap_collect(_mi_theap_default(), force);\n}\n\nvoid mi_heap_collect(mi_heap_t* heap, bool force) {\n // cannot really collect a heap, just a theap..\n mi_theap_collect(mi_heap_theap(heap), force);\n}\n\n/* -----------------------------------------------------------\n Heap new\n----------------------------------------------------------- */\n\nmi_theap_t* mi_theap_get_default(void) {\n mi_theap_t* theap = _mi_theap_default();\n if mi_unlikely(!mi_theap_is_initialized(theap)) {\n mi_thread_init();\n theap = _mi_theap_default();\n mi_assert_internal(mi_theap_is_initialized(theap));\n }\n return theap;\n}\n\n// todo: make order of parameters consistent (but would that break compat with CPython?)\nvoid _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld)\n{\n mi_assert_internal(theap!=NULL);\n mi_assert_internal(heap!=NULL);\n mi_memid_t memid = theap->memid;\n _mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t));\n theap->memid = memid;\n theap->refcount = 1;\n theap->tld = tld; // avoid reading the thread-local tld during initialization\n mi_atomic_store_ptr_relaxed(mi_heap_t,&theap->heap,heap);\n mi_assert_internal(theap->stats.size == sizeof(mi_stats_t));\n \n _mi_theap_options_init(theap);\n if (theap->tld->is_in_threadpool) {\n // if we run as part of a thread pool it is better to not arbitrarily reclaim abandoned pages into our theap.\n // this is checked in `free.c:mi_free_try_collect_mt`\n // .. but abandoning is good in this case: halve the full page retain (possibly to 0)\n // (so blocked threads do not hold on to too much memory)\n if (theap->page_full_retain > 0) {\n theap->page_full_retain = theap->page_full_retain / 4;\n }\n }\n\n // push on the thread local theaps list\n mi_theap_t* head = NULL;\n mi_lock(&theap->tld->theaps_lock) {\n head = theap->tld->theaps;\n theap->tprev = NULL;\n theap->tnext = head;\n if (head!=NULL) { head->tprev = theap; }\n theap->tld->theaps = theap;\n }\n\n // initialize random\n if (head == NULL) { // first theap in this thread?\n #if defined(_WIN32) && !defined(MI_SHARED_LIB)\n _mi_random_init_weak(&theap->random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)\n #else\n _mi_random_init(&theap->random);\n #endif\n }\n else {\n _mi_random_split(&head->random, &theap->random);\n }\n theap->cookie = _mi_theap_random_next(theap) | 1;\n _mi_theap_guarded_init(theap);\n mi_subproc_stat_increase(_mi_subproc(),theaps,1);\n\n // push on the heap's theap list\n mi_lock(&heap->theaps_lock) {\n head = heap->theaps;\n theap->hprev = NULL;\n theap->hnext = head;\n if (head!=NULL) { head->hprev = theap; }\n heap->theaps = theap;\n }\n}\n\nmi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) {\n mi_assert_internal(tld!=NULL);\n mi_assert_internal(heap!=NULL);\n // allocate and initialize a theap\n mi_memid_t memid;\n mi_theap_t* theap;\n //if (!_mi_is_heap_main(heap)) {\n // theap = (mi_theap_t*)mi_heap_zalloc(mi_heap_main(),sizeof(mi_theap_t));\n // memid = _mi_memid_create(MI_MEM_HEAP_MAIN);\n // memid.initially_zero = memid.initially_committed = true;\n //}\n //else\n if (heap->exclusive_arena == NULL) {\n theap = (mi_theap_t*)_mi_meta_zalloc(sizeof(mi_theap_t), &memid);\n }\n else {\n // theaps associated with a specific arena are allocated in that arena\n // note: takes up at least one slice which is quite wasteful...\n const size_t size = _mi_align_up(sizeof(mi_theap_t),MI_ARENA_MIN_OBJ_SIZE);\n theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid);\n mi_assert_internal(memid.mem.os.size >= size);\n }\n if (theap==NULL) {\n _mi_error_message(ENOMEM, \"unable to allocate theap meta-data\\n\");\n return NULL;\n }\n theap->memid = memid;\n _mi_theap_init(theap, heap, tld);\n return theap;\n}\n\nuintptr_t _mi_theap_random_next(mi_theap_t* theap) {\n return _mi_random_next(&theap->random);\n}\n\nstatic void mi_theap_free_mem(mi_theap_t* theap) {\n if (theap!=NULL) {\n mi_subproc_stat_decrease(_mi_subproc(),theaps,1);\n // free the used memory\n if (theap->memid.memkind == MI_MEM_HEAP_MAIN) { // note: for now unused as it would access theap_default stats in mi_free of the current theap\n mi_assert_internal(_mi_is_heap_main(mi_heap_of(theap)));\n mi_free(theap);\n }\n else if (theap->memid.memkind == MI_MEM_META) {\n _mi_meta_free(theap, sizeof(*theap), theap->memid);\n }\n else {\n _mi_arenas_free(theap, _mi_align_up(sizeof(*theap),MI_ARENA_MIN_OBJ_SIZE), theap->memid ); // issue #1168, avoid assertion failure\n }\n }\n}\n\nvoid _mi_theap_incref(mi_theap_t* theap) {\n if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {\n mi_atomic_increment_acq_rel(&theap->refcount);\n }\n}\n\nvoid _mi_theap_decref(mi_theap_t* theap) {\n if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {\n if (mi_atomic_decrement_acq_rel(&theap->refcount) == 1) {\n mi_theap_free_mem(theap);\n }\n }\n}\n\n\n// called from `mi_theap_delete` to free the internal theap resources.\nbool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock) {\n mi_assert(theap != NULL);\n if (theap==NULL) return true;\n\n mi_heap_t* const heap = mi_atomic_exchange_ptr_acq_rel(mi_heap_t, &theap->heap, NULL);\n if (heap==NULL) {\n // concurrent interaction, retry in an outer loop (as the other thread may be blocked on our lock)\n return false;\n }\n else {\n // merge stats to the owning heap\n _mi_stats_merge_into(&heap->stats, &theap->stats);\n\n // remove ourselves from the heap theaps list\n mi_lock_maybe(&heap->theaps_lock, acquire_heap_theaps_lock) {\n if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; }\n if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; }\n else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; }\n theap->hnext = theap->hprev = NULL;\n }\n\n // remove ourselves from the thread local theaps list\n mi_lock_maybe(&theap->tld->theaps_lock, acquire_tld_theaps_lock) {\n if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; }\n if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; }\n else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; }\n theap->tnext = theap->tprev = NULL; \n }\n theap->tld = NULL;\n _mi_theap_decref(theap);\n return true;\n }\n}\n\n\n/* -----------------------------------------------------------\n Heap destroy\n----------------------------------------------------------- */\n/*\n\n// zero out the page queues\nstatic void mi_theap_reset_pages(mi_theap_t* theap) {\n mi_assert_internal(theap != NULL);\n mi_assert_internal(mi_theap_is_initialized(theap));\n // TODO: copy full empty theap instead?\n _mi_memset(&theap->pages_free_direct, 0, sizeof(theap->pages_free_direct));\n _mi_memcpy_aligned(&theap->pages, &_mi_theap_empty.pages, sizeof(theap->pages));\n // theap->thread_delayed_free = NULL;\n theap->page_count = 0;\n}\n\nstatic bool _mi_theap_page_destroy(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {\n MI_UNUSED(arg1);\n MI_UNUSED(arg2);\n MI_UNUSED(pq);\n\n // ensure no more thread_delayed_free will be added\n //_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false);\n\n // stats\n const size_t bsize = mi_page_block_size(page);\n if (bsize > MI_LARGE_MAX_OBJ_SIZE) {\n mi_theap_stat_decrease(theap, malloc_huge, bsize);\n }\n #if (MI_STAT>0)\n _mi_page_free_collect(page, false); // update used count\n const size_t inuse = page->used;\n if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {\n mi_theap_stat_decrease(theap, malloc_normal, bsize * inuse);\n #if (MI_STAT>1)\n mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], inuse);\n #endif\n }\n // mi_theap_stat_decrease(theap, malloc_requested, bsize * inuse); // todo: off for aligned blocks...\n #endif\n\n /// pretend it is all free now\n mi_assert_internal(mi_page_thread_free(page) == NULL);\n page->used = 0;\n\n // and free the page\n // mi_page_free(page,false);\n page->next = NULL;\n page->prev = NULL;\n mi_page_set_theap(page, NULL);\n _mi_arenas_page_free(page, theap);\n\n return true; // keep going\n}\n\nvoid _mi_theap_destroy_pages(mi_theap_t* theap) {\n mi_theap_visit_pages(theap, &_mi_theap_page_destroy, NULL, NULL);\n mi_theap_reset_pages(theap);\n}\n\n#if MI_TRACK_HEAP_DESTROY\nstatic bool mi_cdecl mi_theap_track_block_free(const mi_theap_t* theap, const mi_theap_area_t* area, void* block, size_t block_size, void* arg) {\n MI_UNUSED(theap); MI_UNUSED(area); MI_UNUSED(arg); MI_UNUSED(block_size);\n mi_track_free_size(block,mi_usable_size(block));\n return true;\n}\n#endif\n\nvoid mi_theap_destroy(mi_theap_t* theap) {\n mi_assert(theap != NULL);\n mi_assert(mi_theap_is_initialized(theap));\n mi_assert(!theap->allow_page_reclaim);\n mi_assert(!theap->allow_page_abandon);\n mi_assert_expensive(mi_theap_is_valid(theap));\n if (theap==NULL || !mi_theap_is_initialized(theap)) return;\n #if MI_GUARDED\n // _mi_warning_message(\"'mi_theap_destroy' called but MI_GUARDED is enabled -- using `mi_theap_delete` instead (theap at %p)\\n\", theap);\n mi_theap_delete(theap);\n return;\n #else\n if (theap->allow_page_reclaim) {\n _mi_warning_message(\"'mi_theap_destroy' called but ignored as the theap was not created with 'allow_destroy' (theap at %p)\\n\", theap);\n // don't free in case it may contain reclaimed pages,\n mi_theap_delete(theap);\n }\n else {\n // track all blocks as freed\n #if MI_TRACK_HEAP_DESTROY\n mi_theap_visit_blocks(theap, true, mi_theap_track_block_free, NULL);\n #endif\n // free all pages\n _mi_theap_destroy_pages(theap);\n mi_theap_free(theap,true);\n }\n #endif\n}\n\n// forcefully destroy all theaps in the current thread\nvoid _mi_theap_unsafe_destroy_all(mi_theap_t* theap) {\n mi_assert_internal(theap != NULL);\n if (theap == NULL) return;\n mi_theap_t* curr = theap->tld->theaps;\n while (curr != NULL) {\n mi_theap_t* next = curr->next;\n if (!curr->allow_page_reclaim) {\n mi_theap_destroy(curr);\n }\n else {\n _mi_theap_destroy_pages(curr);\n }\n curr = next;\n }\n}\n*/\n\n/* -----------------------------------------------------------\n Safe Heap delete\n----------------------------------------------------------- */\n\n// Safe delete a theap without freeing any still allocated blocks in that theap.\nvoid _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock)\n{\n mi_assert(theap != NULL);\n mi_assert(mi_theap_is_initialized(theap));\n mi_assert_expensive(mi_theap_is_valid(theap));\n if (theap==NULL || !mi_theap_is_initialized(theap)) return;\n\n // abandon all pages\n _mi_theap_collect_abandon(theap);\n\n mi_assert_internal(theap->page_count==0);\n _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock);\n}\n\n\n\n/* -----------------------------------------------------------\n Load/unload theaps\n----------------------------------------------------------- */\n/*\nvoid mi_theap_unload(mi_theap_t* theap) {\n mi_assert(mi_theap_is_initialized(theap));\n mi_assert_expensive(mi_theap_is_valid(theap));\n if (theap==NULL || !mi_theap_is_initialized(theap)) return;\n if (_mi_theap_heap(theap)->exclusive_arena == NULL) {\n _mi_warning_message(\"cannot unload theaps that are not associated with an exclusive arena\\n\");\n return;\n }\n\n // abandon all pages so all thread'id in the pages are cleared\n _mi_theap_collect_abandon(theap);\n mi_assert_internal(theap->page_count==0);\n\n // remove from theap list\n mi_theap_free(theap, false); // but don't actually free the memory\n\n // disassociate from the current thread-local and static state\n theap->tld = NULL;\n return;\n}\n\nbool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena_id) {\n mi_assert(mi_theap_is_initialized(theap));\n if (theap==NULL || !mi_theap_is_initialized(theap)) return false;\n if (_mi_theap_heap(theap)->exclusive_arena == NULL) {\n _mi_warning_message(\"cannot reload theaps that were not associated with an exclusive arena\\n\");\n return false;\n }\n if (theap->tld != NULL) {\n _mi_warning_message(\"cannot reload theaps that were not unloaded first\\n\");\n return false;\n }\n mi_arena_t* arena = _mi_arena_from_id(arena_id);\n if (_mi_theap_heap(theap)->exclusive_arena != arena) {\n _mi_warning_message(\"trying to reload a theap at a different arena address: %p vs %p\\n\", _mi_theap_heap(theap)->exclusive_arena, arena);\n return false;\n }\n\n mi_assert_internal(theap->page_count==0);\n\n // re-associate with the current thread-local and static state\n theap->tld = mi_theap_get_default()->tld;\n\n // reinit direct pages (as we may be in a different process)\n mi_assert_internal(theap->page_count == 0);\n for (size_t i = 0; i < MI_PAGES_DIRECT; i++) {\n theap->pages_free_direct[i] = (mi_page_t*)&_mi_page_empty;\n }\n\n // push on the thread local theaps list\n theap->tnext = theap->tld->theaps;\n theap->tld->theaps = theap;\n return true;\n}\n*/\n\n\n/* -----------------------------------------------------------\n Visit all theap blocks and areas\n Todo: enable visiting abandoned pages, and\n enable visiting all blocks of all theaps across threads\n----------------------------------------------------------- */\n\nvoid _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) {\n const size_t bsize = mi_page_block_size(page);\n const size_t ubsize = mi_page_usable_block_size(page);\n area->reserved = page->reserved * bsize;\n area->committed = page->capacity * bsize;\n area->blocks = mi_page_start(page);\n area->used = page->used; // number of blocks in use (#553)\n area->block_size = ubsize;\n area->full_block_size = bsize;\n area->reserved1 = page;\n}\n\nstatic void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) {\n mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX);\n *shift = MI_SIZE_BITS - mi_clz(divisor - 1);\n *magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1);\n}\n\nstatic size_t mi_fast_divide(size_t n, uint64_t magic, size_t shift) {\n mi_assert_internal(n <= UINT32_MAX);\n const uint64_t hi = ((uint64_t)n * magic) >> 32;\n return (size_t)((hi + n) >> shift);\n}\n\nbool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg) {\n mi_assert(area != NULL);\n if (area==NULL) return true;\n mi_assert(page != NULL);\n if (page == NULL) return true;\n\n _mi_page_free_collect(page,true); // collect both thread_delayed and local_free\n mi_assert_internal(page->local_free == NULL);\n if (page->used == 0) return true;\n\n size_t psize;\n uint8_t* const pstart = mi_page_area(page, &psize);\n mi_heap_t* const heap = mi_page_heap(page);\n const size_t bsize = mi_page_block_size(page);\n const size_t ubsize = mi_page_usable_block_size(page); // without padding\n\n // optimize page with one block\n if (page->capacity == 1) {\n mi_assert_internal(page->used == 1 && page->free == NULL);\n return visitor(heap, area, pstart, ubsize, arg);\n }\n mi_assert(bsize <= UINT32_MAX);\n\n // optimize full pages\n if (page->used == page->capacity) {\n uint8_t* block = pstart;\n for (size_t i = 0; i < page->capacity; i++) {\n if (!visitor(heap, area, block, ubsize, arg)) return false;\n block += bsize;\n }\n return true;\n }\n\n // create a bitmap of free blocks.\n #define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*))\n uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS];\n const uintptr_t bmapsize = _mi_divide_up(page->capacity, MI_INTPTR_BITS);\n memset(free_map, 0, bmapsize * sizeof(intptr_t));\n if (page->capacity % MI_INTPTR_BITS != 0) {\n // mark left-over bits at the end as free\n size_t shift = (page->capacity % MI_INTPTR_BITS);\n uintptr_t mask = (UINTPTR_MAX << shift);\n free_map[bmapsize - 1] = mask;\n }\n\n // fast repeated division by the block size\n uint64_t magic;\n size_t shift;\n mi_get_fast_divisor(bsize, &magic, &shift);\n\n #if MI_DEBUG>1\n size_t free_count = 0;\n #endif\n for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) {\n #if MI_DEBUG>1\n free_count++;\n #endif\n mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize));\n size_t offset = (uint8_t*)block - pstart;\n mi_assert_internal(offset % bsize == 0);\n mi_assert_internal(offset <= UINT32_MAX);\n size_t blockidx = mi_fast_divide(offset, magic, shift);\n mi_assert_internal(blockidx == offset / bsize);\n mi_assert_internal(blockidx < MI_MAX_BLOCKS);\n size_t bitidx = (blockidx / MI_INTPTR_BITS);\n size_t bit = blockidx - (bitidx * MI_INTPTR_BITS);\n free_map[bitidx] |= ((uintptr_t)1 << bit);\n }\n mi_assert_internal(page->capacity == (free_count + page->used));\n\n // walk through all blocks skipping the free ones\n #if MI_DEBUG>1\n size_t used_count = 0;\n #endif\n uint8_t* block = pstart;\n for (size_t i = 0; i < bmapsize; i++) {\n if (free_map[i] == 0) {\n // every block is in use\n for (size_t j = 0; j < MI_INTPTR_BITS; j++) {\n #if MI_DEBUG>1\n used_count++;\n #endif\n if (!visitor(heap, area, block, ubsize, arg)) return false;\n block += bsize;\n }\n }\n else {\n // visit the used blocks in the mask\n uintptr_t m = ~free_map[i];\n while (m != 0) {\n #if MI_DEBUG>1\n used_count++;\n #endif\n size_t bitidx = mi_ctz(m);\n if (!visitor(heap, area, block + (bitidx * bsize), ubsize, arg)) return false;\n m &= m - 1; // clear least significant bit\n }\n block += bsize * MI_INTPTR_BITS;\n }\n }\n mi_assert_internal(page->used == used_count);\n return true;\n}\n\n\n\n// Separate struct to keep `mi_page_t` out of the public interface\ntypedef struct mi_theap_area_ex_s {\n mi_heap_area_t area;\n mi_page_t* page;\n} mi_theap_area_ex_t;\n\ntypedef bool (mi_theap_area_visit_fun)(const mi_theap_t* theap, const mi_theap_area_ex_t* area, void* arg);\n\nstatic bool mi_theap_visit_areas_page(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) {\n MI_UNUSED(theap);\n MI_UNUSED(pq);\n mi_theap_area_visit_fun* fun = (mi_theap_area_visit_fun*)vfun;\n mi_theap_area_ex_t xarea;\n xarea.page = page;\n _mi_heap_area_init(&xarea.area, page);\n return fun(theap, &xarea, arg);\n}\n\n// Visit all theap pages as areas\nstatic bool mi_theap_visit_areas(const mi_theap_t* theap, mi_theap_area_visit_fun* visitor, void* arg) {\n if (visitor == NULL) return false;\n return mi_theap_visit_pages((mi_theap_t*)theap, &mi_theap_visit_areas_page, true, (void*)(visitor), arg); // note: function pointer to void* :-{\n}\n\n// Just to pass arguments\ntypedef struct mi_visit_blocks_args_s {\n bool visit_blocks;\n mi_block_visit_fun* visitor;\n void* arg;\n} mi_visit_blocks_args_t;\n\nstatic bool mi_theap_area_visitor(const mi_theap_t* theap, const mi_theap_area_ex_t* xarea, void* arg) {\n mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg;\n if (!args->visitor(_mi_theap_heap(theap), &xarea->area, NULL, xarea->area.block_size, args->arg)) return false;\n if (args->visit_blocks) {\n return _mi_theap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg);\n }\n else {\n return true;\n }\n}\n\n// Visit all blocks in a theap\nbool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {\n mi_visit_blocks_args_t args = { visit_blocks, visitor, arg };\n return mi_theap_visit_areas(theap, &mi_theap_area_visitor, &args);\n}\n\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "c_src/mimalloc/v3/src/threadlocal.c", "sha256": "49be69a4b674e09690a63870830f3ffb05698ed4210949ef5add11957a08325b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "/* ----------------------------------------------------------------------------\nCopyright (c) 2019-2026, Microsoft Research, Daan Leijen\nThis is free software; you can redistribute it and/or modify it under the\nterms of the MIT license. A copy of the license can be found in the file\n\"LICENSE\" at the root of this distribution.\n-----------------------------------------------------------------------------*/\n\n/* ----------------------------------------------------------------------------\nImplement dynamic thread local variables (for heap's).\nUnlike most OS native implementations there is no limit on the number\nthat can be allocated.\n-----------------------------------------------------------------------------*/\n\n#include \"mimalloc.h\"\n#include \"mimalloc/internal.h\"\n#include \"mimalloc/prim.h\"\n\n/* -----------------------------------------------------------\n Each thread can have (a dynamically expanding) array of\n thread-local values. Each slot has a value and a version.\n The version is used to safely reuse slots.\n----------------------------------------------------------- */\ntypedef struct mi_tls_slot_s {\n size_t version;\n void* value;\n} mi_tls_slot_t;\n\ntypedef struct mi_thread_locals_s {\n size_t count;\n mi_tls_slot_t slots[1];\n} mi_thread_locals_t;\n\nstatic mi_thread_locals_t mi_thread_locals_empty = { 0, {{0,NULL}} };\n\nmi_decl_thread mi_thread_locals_t* mi_thread_locals = &mi_thread_locals_empty; // always point to a valid `mi_thread_locals_t`\n\n\n/* -----------------------------------------------------------\n Each key consists of the slot index in the lower bits,\n and its version it the top bits. When we get a value\n the version must match or we return NULL. When we set\n a value, we also set the version of the key.\n----------------------------------------------------------- */\n\n#if MI_SIZE_BITS < 64\n#define MI_TLS_IDX_BITS (MI_SIZE_BITS/2) // half for the index, half for the version\n#else\n#define MI_TLS_IDX_BITS (MI_SIZE_BITS/4) // 16 bits for the index, 48 bits for the version\n#endif\n#define MI_TLS_IDX_MASK ((MI_ZU(1)<> MI_TLS_IDX_BITS);\n}\n\nstatic mi_thread_local_t mi_key_create( size_t index, size_t version ) {\n mi_assert_internal(version != 0 && version <= MI_TLS_VERSION_MAX);\n mi_assert_internal(index <= MI_TLS_IDX_MAX);\n const mi_thread_local_t key = ((version << MI_TLS_IDX_BITS) | index);\n mi_assert_internal(key != 0);\n return key;\n}\n\n\n// dynamically reallocate the thread local slots when needed\nstatic mi_thread_locals_t* mi_thread_locals_expand(size_t least_idx) {\n mi_thread_locals_t* tls_old = mi_thread_locals;\n const size_t count_old = tls_old->count;\n size_t count;\n if (count_old==0) {\n tls_old = NULL; // so we allocate fresh from mi_thread_locals_empty\n count = 16; // start with 16 slots\n } \n else if (count_old >= 1024) {\n count = count_old + 1024; // at some point increase linearly\n }\n else {\n count = 2*count_old; // and double initially\n }\n if (count <= least_idx) { \n count = least_idx + 1;\n }\n if (count > MI_TLS_IDX_MAX) { return NULL; } // too large\n mi_thread_locals_t* tls = (mi_thread_locals_t*)mi_rezalloc(tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t));\n if mi_unlikely(tls==NULL) return NULL;\n tls->count = count;\n mi_thread_locals = tls;\n return tls;\n}\n\nstatic mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key, void* val ) {\n if (val==NULL) return true;\n const size_t idx = mi_key_index(key); \n mi_thread_locals_t* tls = mi_thread_locals_expand(idx);\n if (tls==NULL) return false;\n mi_assert_internal(tls == mi_thread_locals);\n mi_assert_internal(idx < tls->count);\n tls->slots[idx].value = val;\n tls->slots[idx].version = mi_key_version(key);\n return true;\n}\n\n// set a tls slot; returns `true` if successful.\n// Can return `false` if we could not reallocate the slots array.\nbool _mi_thread_local_set( mi_thread_local_t key, void* val ) {\n mi_thread_locals_t* tls = mi_thread_locals;\n mi_assert_internal(tls!=NULL);\n mi_assert_internal(key!=0);\n const size_t idx = mi_key_index(key);\n if mi_likely(idx < tls->count) {\n tls->slots[idx].value = val;\n tls->slots[idx].version = mi_key_version(key);\n return true;\n }\n else {\n return mi_thread_local_set_expand( key, val ); // tailcall\n }\n}\n\n// get a tls slot value\nvoid* _mi_thread_local_get( mi_thread_local_t key ) {\n const mi_thread_locals_t* const tls = mi_thread_locals;\n mi_assert_internal(tls!=NULL);\n mi_assert_internal(key!=0);\n const size_t idx = mi_key_index(key);\n if mi_likely(idx < tls->count && mi_key_version(key) == tls->slots[idx].version) {\n return tls->slots[idx].value;\n }\n else {\n return NULL; \n }\n}\n\nvoid _mi_thread_locals_thread_done(void) {\n mi_thread_locals_t* const tls = mi_thread_locals;\n if (tls!=NULL && tls->count > 0) {\n mi_free(tls);\n mi_thread_locals = &mi_thread_locals_empty;\n }\n}\n\n/* -----------------------------------------------------------\nCreate and free fresh TLS key's\n----------------------------------------------------------- */\n#include \"bitmap.h\"\n\nstatic mi_lock_t mi_thread_locals_lock; // we need a lock in order to re-allocate the slot bits\nstatic mi_bitmap_t* mi_thread_locals_free; // reuse an arena bitmap to track which slots were assigned (1=free, 0=in-use)\nstatic size_t mi_thread_locals_version; // version to be able to reuse slots safely\n\nvoid _mi_thread_locals_init(void) {\n mi_lock_init(&mi_thread_locals_lock);\n}\n\nvoid _mi_thread_locals_done(void) {\n mi_lock(&mi_thread_locals_lock) {\n mi_bitmap_t* const slots = mi_thread_locals_free;\n mi_free(slots);\n }\n mi_lock_done(&mi_thread_locals_lock);\n}\n\n// strange signature but allows us to reuse the arena code for claiming free pages\nstatic bool mi_thread_local_claim_fun(size_t _slice_index, mi_arena_t* _arena, bool* keep_set) {\n MI_UNUSED(_slice_index); MI_UNUSED(_arena);\n *keep_set = false;\n return true;\n}\n\n// When we claim a free slot, we increase the global version counter \n// (so if we reuse a slot it will be returning NULL initially when a thread tries to get it)\nstatic mi_thread_local_t mi_thread_local_claim(void) {\n size_t idx = 0;\n if (mi_thread_locals_free != NULL && mi_bitmap_try_find_and_claim(mi_thread_locals_free,0,&idx,&mi_thread_local_claim_fun,NULL)) {\n mi_thread_locals_version++;\n if (mi_thread_locals_version >= MI_TLS_VERSION_MAX) { mi_thread_locals_version = 1; } /* wrap around the version */\n return mi_key_create( idx, mi_thread_locals_version);\n }\n else {\n return 0;\n }\n}\n\nstatic bool mi_thread_local_create_expand(void) {\n mi_bitmap_t* const slots = mi_thread_locals_free;\n // 1024 bits at a time\n const size_t oldcount = (slots==NULL ? 0 : mi_bitmap_max_bits(slots));\n const size_t newcount = 1024 + oldcount;\n if (newcount > MI_TLS_IDX_MAX) { return false; }\n const size_t newsize = mi_bitmap_size( newcount, NULL );\n mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE);\n if (newslots==NULL) { return false; }\n if (slots!=NULL) {\n // copy over the previous bitmap\n _mi_memcpy_aligned(newslots, slots, mi_bitmap_size(oldcount, NULL)); \n mi_free(slots);\n }\n mi_bitmap_init(newslots, newcount, true /* pretend already zero'd so we do not zero out the copied old entries */);\n mi_bitmap_unsafe_setN(newslots, oldcount, newcount - oldcount); /* set the new expanded slots as available */\n mi_thread_locals_free = newslots;\n return true;\n}\n\n\n// create a fresh key\nmi_thread_local_t _mi_thread_local_create(void) {\n mi_thread_local_t key = 0;\n mi_lock(&mi_thread_locals_lock) {\n key = mi_thread_local_claim();\n if (key==0) {\n if (mi_thread_local_create_expand()) {\n key = mi_thread_local_claim();\n }\n }\n }\n return key;\n}\n\n// free a key\nvoid _mi_thread_local_free(mi_thread_local_t key) {\n if (key==0) return;\n const size_t idx = mi_key_index(key);\n mi_lock(&mi_thread_locals_lock) {\n mi_bitmap_t* const slots = mi_thread_locals_free;\n if (slots!=NULL && idx < mi_bitmap_max_bits(slots)) {\n mi_bitmap_set(slots,idx);\n }\n }\n}\n\n" }, { "package": "libmimalloc-sys@0.1.49", "sourcePath": "src/lib.rs", "sha256": "3400737f6f802abffa8a7c30c26ee1240d9dd8b8e9afbd4f370da2dc9b3d23bb", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "#![no_std]\n// Copyright 2019 Octavian Oncescu\n\nuse core::ffi::c_void;\n\n#[cfg(feature = \"extended\")]\nmod extended;\n#[cfg(feature = \"extended\")]\npub use extended::*;\n\nextern \"C\" {\n /// Allocate zero-initialized `size` bytes.\n ///\n /// Returns a pointer to newly allocated zero-initialized memory, or null if\n /// out of memory.\n pub fn mi_zalloc(size: usize) -> *mut c_void;\n\n /// Allocate `size` bytes.\n ///\n /// Returns pointer to the allocated memory or null if out of memory.\n /// Returns a unique pointer if called with `size` 0.\n pub fn mi_malloc(size: usize) -> *mut c_void;\n\n /// Re-allocate memory to `newsize` bytes.\n ///\n /// Return pointer to the allocated memory or null if out of memory. If null\n /// is returned, the pointer `p` is not freed. Otherwise the original\n /// pointer is either freed or returned as the reallocated result (in case\n /// it fits in-place with the new size).\n ///\n /// If `p` is null, it behaves as [`mi_malloc`]. If `newsize` is larger than\n /// the original `size` allocated for `p`, the bytes after `size` are\n /// uninitialized.\n pub fn mi_realloc(p: *mut c_void, newsize: usize) -> *mut c_void;\n\n /// Allocate `size` bytes aligned by `alignment`, initialized to zero.\n ///\n /// Return pointer to the allocated memory or null if out of memory.\n ///\n /// Returns a unique pointer if called with `size` 0.\n pub fn mi_zalloc_aligned(size: usize, alignment: usize) -> *mut c_void;\n\n /// Allocate `size` bytes aligned by `alignment`.\n ///\n /// Return pointer to the allocated memory or null if out of memory.\n ///\n /// Returns a unique pointer if called with `size` 0.\n pub fn mi_malloc_aligned(size: usize, alignment: usize) -> *mut c_void;\n\n /// Re-allocate memory to `newsize` bytes, aligned by `alignment`.\n ///\n /// Return pointer to the allocated memory or null if out of memory. If null\n /// is returned, the pointer `p` is not freed. Otherwise the original\n /// pointer is either freed or returned as the reallocated result (in case\n /// it fits in-place with the new size).\n ///\n /// If `p` is null, it behaves as [`mi_malloc_aligned`]. If `newsize` is\n /// larger than the original `size` allocated for `p`, the bytes after\n /// `size` are uninitialized.\n pub fn mi_realloc_aligned(p: *mut c_void, newsize: usize, alignment: usize) -> *mut c_void;\n\n /// Free previously allocated memory.\n ///\n /// The pointer `p` must have been allocated before (or be null).\n pub fn mi_free(p: *mut c_void);\n}\n\n/// When using the `\"override\"` feature flag, the user wants us to globally\n/// override the system allocator.\n///\n/// However, since we build and link `mimalloc` as a static library/archive,\n/// the linker may decide to not care about our overrides if it can't directly\n/// see references to the symbols, see the following link for details:\n/// \n///\n/// This is problematic if `mimalloc` is used from a library that by itself\n/// doesn't allocate, yet invokes other shared libraries that do, since then\n/// the linker wouldn't see any references to `malloc`/`free`, and the symbols\n/// would not be overridden.\n///\n/// To avoid this problem, we make sure that the allocator functions are\n/// visible to the linker.\n///\n/// To avoid this problem, we reference `mi_malloc` in a `#[used]` static.\n/// This makes it known to `rustc`, which will create a reference to it in a\n/// `symbols.o` stub file that is later passed directly to the linker (instead\n/// of being in an archive). See the following link for details on how this\n/// works: \n///\n/// NOTE: This works because `mimalloc` is compiled into a single object file\n/// in `static.c`. If it was split across multiple files, we'd need to\n/// reference each symbol. See also the comment at the top of `static.c`.\n///\n/// NOTE: On macOS, mimalloc doesn't just override malloc/free, but also\n/// registers itself with the allocator's zone APIs in a ctor\n/// (`_mi_macos_override_malloc`, marked with `__attribute__((constructor))`).\n/// Similarly to above, for the Mach-O linker to actually consider ctors as\n/// \"used\" when defined in an archive member in a static library, so we need\n/// to explicitly reference something in the object file. The constructor\n/// symbol itself is static, so we can't get a reference to that, so instead\n/// we reference `mi_malloc` here too).\n#[cfg(feature = \"override\")]\nmod set_up_statics {\n use super::*;\n #[used] // Could be `#[used(linker)]` once stable\n static USED: unsafe extern \"C\" fn(usize) -> *mut c_void = mi_malloc;\n}\n\n#[cfg(test)]\nmod tests {\n extern crate libc;\n use super::*;\n\n #[test]\n fn it_frees_memory_malloc() {\n let ptr = unsafe { mi_malloc_aligned(8, 8) } as *mut u8;\n unsafe { mi_free(ptr as *mut c_void) };\n }\n\n #[test]\n fn it_frees_memory_zalloc() {\n let ptr = unsafe { mi_zalloc_aligned(8, 8) } as *mut u8;\n unsafe { mi_free(ptr as *mut c_void) };\n }\n\n #[test]\n fn it_frees_memory_realloc() {\n let ptr = unsafe { mi_malloc_aligned(8, 8) } as *mut u8;\n let ptr = unsafe { mi_realloc_aligned(ptr as *mut c_void, 8, 8) } as *mut u8;\n unsafe { mi_free(ptr as *mut c_void) };\n }\n\n #[cfg(all(feature = \"override\", target_vendor = \"apple\"))]\n #[test]\n fn mimalloc_and_libc_are_interoperable_when_overridden() {\n let ptr = unsafe { mi_malloc(42) };\n unsafe { libc::free(ptr) };\n }\n}\n" }, { "package": "log@0.4.34", "sourcePath": "src/lib.rs", "sha256": "f6633a72e77e5039ef767189116986ba12a57e43304b40eebb466959e03b9a92", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! A lightweight logging facade.\n//!\n//! The `log` crate provides a single logging API that abstracts over the\n//! actual logging implementation. Libraries can use the logging API provided\n//! by this crate, and the consumer of those libraries can choose the logging\n//! implementation that is most suitable for its use case.\n//!\n//! If no logging implementation is selected, the facade falls back to a \"noop\"\n//! implementation that ignores all log messages. The overhead in this case\n//! is very small - just an integer load, comparison and jump.\n//!\n//! A log request consists of a _target_, a _level_, and a _body_. A target is a\n//! string which defaults to the module path of the location of the log request,\n//! though that default may be overridden. Logger implementations typically use\n//! the target to filter requests based on some user configuration.\n//!\n//! # Usage\n//!\n//! The basic use of the log crate is through the five logging macros: [`error!`],\n//! [`warn!`], [`info!`], [`debug!`] and [`trace!`]\n//! where `error!` represents the highest-priority log messages\n//! and `trace!` the lowest. The log messages are filtered by configuring\n//! the log level to exclude messages with a lower priority.\n//! Each of these macros accept format strings similarly to [`println!`].\n//!\n//!\n//! [`error!`]: ./macro.error.html\n//! [`warn!`]: ./macro.warn.html\n//! [`info!`]: ./macro.info.html\n//! [`debug!`]: ./macro.debug.html\n//! [`trace!`]: ./macro.trace.html\n//! [`println!`]: https://doc.rust-lang.org/stable/std/macro.println.html\n//!\n//! Avoid writing expressions with side-effects in log statements. They may not be evaluated.\n//!\n//! ## In libraries\n//!\n//! Libraries should link only to the `log` crate, and use the provided\n//! macros to log whatever information will be useful to downstream consumers.\n//!\n//! ### Examples\n//!\n//! ```\n//! # #[derive(Debug)] pub struct Yak(String);\n//! # impl Yak { fn shave(&mut self, _: u32) {} }\n//! # fn find_a_razor() -> Result { Ok(1) }\n//! use log::{info, warn};\n//!\n//! pub fn shave_the_yak(yak: &mut Yak) {\n//! info!(target: \"yak_events\", \"Commencing yak shaving for {yak:?}\");\n//!\n//! loop {\n//! match find_a_razor() {\n//! Ok(razor) => {\n//! info!(\"Razor located: {razor}\");\n//! yak.shave(razor);\n//! break;\n//! }\n//! Err(err) => {\n//! warn!(\"Unable to locate a razor: {err}, retrying\");\n//! }\n//! }\n//! }\n//! }\n//! # fn main() {}\n//! ```\n//!\n//! ## In executables\n//!\n//! Executables should choose a logging implementation and initialize it early in the\n//! runtime of the program. Logging implementations will typically include a\n//! function to do this. Any log messages generated before\n//! the implementation is initialized will be ignored.\n//!\n//! The executable itself may use the `log` crate to log as well.\n//!\n//! ### Warning\n//!\n//! The logging system may only be initialized once.\n//!\n//! ## Structured logging\n//!\n//! If you enable the `kv` feature you can associate structured values\n//! with your log records. If we take the example from before, we can include\n//! some additional context besides what's in the formatted message:\n//!\n//! ```\n//! # use serde::Serialize;\n//! # #[derive(Debug, Serialize)] pub struct Yak(String);\n//! # impl Yak { fn shave(&mut self, _: u32) {} }\n//! # fn find_a_razor() -> Result { Ok(1) }\n//! # #[cfg(feature = \"kv_serde\")]\n//! # fn main() {\n//! use log::{info, warn};\n//!\n//! pub fn shave_the_yak(yak: &mut Yak) {\n//! info!(target: \"yak_events\", yak:serde; \"Commencing yak shaving\");\n//!\n//! loop {\n//! match find_a_razor() {\n//! Ok(razor) => {\n//! info!(razor; \"Razor located\");\n//! yak.shave(razor);\n//! break;\n//! }\n//! Err(e) => {\n//! warn!(e:err; \"Unable to locate a razor, retrying\");\n//! }\n//! }\n//! }\n//! }\n//! # }\n//! # #[cfg(not(feature = \"kv_serde\"))]\n//! # fn main() {}\n//! ```\n//!\n//! See the [`kv`] module documentation for more details.\n//!\n//! # Available logging implementations\n//!\n//! In order to produce log output executables have to use\n//! a logger implementation compatible with the facade.\n//! There are many available implementations to choose from,\n//! here are some of the most popular ones:\n//!\n//! * Simple minimal loggers:\n//! * [env_logger]\n//! * [colog]\n//! * [simple_logger]\n//! * [simplelog]\n//! * [pretty_env_logger]\n//! * [stderrlog]\n//! * [flexi_logger]\n//! * [call_logger]\n//! * [std-logger]\n//! * [structured-logger]\n//! * [clang_log]\n//! * [ftail]\n//! * Complex configurable frameworks:\n//! * [log4rs]\n//! * [logforth]\n//! * [fern]\n//! * [spdlog-rs]\n//! * Adaptors for other facilities:\n//! * [syslog]\n//! * [slog-stdlog]\n//! * [systemd-journal-logger]\n//! * [android_log]\n//! * [win_dbg_logger]\n//! * [db_logger]\n//! * [log-to-defmt]\n//! * [logcontrol-log]\n//! * For WebAssembly binaries:\n//! * [console_log]\n//! * For dynamic libraries:\n//! * You may need to construct an FFI-safe wrapper over `log` to initialize in your libraries\n//! * Utilities:\n//! * [log_err]\n//! * [log-reload]\n//! * [alterable_logger]\n//!\n//! # Implementing a Logger\n//!\n//! Loggers implement the [`Log`] trait. Here's a very basic example that simply\n//! logs all messages at the [`Error`][level_link], [`Warn`][level_link] or\n//! [`Info`][level_link] levels to stdout:\n//!\n//! ```\n//! use log::{Record, Level, Metadata};\n//!\n//! struct SimpleLogger;\n//!\n//! impl log::Log for SimpleLogger {\n//! fn enabled(&self, metadata: &Metadata) -> bool {\n//! metadata.level() <= Level::Info\n//! }\n//!\n//! fn log(&self, record: &Record) {\n//! if self.enabled(record.metadata()) {\n//! println!(\"{} - {}\", record.level(), record.args());\n//! }\n//! }\n//!\n//! fn flush(&self) {}\n//! }\n//!\n//! # fn main() {}\n//! ```\n//!\n//! Loggers are installed by calling the [`set_logger`] function. The maximum\n//! log level also needs to be adjusted via the [`set_max_level`] function. The\n//! logging facade uses this as an optimization to improve performance of log\n//! messages at levels that are disabled. It's important to set it, as it\n//! defaults to [`Off`][filter_link], so no log messages will ever be captured!\n//! In the case of our example logger, we'll want to set the maximum log level\n//! to [`Info`][filter_link], since we ignore any [`Debug`][level_link] or\n//! [`Trace`][level_link] level log messages. A logging implementation should\n//! provide a function that wraps a call to [`set_logger`] and\n//! [`set_max_level`], handling initialization of the logger:\n//!\n//! ```\n//! # use log::{Level, Metadata};\n//! # struct SimpleLogger;\n//! # impl log::Log for SimpleLogger {\n//! # fn enabled(&self, _: &Metadata) -> bool { false }\n//! # fn log(&self, _: &log::Record) {}\n//! # fn flush(&self) {}\n//! # }\n//! # fn main() {}\n//! use log::{SetLoggerError, LevelFilter};\n//!\n//! static LOGGER: SimpleLogger = SimpleLogger;\n//!\n//! pub fn init() -> Result<(), SetLoggerError> {\n//! log::set_logger(&LOGGER)\n//! .map(|()| log::set_max_level(LevelFilter::Info))\n//! }\n//! ```\n//!\n//! Implementations that adjust their configurations at runtime should take care\n//! to adjust the maximum log level as well.\n//!\n//! # Use with `alloc`\n//!\n//! `set_logger` requires you to provide a `&'static Log`, which can be hard to\n//! obtain if your logger depends on some runtime configuration. The\n//! `set_boxed_logger` function is available with the `alloc` Cargo feature. It\n//! is identical to `set_logger` except that it takes a `Box` rather than a\n//! `&'static Log`:\n//!\n//! ```\n//! # use log::{Level, LevelFilter, Log, SetLoggerError, Metadata};\n//! # struct SimpleLogger;\n//! # impl log::Log for SimpleLogger {\n//! # fn enabled(&self, _: &Metadata) -> bool { false }\n//! # fn log(&self, _: &log::Record) {}\n//! # fn flush(&self) {}\n//! # }\n//! # fn main() {}\n//! # #[cfg(feature = \"alloc\")]\n//! # extern crate alloc;\n//! # #[cfg(feature = \"alloc\")]\n//! # use alloc::boxed::Box;\n//! # #[cfg(feature = \"alloc\")]\n//! pub fn init() -> Result<(), SetLoggerError> {\n//! log::set_boxed_logger(Box::new(SimpleLogger))\n//! .map(|()| log::set_max_level(LevelFilter::Info))\n//! }\n//! ```\n//!\n//! # Compile time filters\n//!\n//! Log levels can be statically disabled at compile time by enabling one of these Cargo features:\n//!\n//! * `max_level_off`\n//! * `max_level_error`\n//! * `max_level_warn`\n//! * `max_level_info`\n//! * `max_level_debug`\n//! * `max_level_trace`\n//!\n//! Log invocations at disabled levels will be skipped and will not even be present in the\n//! resulting binary. These features control the value of the `STATIC_MAX_LEVEL` constant. The\n//! logging macros check this value before logging a message. By default, no levels are disabled.\n//!\n//! It is possible to override this level for release builds only with the following features:\n//!\n//! * `release_max_level_off`\n//! * `release_max_level_error`\n//! * `release_max_level_warn`\n//! * `release_max_level_info`\n//! * `release_max_level_debug`\n//! * `release_max_level_trace`\n//!\n//! Libraries should avoid using the max level features because they're global and can't be changed\n//! once they're set.\n//!\n//! For example, a crate can disable trace level logs in debug builds and trace, debug, and info\n//! level logs in release builds with the following configuration:\n//!\n//! ```toml\n//! [dependencies]\n//! log = { version = \"0.4\", features = [\"max_level_debug\", \"release_max_level_warn\"] }\n//! ```\n//! # Crate Feature Flags\n//!\n//! The following crate feature flags are available in addition to the filters. They are\n//! configured in your `Cargo.toml`.\n//!\n//! * `alloc` enables using `alloc::boxed::Box` and `set_boxed_logger`.\n//! * `std` enables `alloc` and allows use of the `std` crate instead of the default `core`.\n//! It also enables using `std::error`.\n//! * `serde` enables support for serialization and deserialization of `Level` and `LevelFilter`.\n//!\n//! ```toml\n//! [dependencies]\n//! log = { version = \"0.4\", features = [\"std\", \"serde\"] }\n//! ```\n//!\n//! # Version compatibility\n//!\n//! The 0.3 and 0.4 versions of the `log` crate are almost entirely compatible. Log messages\n//! made using `log` 0.3 will forward transparently to a logger implementation using `log` 0.4. Log\n//! messages made using `log` 0.4 will forward to a logger implementation using `log` 0.3, but the\n//! module path and file name information associated with the message will unfortunately be lost.\n//!\n//! [`Log`]: trait.Log.html\n//! [level_link]: enum.Level.html\n//! [filter_link]: enum.LevelFilter.html\n//! [`set_logger`]: fn.set_logger.html\n//! [`set_max_level`]: fn.set_max_level.html\n//! [`try_set_logger_raw`]: fn.try_set_logger_raw.html\n//! [`shutdown_logger_raw`]: fn.shutdown_logger_raw.html\n//! [env_logger]: https://docs.rs/env_logger/*/env_logger/\n//! [colog]: https://docs.rs/colog/*/colog/\n//! [simple_logger]: https://github.com/borntyping/rust-simple_logger\n//! [simplelog]: https://github.com/drakulix/simplelog.rs\n//! [pretty_env_logger]: https://docs.rs/pretty_env_logger/*/pretty_env_logger/\n//! [stderrlog]: https://docs.rs/stderrlog/*/stderrlog/\n//! [flexi_logger]: https://docs.rs/flexi_logger/*/flexi_logger/\n//! [call_logger]: https://docs.rs/call_logger/*/call_logger/\n//! [std-logger]: https://docs.rs/std-logger/*/std_logger/\n//! [syslog]: https://docs.rs/syslog/*/syslog/\n//! [slog-stdlog]: https://docs.rs/slog-stdlog/*/slog_stdlog/\n//! [log4rs]: https://docs.rs/log4rs/*/log4rs/\n//! [logforth]: https://docs.rs/logforth/*/logforth/\n//! [fern]: https://docs.rs/fern/*/fern/\n//! [spdlog-rs]: https://docs.rs/spdlog-rs/*/spdlog/\n//! [systemd-journal-logger]: https://docs.rs/systemd-journal-logger/*/systemd_journal_logger/\n//! [android_log]: https://docs.rs/android_log/*/android_log/\n//! [win_dbg_logger]: https://docs.rs/win_dbg_logger/*/win_dbg_logger/\n//! [db_logger]: https://docs.rs/db_logger/*/db_logger/\n//! [log-to-defmt]: https://docs.rs/log-to-defmt/*/log_to_defmt/\n//! [console_log]: https://docs.rs/console_log/*/console_log/\n//! [structured-logger]: https://docs.rs/structured-logger/latest/structured_logger/\n//! [logcontrol-log]: https://docs.rs/logcontrol-log/*/logcontrol_log/\n//! [log_err]: https://docs.rs/log_err/*/log_err/\n//! [log-reload]: https://docs.rs/log-reload/*/log_reload/\n//! [alterable_logger]: https://docs.rs/alterable_logger/*/alterable_logger\n//! [clang_log]: https://docs.rs/clang_log/latest/clang_log\n//! [ftail]: https://docs.rs/ftail/latest/ftail\n\n#![doc(\n html_logo_url = \"https://prev.rust-lang.org/logos/rust-logo-128x128-blk-v2.png\",\n html_favicon_url = \"https://prev.rust-lang.org/favicon.ico\",\n html_root_url = \"https://docs.rs/log/0.4.34\"\n)]\n#![warn(missing_docs)]\n#![deny(missing_debug_implementations, unconditional_recursion)]\n#![cfg_attr(all(not(feature = \"std\"), not(test)), no_std)]\n\n#[cfg(any(\n all(feature = \"max_level_off\", feature = \"max_level_error\"),\n all(feature = \"max_level_off\", feature = \"max_level_warn\"),\n all(feature = \"max_level_off\", feature = \"max_level_info\"),\n all(feature = \"max_level_off\", feature = \"max_level_debug\"),\n all(feature = \"max_level_off\", feature = \"max_level_trace\"),\n all(feature = \"max_level_error\", feature = \"max_level_warn\"),\n all(feature = \"max_level_error\", feature = \"max_level_info\"),\n all(feature = \"max_level_error\", feature = \"max_level_debug\"),\n all(feature = \"max_level_error\", feature = \"max_level_trace\"),\n all(feature = \"max_level_warn\", feature = \"max_level_info\"),\n all(feature = \"max_level_warn\", feature = \"max_level_debug\"),\n all(feature = \"max_level_warn\", feature = \"max_level_trace\"),\n all(feature = \"max_level_info\", feature = \"max_level_debug\"),\n all(feature = \"max_level_info\", feature = \"max_level_trace\"),\n all(feature = \"max_level_debug\", feature = \"max_level_trace\"),\n))]\ncompile_error!(\"multiple max_level_* features set\");\n\n#[rustfmt::skip]\n#[cfg(any(\n all(feature = \"release_max_level_off\", feature = \"release_max_level_error\"),\n all(feature = \"release_max_level_off\", feature = \"release_max_level_warn\"),\n all(feature = \"release_max_level_off\", feature = \"release_max_level_info\"),\n all(feature = \"release_max_level_off\", feature = \"release_max_level_debug\"),\n all(feature = \"release_max_level_off\", feature = \"release_max_level_trace\"),\n all(feature = \"release_max_level_error\", feature = \"release_max_level_warn\"),\n all(feature = \"release_max_level_error\", feature = \"release_max_level_info\"),\n all(feature = \"release_max_level_error\", feature = \"release_max_level_debug\"),\n all(feature = \"release_max_level_error\", feature = \"release_max_level_trace\"),\n all(feature = \"release_max_level_warn\", feature = \"release_max_level_info\"),\n all(feature = \"release_max_level_warn\", feature = \"release_max_level_debug\"),\n all(feature = \"release_max_level_warn\", feature = \"release_max_level_trace\"),\n all(feature = \"release_max_level_info\", feature = \"release_max_level_debug\"),\n all(feature = \"release_max_level_info\", feature = \"release_max_level_trace\"),\n all(feature = \"release_max_level_debug\", feature = \"release_max_level_trace\"),\n))]\ncompile_error!(\"multiple release_max_level_* features set\");\n\n#[cfg(feature = \"alloc\")]\nextern crate alloc;\n#[cfg(all(not(feature = \"std\"), not(test)))]\nextern crate core as std;\n\n#[cfg(feature = \"alloc\")]\nuse alloc::boxed::Box;\nuse std::cfg;\n#[cfg(feature = \"std\")]\nuse std::error;\nuse std::str::FromStr;\nuse std::{cmp, fmt, mem};\n\n#[macro_use]\nmod macros;\nmod serde;\n\n#[cfg(feature = \"kv\")]\npub mod kv;\n\n#[cfg(target_has_atomic = \"ptr\")]\nuse std::sync::atomic::{AtomicUsize, Ordering};\n\n#[cfg(not(target_has_atomic = \"ptr\"))]\nuse std::cell::Cell;\n#[cfg(not(target_has_atomic = \"ptr\"))]\nuse std::sync::atomic::Ordering;\n\n#[cfg(not(target_has_atomic = \"ptr\"))]\nstruct AtomicUsize {\n v: Cell,\n}\n\n#[cfg(not(target_has_atomic = \"ptr\"))]\nimpl AtomicUsize {\n const fn new(v: usize) -> AtomicUsize {\n AtomicUsize { v: Cell::new(v) }\n }\n\n fn load(&self, _order: Ordering) -> usize {\n self.v.get()\n }\n\n fn store(&self, val: usize, _order: Ordering) {\n self.v.set(val)\n }\n}\n\n// Any platform without atomics is unlikely to have multiple cores, so\n// writing via Cell will not be a race condition.\n#[cfg(not(target_has_atomic = \"ptr\"))]\nunsafe impl Sync for AtomicUsize {}\n\n// The LOGGER static holds a pointer to the global logger. It is protected by\n// the STATE static which determines whether LOGGER has been initialized yet.\nstatic mut LOGGER: &dyn Log = &NopLogger;\n\nstatic STATE: AtomicUsize = AtomicUsize::new(0);\n\n// There are three different states that we care about: the logger's\n// uninitialized, the logger's initializing (set_logger's been called but\n// LOGGER hasn't actually been set yet), or the logger's active.\nconst UNINITIALIZED: usize = 0;\nconst INITIALIZING: usize = 1;\nconst INITIALIZED: usize = 2;\n\nstatic MAX_LOG_LEVEL_FILTER: AtomicUsize = AtomicUsize::new(0);\n\nstatic LOG_LEVEL_NAMES: [&str; 6] = [\"OFF\", \"ERROR\", \"WARN\", \"INFO\", \"DEBUG\", \"TRACE\"];\n\nstatic SET_LOGGER_ERROR: &str = \"attempted to set a logger after the logging system \\\n was already initialized\";\nstatic LEVEL_PARSE_ERROR: &str =\n \"attempted to convert a string that doesn't match an existing log level\";\n\n/// An enum representing the available verbosity levels of the logger.\n///\n/// Typical usage includes: checking if a certain `Level` is enabled with\n/// [`log_enabled!`](macro.log_enabled.html), specifying the `Level` of\n/// [`log!`](macro.log.html), and comparing a `Level` directly to a\n/// [`LevelFilter`](enum.LevelFilter.html).\n#[repr(usize)]\n#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]\npub enum Level {\n /// The \"error\" level.\n ///\n /// Designates very serious errors.\n // This way these line up with the discriminants for LevelFilter below\n // This works because Rust treats field-less enums the same way as C does:\n // https://doc.rust-lang.org/reference/items/enumerations.html#custom-discriminant-values-for-field-less-enumerations\n Error = 1,\n /// The \"warn\" level.\n ///\n /// Designates hazardous situations.\n Warn,\n /// The \"info\" level.\n ///\n /// Designates useful information.\n Info,\n /// The \"debug\" level.\n ///\n /// Designates lower priority information.\n Debug,\n /// The \"trace\" level.\n ///\n /// Designates very low priority, often extremely verbose, information.\n Trace,\n}\n\nimpl PartialEq for Level {\n #[inline]\n fn eq(&self, other: &LevelFilter) -> bool {\n *self as usize == *other as usize\n }\n}\n\nimpl PartialOrd for Level {\n #[inline]\n fn partial_cmp(&self, other: &LevelFilter) -> Option {\n Some((*self as usize).cmp(&(*other as usize)))\n }\n}\n\nimpl FromStr for Level {\n type Err = ParseLevelError;\n fn from_str(level: &str) -> Result {\n // iterate from 1, excluding \"OFF\"\n for idx in 1..LOG_LEVEL_NAMES.len() {\n if LOG_LEVEL_NAMES[idx].eq_ignore_ascii_case(level) {\n return Ok(Level::from_usize(idx).unwrap());\n }\n }\n Err(ParseLevelError(()))\n }\n}\n\nimpl fmt::Display for Level {\n fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {\n fmt.pad(self.as_str())\n }\n}\n\nimpl Level {\n fn from_usize(u: usize) -> Option {\n match u {\n 1 => Some(Level::Error),\n 2 => Some(Level::Warn),\n 3 => Some(Level::Info),\n 4 => Some(Level::Debug),\n 5 => Some(Level::Trace),\n _ => None,\n }\n }\n\n /// Returns the most verbose logging level.\n #[inline]\n pub fn max() -> Level {\n Level::Trace\n }\n\n /// Converts the `Level` to the equivalent `LevelFilter`.\n #[inline]\n pub fn to_level_filter(&self) -> LevelFilter {\n LevelFilter::from_usize(*self as usize).unwrap()\n }\n\n /// Returns the string representation of the `Level`.\n ///\n /// This returns the same string as the `fmt::Display` implementation.\n pub fn as_str(&self) -> &'static str {\n LOG_LEVEL_NAMES[*self as usize]\n }\n\n /// Iterate through all supported logging levels.\n ///\n /// The order of iteration is from more severe to less severe log messages.\n ///\n /// # Examples\n ///\n /// ```\n /// use log::Level;\n ///\n /// let mut levels = Level::iter();\n ///\n /// assert_eq!(Some(Level::Error), levels.next());\n /// assert_eq!(Some(Level::Trace), levels.last());\n /// ```\n pub fn iter() -> impl Iterator {\n (1..6).map(|i| Self::from_usize(i).unwrap())\n }\n\n /// Get the next-highest `Level` from this one.\n ///\n /// If the current `Level` is at the highest level, the returned `Level` will be the same as the\n /// current one.\n ///\n /// # Examples\n ///\n /// ```\n /// use log::Level;\n ///\n /// let level = Level::Info;\n ///\n /// assert_eq!(Level::Debug, level.increment_severity());\n /// assert_eq!(Level::Trace, level.increment_severity().increment_severity());\n /// assert_eq!(Level::Trace, level.increment_severity().increment_severity().increment_severity()); // max level\n /// ```\n pub fn increment_severity(&self) -> Self {\n let current = *self as usize;\n Self::from_usize(current + 1).unwrap_or(*self)\n }\n\n /// Get the next-lowest `Level` from this one.\n ///\n /// If the current `Level` is at the lowest level, the returned `Level` will be the same as the\n /// current one.\n ///\n /// # Examples\n ///\n /// ```\n /// use log::Level;\n ///\n /// let level = Level::Info;\n ///\n /// assert_eq!(Level::Warn, level.decrement_severity());\n /// assert_eq!(Level::Error, level.decrement_severity().decrement_severity());\n /// assert_eq!(Level::Error, level.decrement_severity().decrement_severity().decrement_severity()); // min level\n /// ```\n pub fn decrement_severity(&self) -> Self {\n let current = *self as usize;\n Self::from_usize(current.saturating_sub(1)).unwrap_or(*self)\n }\n}\n\n/// An enum representing the available verbosity level filters of the logger.\n///\n/// A `LevelFilter` may be compared directly to a [`Level`]. Use this type\n/// to get and set the maximum log level with [`max_level()`] and [`set_max_level`].\n///\n/// [`Level`]: enum.Level.html\n/// [`max_level()`]: fn.max_level.html\n/// [`set_max_level`]: fn.set_max_level.html\n#[repr(usize)]\n#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]\npub enum LevelFilter {\n /// A level lower than all log levels.\n Off,\n /// Corresponds to the `Error` log level.\n Error,\n /// Corresponds to the `Warn` log level.\n Warn,\n /// Corresponds to the `Info` log level.\n Info,\n /// Corresponds to the `Debug` log level.\n Debug,\n /// Corresponds to the `Trace` log level.\n Trace,\n}\n\nimpl PartialEq for LevelFilter {\n #[inline]\n fn eq(&self, other: &Level) -> bool {\n other.eq(self)\n }\n}\n\nimpl PartialOrd for LevelFilter {\n #[inline]\n fn partial_cmp(&self, other: &Level) -> Option {\n Some((*self as usize).cmp(&(*other as usize)))\n }\n}\n\nimpl FromStr for LevelFilter {\n type Err = ParseLevelError;\n fn from_str(level: &str) -> Result {\n // iterate from 0, including \"OFF\"\n for idx in 0..LOG_LEVEL_NAMES.len() {\n if LOG_LEVEL_NAMES[idx].eq_ignore_ascii_case(level) {\n return Ok(LevelFilter::from_usize(idx).unwrap());\n }\n }\n Err(ParseLevelError(()))\n }\n}\n\nimpl fmt::Display for LevelFilter {\n fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {\n fmt.pad(self.as_str())\n }\n}\n\nimpl LevelFilter {\n fn from_usize(u: usize) -> Option {\n match u {\n 0 => Some(LevelFilter::Off),\n 1 => Some(LevelFilter::Error),\n 2 => Some(LevelFilter::Warn),\n 3 => Some(LevelFilter::Info),\n 4 => Some(LevelFilter::Debug),\n 5 => Some(LevelFilter::Trace),\n _ => None,\n }\n }\n\n /// Returns the most verbose logging level filter.\n #[inline]\n pub fn max() -> LevelFilter {\n LevelFilter::Trace\n }\n\n /// Converts `self` to the equivalent `Level`.\n ///\n /// Returns `None` if `self` is `LevelFilter::Off`.\n #[inline]\n pub fn to_level(&self) -> Option {\n Level::from_usize(*self as usize)\n }\n\n /// Returns the string representation of the `LevelFilter`.\n ///\n /// This returns the same string as the `fmt::Display` implementation.\n pub fn as_str(&self) -> &'static str {\n LOG_LEVEL_NAMES[*self as usize]\n }\n\n /// Iterate through all supported filtering levels.\n ///\n /// The order of iteration is from less to more verbose filtering.\n ///\n /// # Examples\n ///\n /// ```\n /// use log::LevelFilter;\n ///\n /// let mut levels = LevelFilter::iter();\n ///\n /// assert_eq!(Some(LevelFilter::Off), levels.next());\n /// assert_eq!(Some(LevelFilter::Trace), levels.last());\n /// ```\n pub fn iter() -> impl Iterator {\n (0..6).map(|i| Self::from_usize(i).unwrap())\n }\n\n /// Get the next-highest `LevelFilter` from this one.\n ///\n /// If the current `LevelFilter` is at the highest level, the returned `LevelFilter` will be the\n /// same as the current one.\n ///\n /// # Examples\n ///\n /// ```\n /// use log::LevelFilter;\n ///\n /// let level_filter = LevelFilter::Info;\n ///\n /// assert_eq!(LevelFilter::Debug, level_filter.increment_severity());\n /// assert_eq!(LevelFilter::Trace, level_filter.increment_severity().increment_severity());\n /// assert_eq!(LevelFilter::Trace, level_filter.increment_severity().increment_severity().increment_severity()); // max level\n /// ```\n pub fn increment_severity(&self) -> Self {\n let current = *self as usize;\n Self::from_usize(current + 1).unwrap_or(*self)\n }\n\n /// Get the next-lowest `LevelFilter` from this one.\n ///\n /// If the current `LevelFilter` is at the lowest level, the returned `LevelFilter` will be the\n /// same as the current one.\n ///\n /// # Examples\n ///\n /// ```\n /// use log::LevelFilter;\n ///\n /// let level_filter = LevelFilter::Info;\n ///\n /// assert_eq!(LevelFilter::Warn, level_filter.decrement_severity());\n /// assert_eq!(LevelFilter::Error, level_filter.decrement_severity().decrement_severity());\n /// assert_eq!(LevelFilter::Off, level_filter.decrement_severity().decrement_severity().decrement_severity());\n /// assert_eq!(LevelFilter::Off, level_filter.decrement_severity().decrement_severity().decrement_severity().decrement_severity()); // min level\n /// ```\n pub fn decrement_severity(&self) -> Self {\n let current = *self as usize;\n Self::from_usize(current.saturating_sub(1)).unwrap_or(*self)\n }\n}\n\n#[derive(Copy, Clone, Debug)]\nenum MaybeStaticStr<'a> {\n Static(&'static str),\n Borrowed(&'a str),\n}\n\nimpl<'a> MaybeStaticStr<'a> {\n #[inline]\n fn get(&self) -> &'a str {\n match *self {\n MaybeStaticStr::Static(s) => s,\n MaybeStaticStr::Borrowed(s) => s,\n }\n }\n}\n\nimpl Eq for MaybeStaticStr<'_> {}\n\nimpl PartialEq for MaybeStaticStr<'_> {\n fn eq(&self, other: &Self) -> bool {\n self.get() == other.get()\n }\n}\n\nimpl Ord for MaybeStaticStr<'_> {\n fn cmp(&self, other: &Self) -> core::cmp::Ordering {\n self.get().cmp(other.get())\n }\n}\n\nimpl PartialOrd for MaybeStaticStr<'_> {\n fn partial_cmp(&self, other: &Self) -> Option {\n Some(self.cmp(other))\n }\n}\n\nimpl std::hash::Hash for MaybeStaticStr<'_> {\n fn hash(&self, state: &mut H) {\n self.get().hash(state);\n }\n}\n\n/// The \"payload\" of a log message.\n///\n/// # Use\n///\n/// `Record` structures are passed as parameters to the [`log`][method.log]\n/// method of the [`Log`] trait. Logger implementors manipulate these\n/// structures in order to display log messages. `Record`s are automatically\n/// created by the [`log!`] macro and so are not seen by log users.\n///\n/// Note that the [`level()`] and [`target()`] accessors are equivalent to\n/// `self.metadata().level()` and `self.metadata().target()` respectively.\n/// These methods are provided as a convenience for users of this structure.\n///\n/// # Example\n///\n/// The following example shows a simple logger that displays the level,\n/// module path, and message of any `Record` that is passed to it.\n///\n/// ```\n/// struct SimpleLogger;\n///\n/// impl log::Log for SimpleLogger {\n/// fn enabled(&self, _metadata: &log::Metadata) -> bool {\n/// true\n/// }\n///\n/// fn log(&self, record: &log::Record) {\n/// if !self.enabled(record.metadata()) {\n/// return;\n/// }\n///\n/// println!(\"{}:{} -- {}\",\n/// record.level(),\n/// record.target(),\n/// record.args());\n/// }\n/// fn flush(&self) {}\n/// }\n/// ```\n///\n/// [method.log]: trait.Log.html#tymethod.log\n/// [`Log`]: trait.Log.html\n/// [`log!`]: macro.log.html\n/// [`level()`]: struct.Record.html#method.level\n/// [`target()`]: struct.Record.html#method.target\n#[derive(Clone, Debug)]\npub struct Record<'a> {\n metadata: Metadata<'a>,\n args: fmt::Arguments<'a>,\n module_path: Option>,\n file: Option>,\n line: Option,\n #[cfg(feature = \"kv\")]\n key_values: KeyValues<'a>,\n}\n\n// This wrapper type is only needed so we can\n// `#[derive(Debug)]` on `Record`. It also\n// provides a useful `Debug` implementation for\n// the underlying `Source`.\n#[cfg(feature = \"kv\")]\n#[derive(Clone)]\nstruct KeyValues<'a>(&'a dyn kv::Source);\n\n#[cfg(feature = \"kv\")]\nimpl<'a> fmt::Debug for KeyValues<'a> {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n let mut visitor = f.debug_map();\n self.0.visit(&mut visitor).map_err(|_| fmt::Error)?;\n visitor.finish()\n }\n}\n\nimpl<'a> Record<'a> {\n /// Returns a new builder.\n #[inline]\n pub fn builder() -> RecordBuilder<'a> {\n RecordBuilder::new()\n }\n\n /// The message body.\n #[inline]\n pub fn args(&self) -> &fmt::Arguments<'a> {\n &self.args\n }\n\n /// Metadata about the log directive.\n #[inline]\n pub fn metadata(&self) -> &Metadata<'a> {\n &self.metadata\n }\n\n /// The verbosity level of the message.\n #[inline]\n pub fn level(&self) -> Level {\n self.metadata.level()\n }\n\n /// The name of the target of the directive.\n #[inline]\n pub fn target(&self) -> &'a str {\n self.metadata.target()\n }\n\n /// The module path of the message.\n #[inline]\n pub fn module_path(&self) -> Option<&'a str> {\n self.module_path.map(|s| s.get())\n }\n\n /// The module path of the message, if it is a `'static` string.\n #[inline]\n pub fn module_path_static(&self) -> Option<&'static str> {\n match self.module_path {\n Some(MaybeStaticStr::Static(s)) => Some(s),\n _ => None,\n }\n }\n\n /// The source file containing the message.\n #[inline]\n pub fn file(&self) -> Option<&'a str> {\n self.file.map(|s| s.get())\n }\n\n /// The source file containing the message, if it is a `'static` string.\n #[inline]\n pub fn file_static(&self) -> Option<&'static str> {\n match self.file {\n Some(MaybeStaticStr::Static(s)) => Some(s),\n _ => None,\n }\n }\n\n /// The line containing the message.\n #[inline]\n pub fn line(&self) -> Option {\n self.line\n }\n\n /// The structured key-value pairs associated with the message.\n #[cfg(feature = \"kv\")]\n #[inline]\n pub fn key_values(&self) -> &dyn kv::Source {\n self.key_values.0\n }\n\n /// Create a new [`RecordBuilder`](struct.RecordBuilder.html) based on this record.\n #[cfg(feature = \"kv\")]\n #[inline]\n pub fn to_builder(&self) -> RecordBuilder<'_> {\n RecordBuilder {\n record: Record {\n metadata: Metadata {\n level: self.metadata.level,\n target: self.metadata.target,\n },\n args: self.args,\n module_path: self.module_path,\n file: self.file,\n line: self.line,\n key_values: self.key_values.clone(),\n },\n }\n }\n}\n\n/// Builder for [`Record`](struct.Record.html).\n///\n/// Typically should only be used by log library creators or for testing and \"shim loggers\".\n/// The `RecordBuilder` can set the different parameters of `Record` object, and returns\n/// the created object when `build` is called.\n///\n/// # Examples\n///\n/// ```\n/// use log::{Level, Record};\n///\n/// let record = Record::builder()\n/// .args(format_args!(\"Error!\"))\n/// .level(Level::Error)\n/// .target(\"myApp\")\n/// .file(Some(\"server.rs\"))\n/// .line(Some(144))\n/// .module_path(Some(\"server\"))\n/// .build();\n/// ```\n///\n/// Alternatively, use [`MetadataBuilder`](struct.MetadataBuilder.html):\n///\n/// ```\n/// use log::{Record, Level, MetadataBuilder};\n///\n/// let error_metadata = MetadataBuilder::new()\n/// .target(\"myApp\")\n/// .level(Level::Error)\n/// .build();\n///\n/// let record = Record::builder()\n/// .metadata(error_metadata)\n/// .args(format_args!(\"Error!\"))\n/// .line(Some(433))\n/// .file(Some(\"app.rs\"))\n/// .module_path(Some(\"server\"))\n/// .build();\n/// ```\n#[derive(Debug)]\npub struct RecordBuilder<'a> {\n record: Record<'a>,\n}\n\nimpl<'a> RecordBuilder<'a> {\n /// Construct new `RecordBuilder`.\n ///\n /// The default options are:\n ///\n /// - `args`: [`format_args!(\"\")`]\n /// - `metadata`: [`Metadata::builder().build()`]\n /// - `module_path`: `None`\n /// - `file`: `None`\n /// - `line`: `None`\n ///\n /// [`format_args!(\"\")`]: https://doc.rust-lang.org/std/macro.format_args.html\n /// [`Metadata::builder().build()`]: struct.MetadataBuilder.html#method.build\n #[inline]\n pub fn new() -> RecordBuilder<'a> {\n RecordBuilder {\n record: Record {\n args: format_args!(\"\"),\n metadata: Metadata::builder().build(),\n module_path: None,\n file: None,\n line: None,\n #[cfg(feature = \"kv\")]\n key_values: KeyValues(&None::<(kv::Key, kv::Value)>),\n },\n }\n }\n\n /// Set [`args`](struct.Record.html#method.args).\n #[inline]\n pub fn args(&mut self, args: fmt::Arguments<'a>) -> &mut RecordBuilder<'a> {\n self.record.args = args;\n self\n }\n\n /// Set [`metadata`](struct.Record.html#method.metadata). Construct a `Metadata` object with [`MetadataBuilder`](struct.MetadataBuilder.html).\n #[inline]\n pub fn metadata(&mut self, metadata: Metadata<'a>) -> &mut RecordBuilder<'a> {\n self.record.metadata = metadata;\n self\n }\n\n /// Set [`Metadata::level`](struct.Metadata.html#method.level).\n #[inline]\n pub fn level(&mut self, level: Level) -> &mut RecordBuilder<'a> {\n self.record.metadata.level = level;\n self\n }\n\n /// Set [`Metadata::target`](struct.Metadata.html#method.target)\n #[inline]\n pub fn target(&mut self, target: &'a str) -> &mut RecordBuilder<'a> {\n self.record.metadata.target = target;\n self\n }\n\n /// Set [`module_path`](struct.Record.html#method.module_path)\n #[inline]\n pub fn module_path(&mut self, path: Option<&'a str>) -> &mut RecordBuilder<'a> {\n self.record.module_path = path.map(MaybeStaticStr::Borrowed);\n self\n }\n\n /// Set [`module_path`](struct.Record.html#method.module_path) to a `'static` string\n #[inline]\n pub fn module_path_static(&mut self, path: Option<&'static str>) -> &mut RecordBuilder<'a> {\n self.record.module_path = path.map(MaybeStaticStr::Static);\n self\n }\n\n /// Set [`file`](struct.Record.html#method.file)\n #[inline]\n pub fn file(&mut self, file: Option<&'a str>) -> &mut RecordBuilder<'a> {\n self.record.file = file.map(MaybeStaticStr::Borrowed);\n self\n }\n\n /// Set [`file`](struct.Record.html#method.file) to a `'static` string.\n #[inline]\n pub fn file_static(&mut self, file: Option<&'static str>) -> &mut RecordBuilder<'a> {\n self.record.file = file.map(MaybeStaticStr::Static);\n self\n }\n\n /// Set [`line`](struct.Record.html#method.line)\n #[inline]\n pub fn line(&mut self, line: Option) -> &mut RecordBuilder<'a> {\n self.record.line = line;\n self\n }\n\n /// Set [`key_values`](struct.Record.html#method.key_values)\n #[cfg(feature = \"kv\")]\n #[inline]\n pub fn key_values(&mut self, kvs: &'a dyn kv::Source) -> &mut RecordBuilder<'a> {\n self.record.key_values = KeyValues(kvs);\n self\n }\n\n /// Invoke the builder and return a `Record`\n #[inline]\n pub fn build(&self) -> Record<'a> {\n self.record.clone()\n }\n}\n\nimpl Default for RecordBuilder<'_> {\n fn default() -> Self {\n Self::new()\n }\n}\n\n/// Metadata about a log message.\n///\n/// # Use\n///\n/// `Metadata` structs are created when users of the library use\n/// logging macros.\n///\n/// They are consumed by implementations of the `Log` trait in the\n/// `enabled` method.\n///\n/// `Record`s use `Metadata` to determine the log message's severity\n/// and target.\n///\n/// Users should use the `log_enabled!` macro in their code to avoid\n/// constructing expensive log messages.\n///\n/// # Examples\n///\n/// ```\n/// use log::{Record, Level, Metadata};\n///\n/// struct MyLogger;\n///\n/// impl log::Log for MyLogger {\n/// fn enabled(&self, metadata: &Metadata) -> bool {\n/// metadata.level() <= Level::Info\n/// }\n///\n/// fn log(&self, record: &Record) {\n/// if self.enabled(record.metadata()) {\n/// println!(\"{} - {}\", record.level(), record.args());\n/// }\n/// }\n/// fn flush(&self) {}\n/// }\n///\n/// # fn main(){}\n/// ```\n#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)]\npub struct Metadata<'a> {\n level: Level,\n target: &'a str,\n}\n\nimpl<'a> Metadata<'a> {\n /// Returns a new builder.\n #[inline]\n pub fn builder() -> MetadataBuilder<'a> {\n MetadataBuilder::new()\n }\n\n /// The verbosity level of the message.\n #[inline]\n pub fn level(&self) -> Level {\n self.level\n }\n\n /// The name of the target of the directive.\n #[inline]\n pub fn target(&self) -> &'a str {\n self.target\n }\n}\n\n/// Builder for [`Metadata`](struct.Metadata.html).\n///\n/// Typically should only be used by log library creators or for testing and \"shim loggers\".\n/// The `MetadataBuilder` can set the different parameters of a `Metadata` object, and returns\n/// the created object when `build` is called.\n///\n/// # Example\n///\n/// ```\n/// let target = \"myApp\";\n/// use log::{Level, MetadataBuilder};\n/// let metadata = MetadataBuilder::new()\n/// .level(Level::Debug)\n/// .target(target)\n/// .build();\n/// ```\n#[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Debug)]\npub struct MetadataBuilder<'a> {\n metadata: Metadata<'a>,\n}\n\nimpl<'a> MetadataBuilder<'a> {\n /// Construct a new `MetadataBuilder`.\n ///\n /// The default options are:\n ///\n /// - `level`: `Level::Info`\n /// - `target`: `\"\"`\n #[inline]\n pub fn new() -> MetadataBuilder<'a> {\n MetadataBuilder {\n metadata: Metadata {\n level: Level::Info,\n target: \"\",\n },\n }\n }\n\n /// Setter for [`level`](struct.Metadata.html#method.level).\n #[inline]\n pub fn level(&mut self, arg: Level) -> &mut MetadataBuilder<'a> {\n self.metadata.level = arg;\n self\n }\n\n /// Setter for [`target`](struct.Metadata.html#method.target).\n #[inline]\n pub fn target(&mut self, target: &'a str) -> &mut MetadataBuilder<'a> {\n self.metadata.target = target;\n self\n }\n\n /// Returns a `Metadata` object.\n #[inline]\n pub fn build(&self) -> Metadata<'a> {\n self.metadata.clone()\n }\n}\n\nimpl Default for MetadataBuilder<'_> {\n fn default() -> Self {\n Self::new()\n }\n}\n\n/// A trait encapsulating the operations required of a logger.\npub trait Log: Sync + Send {\n /// Determines if a log message with the specified metadata would be\n /// logged.\n ///\n /// This is used by the `log_enabled!` macro to allow callers to avoid\n /// expensive computation of log message arguments if the message would be\n /// discarded anyway.\n ///\n /// # For implementors\n ///\n /// This method isn't called automatically by the `log!` macros.\n /// It's up to an implementation of the `Log` trait to call `enabled` in its own\n /// `log` method implementation to guarantee that filtering is applied.\n fn enabled(&self, metadata: &Metadata) -> bool;\n\n /// Logs the `Record`.\n ///\n /// # For implementors\n ///\n /// Note that `enabled` is *not* necessarily called before this method.\n /// Implementations of `log` should perform all necessary filtering\n /// internally.\n fn log(&self, record: &Record);\n\n /// Flushes any buffered records.\n ///\n /// # For implementors\n ///\n /// This method isn't called automatically by the `log!` macros.\n /// It can be called manually on shut-down to ensure any in-flight records are flushed.\n fn flush(&self);\n}\n\n/// A dummy initial value for LOGGER.\nstruct NopLogger;\n\nimpl Log for NopLogger {\n fn enabled(&self, _: &Metadata) -> bool {\n false\n }\n\n fn log(&self, _: &Record) {}\n fn flush(&self) {}\n}\n\nimpl Log for &'_ T\nwhere\n T: ?Sized + Log,\n{\n fn enabled(&self, metadata: &Metadata) -> bool {\n (**self).enabled(metadata)\n }\n\n fn log(&self, record: &Record) {\n (**self).log(record);\n }\n fn flush(&self) {\n (**self).flush();\n }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl Log for Box\nwhere\n T: ?Sized + Log,\n{\n fn enabled(&self, metadata: &Metadata) -> bool {\n self.as_ref().enabled(metadata)\n }\n\n fn log(&self, record: &Record) {\n self.as_ref().log(record);\n }\n fn flush(&self) {\n self.as_ref().flush();\n }\n}\n\n#[cfg(feature = \"std\")]\nimpl Log for std::sync::Arc\nwhere\n T: ?Sized + Log,\n{\n fn enabled(&self, metadata: &Metadata) -> bool {\n self.as_ref().enabled(metadata)\n }\n\n fn log(&self, record: &Record) {\n self.as_ref().log(record);\n }\n fn flush(&self) {\n self.as_ref().flush();\n }\n}\n\n/// Sets the global maximum log level.\n///\n/// Generally, this should only be called by the active logging implementation.\n///\n/// Note that `Trace` is the maximum level, because it provides the maximum amount of detail in the emitted logs.\n#[inline]\n#[cfg(target_has_atomic = \"ptr\")]\npub fn set_max_level(level: LevelFilter) {\n MAX_LOG_LEVEL_FILTER.store(level as usize, Ordering::Relaxed);\n}\n\n/// A thread-unsafe version of [`set_max_level`].\n///\n/// This function is available on all platforms, even those that do not have\n/// support for atomics that is needed by [`set_max_level`].\n///\n/// In almost all cases, [`set_max_level`] should be preferred.\n///\n/// # Safety\n///\n/// This function is only safe to call when it cannot race with any other\n/// calls to `set_max_level` or `set_max_level_racy`.\n///\n/// This can be upheld by (for example) making sure that **there are no other\n/// threads**, and (on embedded) that **interrupts are disabled**.\n///\n/// It is safe to use all other logging functions while this function runs\n/// (including all logging macros).\n///\n/// [`set_max_level`]: fn.set_max_level.html\n#[inline]\npub unsafe fn set_max_level_racy(level: LevelFilter) {\n // `MAX_LOG_LEVEL_FILTER` uses a `Cell` as the underlying primitive when a\n // platform doesn't support `target_has_atomic = \"ptr\"`, so even though this looks the same\n // as `set_max_level` it may have different safety properties.\n MAX_LOG_LEVEL_FILTER.store(level as usize, Ordering::Relaxed);\n}\n\n/// Returns the current maximum log level.\n///\n/// The [`log!`], [`error!`], [`warn!`], [`info!`], [`debug!`], and [`trace!`] macros check\n/// this value and discard any message logged at a higher level. The maximum\n/// log level is set by the [`set_max_level`] function.\n///\n/// [`log!`]: macro.log.html\n/// [`error!`]: macro.error.html\n/// [`warn!`]: macro.warn.html\n/// [`info!`]: macro.info.html\n/// [`debug!`]: macro.debug.html\n/// [`trace!`]: macro.trace.html\n/// [`set_max_level`]: fn.set_max_level.html\n#[inline(always)]\npub fn max_level() -> LevelFilter {\n // Since `LevelFilter` is `repr(usize)`,\n // this transmute is sound if and only if `MAX_LOG_LEVEL_FILTER`\n // is set to a usize that is a valid discriminant for `LevelFilter`.\n // Since `MAX_LOG_LEVEL_FILTER` is private, the only time it's set\n // is by `set_max_level` above, i.e. by casting a `LevelFilter` to `usize`.\n // So any usize stored in `MAX_LOG_LEVEL_FILTER` is a valid discriminant.\n unsafe { mem::transmute(MAX_LOG_LEVEL_FILTER.load(Ordering::Relaxed)) }\n}\n\n/// Sets the global logger to a `Box`.\n///\n/// This is a simple convenience wrapper over `set_logger`, which takes a\n/// `Box` rather than a `&'static Log`. See the documentation for\n/// [`set_logger`] for more details.\n///\n/// Requires the `alloc` feature.\n///\n/// # Errors\n///\n/// An error is returned if a logger has already been set.\n///\n/// [`set_logger`]: fn.set_logger.html\n#[cfg(all(feature = \"alloc\", target_has_atomic = \"ptr\"))]\npub fn set_boxed_logger(logger: Box) -> Result<(), SetLoggerError> {\n set_logger_inner(|| Box::leak(logger))\n}\n\n/// Sets the global logger to a `&'static Log`.\n///\n/// This function may only be called once in the lifetime of a program. Any log\n/// events that occur before the call to `set_logger` completes will be ignored.\n///\n/// This function does not typically need to be called manually. Logger\n/// implementations should provide an initialization method that installs the\n/// logger internally.\n///\n/// # Availability\n///\n/// This method is available even when the `std` feature is disabled. However,\n/// it is currently unavailable on `thumbv6` targets, which lack support for\n/// some atomic operations which are used by this function. Even on those\n/// targets, [`set_logger_racy`] will be available.\n///\n/// # Errors\n///\n/// An error is returned if a logger has already been set.\n///\n/// # Examples\n///\n/// ```\n/// use log::{error, info, warn, Record, Level, Metadata, LevelFilter};\n///\n/// static MY_LOGGER: MyLogger = MyLogger;\n///\n/// struct MyLogger;\n///\n/// impl log::Log for MyLogger {\n/// fn enabled(&self, metadata: &Metadata) -> bool {\n/// metadata.level() <= Level::Info\n/// }\n///\n/// fn log(&self, record: &Record) {\n/// if self.enabled(record.metadata()) {\n/// println!(\"{} - {}\", record.level(), record.args());\n/// }\n/// }\n/// fn flush(&self) {}\n/// }\n///\n/// # fn main(){\n/// log::set_logger(&MY_LOGGER).unwrap();\n/// log::set_max_level(LevelFilter::Info);\n///\n/// info!(\"hello log\");\n/// warn!(\"warning\");\n/// error!(\"oops\");\n/// # }\n/// ```\n///\n/// [`set_logger_racy`]: fn.set_logger_racy.html\n#[cfg(target_has_atomic = \"ptr\")]\npub fn set_logger(logger: &'static dyn Log) -> Result<(), SetLoggerError> {\n set_logger_inner(|| logger)\n}\n\n#[cfg(target_has_atomic = \"ptr\")]\nfn set_logger_inner(make_logger: F) -> Result<(), SetLoggerError>\nwhere\n F: FnOnce() -> &'static dyn Log,\n{\n match STATE.compare_exchange(\n UNINITIALIZED,\n INITIALIZING,\n Ordering::Acquire,\n Ordering::Relaxed,\n ) {\n Ok(UNINITIALIZED) => {\n unsafe {\n LOGGER = make_logger();\n }\n STATE.store(INITIALIZED, Ordering::Release);\n Ok(())\n }\n Err(INITIALIZING) => {\n while STATE.load(Ordering::Relaxed) == INITIALIZING {\n std::hint::spin_loop();\n }\n Err(SetLoggerError(()))\n }\n _ => Err(SetLoggerError(())),\n }\n}\n\n/// A thread-unsafe version of [`set_logger`].\n///\n/// This function is available on all platforms, even those that do not have\n/// support for atomics that is needed by [`set_logger`].\n///\n/// In almost all cases, [`set_logger`] should be preferred.\n///\n/// # Safety\n///\n/// This function is only safe to call when it cannot race with any other\n/// calls to `set_logger` or `set_logger_racy`.\n///\n/// This can be upheld by (for example) making sure that **there are no other\n/// threads**, and (on embedded) that **interrupts are disabled**.\n///\n/// It is safe to use other logging functions while this function runs\n/// (including all logging macros).\n///\n/// [`set_logger`]: fn.set_logger.html\npub unsafe fn set_logger_racy(logger: &'static dyn Log) -> Result<(), SetLoggerError> {\n match STATE.load(Ordering::Acquire) {\n UNINITIALIZED => {\n LOGGER = logger;\n STATE.store(INITIALIZED, Ordering::Release);\n Ok(())\n }\n INITIALIZING => {\n // This is just plain UB, since we were racing another initialization function\n unreachable!(\"set_logger_racy must not be used with other initialization functions\")\n }\n _ => Err(SetLoggerError(())),\n }\n}\n\n/// The type returned by [`set_logger`] if [`set_logger`] has already been called.\n///\n/// [`set_logger`]: fn.set_logger.html\n#[allow(missing_copy_implementations)]\n#[derive(Debug)]\npub struct SetLoggerError(());\n\nimpl fmt::Display for SetLoggerError {\n fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {\n fmt.write_str(SET_LOGGER_ERROR)\n }\n}\n\n// The Error trait is not available in libcore\n#[cfg(feature = \"std\")]\nimpl error::Error for SetLoggerError {}\n\n/// The type returned by [`from_str`] when the string doesn't match any of the log levels.\n///\n/// [`from_str`]: https://doc.rust-lang.org/std/str/trait.FromStr.html#tymethod.from_str\n#[allow(missing_copy_implementations)]\n#[derive(Debug, PartialEq, Eq)]\npub struct ParseLevelError(());\n\nimpl fmt::Display for ParseLevelError {\n fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {\n fmt.write_str(LEVEL_PARSE_ERROR)\n }\n}\n\n// The Error trait is not available in libcore\n#[cfg(feature = \"std\")]\nimpl error::Error for ParseLevelError {}\n\n/// Returns a reference to the logger.\n///\n/// If a logger has not been set, a no-op implementation is returned.\npub fn logger() -> &'static dyn Log {\n // Acquire memory ordering guarantees that current thread would see any\n // memory writes that happened before store of the value\n // into `STATE` with memory ordering `Release` or stronger.\n //\n // Since the value `INITIALIZED` is written only after `LOGGER` was\n // initialized, observing it after `Acquire` load here makes both\n // write to the `LOGGER` static and initialization of the logger\n // internal state synchronized with current thread.\n if STATE.load(Ordering::Acquire) != INITIALIZED {\n static NOP: NopLogger = NopLogger;\n &NOP\n } else {\n unsafe { LOGGER }\n }\n}\n\n// WARNING: this is not part of the crate's public API and is subject to change at any time\n#[doc(hidden)]\npub mod __private_api;\n\n/// The statically resolved maximum log level.\n///\n/// See the crate level documentation for information on how to configure this.\n///\n/// This value is checked by the log macros, but not by the `Log`ger returned by\n/// the [`logger`] function. Code that manually calls functions on that value\n/// should compare the level against this value.\n///\n/// [`logger`]: fn.logger.html\npub const STATIC_MAX_LEVEL: LevelFilter = match cfg!(debug_assertions) {\n false if cfg!(feature = \"release_max_level_off\") => LevelFilter::Off,\n false if cfg!(feature = \"release_max_level_error\") => LevelFilter::Error,\n false if cfg!(feature = \"release_max_level_warn\") => LevelFilter::Warn,\n false if cfg!(feature = \"release_max_level_info\") => LevelFilter::Info,\n false if cfg!(feature = \"release_max_level_debug\") => LevelFilter::Debug,\n false if cfg!(feature = \"release_max_level_trace\") => LevelFilter::Trace,\n _ if cfg!(feature = \"max_level_off\") => LevelFilter::Off,\n _ if cfg!(feature = \"max_level_error\") => LevelFilter::Error,\n _ if cfg!(feature = \"max_level_warn\") => LevelFilter::Warn,\n _ if cfg!(feature = \"max_level_info\") => LevelFilter::Info,\n _ if cfg!(feature = \"max_level_debug\") => LevelFilter::Debug,\n _ => LevelFilter::Trace,\n};\n\n#[cfg(test)]\nmod tests {\n use super::{Level, LevelFilter, ParseLevelError, STATIC_MAX_LEVEL};\n\n #[test]\n fn test_levelfilter_from_str() {\n let tests = [\n (\"off\", Ok(LevelFilter::Off)),\n (\"error\", Ok(LevelFilter::Error)),\n (\"warn\", Ok(LevelFilter::Warn)),\n (\"info\", Ok(LevelFilter::Info)),\n (\"debug\", Ok(LevelFilter::Debug)),\n (\"trace\", Ok(LevelFilter::Trace)),\n (\"OFF\", Ok(LevelFilter::Off)),\n (\"ERROR\", Ok(LevelFilter::Error)),\n (\"WARN\", Ok(LevelFilter::Warn)),\n (\"INFO\", Ok(LevelFilter::Info)),\n (\"DEBUG\", Ok(LevelFilter::Debug)),\n (\"TRACE\", Ok(LevelFilter::Trace)),\n (\"asdf\", Err(ParseLevelError(()))),\n ];\n for &(s, ref expected) in &tests {\n assert_eq!(expected, &s.parse());\n }\n }\n\n #[test]\n fn test_level_from_str() {\n let tests = [\n (\"OFF\", Err(ParseLevelError(()))),\n (\"error\", Ok(Level::Error)),\n (\"warn\", Ok(Level::Warn)),\n (\"info\", Ok(Level::Info)),\n (\"debug\", Ok(Level::Debug)),\n (\"trace\", Ok(Level::Trace)),\n (\"ERROR\", Ok(Level::Error)),\n (\"WARN\", Ok(Level::Warn)),\n (\"INFO\", Ok(Level::Info)),\n (\"DEBUG\", Ok(Level::Debug)),\n (\"TRACE\", Ok(Level::Trace)),\n (\"asdf\", Err(ParseLevelError(()))),\n ];\n for &(s, ref expected) in &tests {\n assert_eq!(expected, &s.parse());\n }\n }\n\n #[test]\n fn test_level_as_str() {\n let tests = &[\n (Level::Error, \"ERROR\"),\n (Level::Warn, \"WARN\"),\n (Level::Info, \"INFO\"),\n (Level::Debug, \"DEBUG\"),\n (Level::Trace, \"TRACE\"),\n ];\n for (input, expected) in tests {\n assert_eq!(*expected, input.as_str());\n }\n }\n\n #[test]\n fn test_level_show() {\n assert_eq!(\"INFO\", Level::Info.to_string());\n assert_eq!(\"ERROR\", Level::Error.to_string());\n }\n\n #[test]\n fn test_levelfilter_show() {\n assert_eq!(\"OFF\", LevelFilter::Off.to_string());\n assert_eq!(\"ERROR\", LevelFilter::Error.to_string());\n }\n\n #[test]\n fn test_cross_cmp() {\n assert!(Level::Debug > LevelFilter::Error);\n assert!(LevelFilter::Warn < Level::Trace);\n assert!(LevelFilter::Off < Level::Error);\n }\n\n #[test]\n fn test_cross_eq() {\n assert!(Level::Error == LevelFilter::Error);\n assert!(LevelFilter::Off != Level::Error);\n assert!(Level::Trace == LevelFilter::Trace);\n }\n\n #[test]\n fn test_to_level() {\n assert_eq!(Some(Level::Error), LevelFilter::Error.to_level());\n assert_eq!(None, LevelFilter::Off.to_level());\n assert_eq!(Some(Level::Debug), LevelFilter::Debug.to_level());\n }\n\n #[test]\n fn test_to_level_filter() {\n assert_eq!(LevelFilter::Error, Level::Error.to_level_filter());\n assert_eq!(LevelFilter::Trace, Level::Trace.to_level_filter());\n }\n\n #[test]\n fn test_level_filter_as_str() {\n let tests = &[\n (LevelFilter::Off, \"OFF\"),\n (LevelFilter::Error, \"ERROR\"),\n (LevelFilter::Warn, \"WARN\"),\n (LevelFilter::Info, \"INFO\"),\n (LevelFilter::Debug, \"DEBUG\"),\n (LevelFilter::Trace, \"TRACE\"),\n ];\n for (input, expected) in tests {\n assert_eq!(*expected, input.as_str());\n }\n }\n\n #[test]\n fn test_level_up() {\n let info = Level::Info;\n let up = info.increment_severity();\n assert_eq!(up, Level::Debug);\n\n let trace = Level::Trace;\n let up = trace.increment_severity();\n // trace is already highest level\n assert_eq!(up, trace);\n }\n\n #[test]\n fn test_level_filter_up() {\n let info = LevelFilter::Info;\n let up = info.increment_severity();\n assert_eq!(up, LevelFilter::Debug);\n\n let trace = LevelFilter::Trace;\n let up = trace.increment_severity();\n // trace is already highest level\n assert_eq!(up, trace);\n }\n\n #[test]\n fn test_level_down() {\n let info = Level::Info;\n let down = info.decrement_severity();\n assert_eq!(down, Level::Warn);\n\n let error = Level::Error;\n let down = error.decrement_severity();\n // error is already lowest level\n assert_eq!(down, error);\n }\n\n #[test]\n fn test_level_filter_down() {\n let info = LevelFilter::Info;\n let down = info.decrement_severity();\n assert_eq!(down, LevelFilter::Warn);\n\n let error = LevelFilter::Error;\n let down = error.decrement_severity();\n assert_eq!(down, LevelFilter::Off);\n // Off is already the lowest\n assert_eq!(down.decrement_severity(), down);\n }\n\n #[test]\n #[cfg_attr(not(debug_assertions), ignore)]\n fn test_static_max_level_debug() {\n if cfg!(feature = \"max_level_off\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Off);\n } else if cfg!(feature = \"max_level_error\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Error);\n } else if cfg!(feature = \"max_level_warn\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Warn);\n } else if cfg!(feature = \"max_level_info\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Info);\n } else if cfg!(feature = \"max_level_debug\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Debug);\n } else {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Trace);\n }\n }\n\n #[test]\n #[cfg_attr(debug_assertions, ignore)]\n fn test_static_max_level_release() {\n if cfg!(feature = \"release_max_level_off\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Off);\n } else if cfg!(feature = \"release_max_level_error\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Error);\n } else if cfg!(feature = \"release_max_level_warn\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Warn);\n } else if cfg!(feature = \"release_max_level_info\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Info);\n } else if cfg!(feature = \"release_max_level_debug\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Debug);\n } else if cfg!(feature = \"release_max_level_trace\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Trace);\n } else if cfg!(feature = \"max_level_off\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Off);\n } else if cfg!(feature = \"max_level_error\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Error);\n } else if cfg!(feature = \"max_level_warn\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Warn);\n } else if cfg!(feature = \"max_level_info\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Info);\n } else if cfg!(feature = \"max_level_debug\") {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Debug);\n } else {\n assert_eq!(STATIC_MAX_LEVEL, LevelFilter::Trace);\n }\n }\n\n #[test]\n #[cfg(feature = \"std\")]\n fn test_error_trait() {\n use super::SetLoggerError;\n let e = SetLoggerError(());\n assert_eq!(\n &e.to_string(),\n \"attempted to set a logger after the logging system \\\n was already initialized\"\n );\n }\n\n #[test]\n fn test_metadata_builder() {\n use super::MetadataBuilder;\n let target = \"myApp\";\n let metadata_test = MetadataBuilder::new()\n .level(Level::Debug)\n .target(target)\n .build();\n assert_eq!(metadata_test.level(), Level::Debug);\n assert_eq!(metadata_test.target(), \"myApp\");\n }\n\n #[test]\n fn test_metadata_convenience_builder() {\n use super::Metadata;\n let target = \"myApp\";\n let metadata_test = Metadata::builder()\n .level(Level::Debug)\n .target(target)\n .build();\n assert_eq!(metadata_test.level(), Level::Debug);\n assert_eq!(metadata_test.target(), \"myApp\");\n }\n\n #[test]\n fn test_record_builder() {\n use super::{MetadataBuilder, RecordBuilder};\n let target = \"myApp\";\n let metadata = MetadataBuilder::new().target(target).build();\n let fmt_args = format_args!(\"hello\");\n let record_test = RecordBuilder::new()\n .args(fmt_args)\n .metadata(metadata)\n .module_path(Some(\"foo\"))\n .file(Some(\"bar\"))\n .line(Some(30))\n .build();\n assert_eq!(record_test.metadata().target(), \"myApp\");\n assert_eq!(record_test.module_path(), Some(\"foo\"));\n assert_eq!(record_test.file(), Some(\"bar\"));\n assert_eq!(record_test.line(), Some(30));\n }\n\n #[test]\n fn test_record_convenience_builder() {\n use super::{Metadata, Record};\n let target = \"myApp\";\n let metadata = Metadata::builder().target(target).build();\n let fmt_args = format_args!(\"hello\");\n let record_test = Record::builder()\n .args(fmt_args)\n .metadata(metadata)\n .module_path(Some(\"foo\"))\n .file(Some(\"bar\"))\n .line(Some(30))\n .build();\n assert_eq!(record_test.target(), \"myApp\");\n assert_eq!(record_test.module_path(), Some(\"foo\"));\n assert_eq!(record_test.file(), Some(\"bar\"));\n assert_eq!(record_test.line(), Some(30));\n }\n\n #[test]\n fn test_record_complete_builder() {\n use super::{Level, Record};\n let target = \"myApp\";\n let record_test = Record::builder()\n .module_path(Some(\"foo\"))\n .file(Some(\"bar\"))\n .line(Some(30))\n .target(target)\n .level(Level::Error)\n .build();\n assert_eq!(record_test.target(), \"myApp\");\n assert_eq!(record_test.level(), Level::Error);\n assert_eq!(record_test.module_path(), Some(\"foo\"));\n assert_eq!(record_test.file(), Some(\"bar\"));\n assert_eq!(record_test.line(), Some(30));\n }\n\n #[test]\n #[cfg(feature = \"kv\")]\n fn test_record_key_values_builder() {\n use super::Record;\n use crate::kv::{self, VisitSource};\n\n struct TestVisitSource {\n seen_pairs: usize,\n }\n\n impl<'kvs> VisitSource<'kvs> for TestVisitSource {\n fn visit_pair(\n &mut self,\n _: kv::Key<'kvs>,\n _: kv::Value<'kvs>,\n ) -> Result<(), kv::Error> {\n self.seen_pairs += 1;\n Ok(())\n }\n }\n\n let kvs: &[(&str, i32)] = &[(\"a\", 1), (\"b\", 2)];\n let record_test = Record::builder().key_values(&kvs).build();\n\n let mut visitor = TestVisitSource { seen_pairs: 0 };\n\n record_test.key_values().visit(&mut visitor).unwrap();\n\n assert_eq!(2, visitor.seen_pairs);\n }\n\n #[test]\n #[cfg(feature = \"kv\")]\n fn test_record_key_values_get_coerce() {\n use super::Record;\n\n let kvs: &[(&str, &str)] = &[(\"a\", \"1\"), (\"b\", \"2\")];\n let record = Record::builder().key_values(&kvs).build();\n\n assert_eq!(\n \"2\",\n record\n .key_values()\n .get(\"b\".into())\n .expect(\"missing key\")\n .to_borrowed_str()\n .expect(\"invalid value\")\n );\n }\n\n // Test that the `impl Log for Foo` blocks work\n // This test mostly operates on a type level, so failures will be compile errors\n #[test]\n fn test_foreign_impl() {\n use super::Log;\n #[cfg(feature = \"std\")]\n use std::sync::Arc;\n\n fn assert_is_log() {}\n\n assert_is_log::<&dyn Log>();\n\n #[cfg(feature = \"alloc\")]\n assert_is_log::>();\n\n #[cfg(feature = \"std\")]\n assert_is_log::>();\n\n // Assert these statements for all T: Log + ?Sized\n #[allow(unused)]\n fn forall() {\n #[cfg(feature = \"alloc\")]\n assert_is_log::>();\n\n assert_is_log::<&T>();\n\n #[cfg(feature = \"std\")]\n assert_is_log::>();\n }\n }\n}\n" }, { "package": "log@0.4.34", "sourcePath": "src/macros.rs", "sha256": "6917080cd9c4a2ca8128024f562a2bfdbbcd43616e41ef125289a30c6a9a4be2", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n/// The standard logging macro.\n///\n/// This macro will generically log with the specified `Level` and `format!`\n/// based argument list.\n///\n/// ```\n/// use log::{log, Level};\n///\n/// let data = (42, \"Forty-two\");\n/// let private_data = \"private\";\n///\n/// log!(Level::Error, \"Received errors: {}, {}\", data.0, data.1);\n/// ```\n///\n/// Optionally, you can specify a `target` argument to attach a specific target\n/// to the log record. By default, the target is the module path of the caller.\n///\n/// ```\n/// use log::{log, Level};\n///\n/// let data = (42, \"Forty-two\");\n/// let private_data = \"private\";\n///\n/// log!(\n/// target: \"app_events\",\n/// Level::Error,\n/// \"Received errors: {}, {}\",\n/// data.0, data.1\n/// );\n/// ```\n///\n/// And optionally, you can specify a `logger` argument to use a specific logger\n/// instead of the default global logger.\n///\n/// ```\n/// # struct MyLogger {}\n/// # impl Log for MyLogger {\n/// # fn enabled(&self, _metadata: &log::Metadata) -> bool {\n/// # false\n/// # }\n/// # fn log(&self, _record: &log::Record) {}\n/// # fn flush(&self) {}\n/// # }\n/// use log::{log, Level, Log};\n///\n/// let data = (42, \"Forty-two\");\n/// let private_data = \"private\";\n///\n/// let my_logger = MyLogger {};\n/// log!(\n/// logger: my_logger,\n/// Level::Error,\n/// \"Received errors: {}, {}\",\n/// data.0, data.1\n/// );\n/// ```\n///\n/// The `logger` argument accepts a value that implements the `Log` trait. The value\n/// will be borrowed within the macro.\n///\n/// Note that the global level set via Cargo features or through `set_max_level` will\n/// still apply, even when a custom logger is supplied with the `logger` argument.\n#[macro_export]\n#[clippy::format_args]\nmacro_rules! log {\n // log!(logger: my_logger, target: \"my_target\", Level::Info, \"a {} event\", \"log\");\n (logger: $logger:expr, target: $target:expr, $lvl:expr, $($arg:tt)+) => ({\n $crate::__log!(\n logger: $crate::__log_logger!($logger),\n target: $target,\n $lvl,\n $($arg)+\n )\n });\n\n // log!(logger: my_logger, Level::Info, \"a log event\")\n (logger: $logger:expr, $lvl:expr, $($arg:tt)+) => ({\n $crate::__log!(\n logger: $crate::__log_logger!($logger),\n target: $crate::__private_api::module_path!(),\n $lvl,\n $($arg)+\n )\n });\n\n // log!(target: \"my_target\", Level::Info, \"a log event\")\n (target: $target:expr, $lvl:expr, $($arg:tt)+) => ({\n $crate::__log!(\n logger: $crate::__log_logger!(__log_global_logger),\n target: $target,\n $lvl,\n $($arg)+\n )\n });\n\n // log!(Level::Info, \"a log event\")\n ($lvl:expr, $($arg:tt)+) => ({\n $crate::__log!(\n logger: $crate::__log_logger!(__log_global_logger),\n target: $crate::__private_api::module_path!(),\n $lvl,\n $($arg)+\n )\n });\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __log {\n // log!(logger: my_logger, target: \"my_target\", Level::Info, key1:? = 42, key2 = true; \"a {} event\", \"log\");\n (logger: $logger:expr, target: $target:expr, $lvl:expr, $($key:tt $(:$capture:tt)? $(= $value:expr)?),+; $($arg:tt)+) => ({\n let lvl = $lvl;\n if lvl <= $crate::STATIC_MAX_LEVEL && lvl <= $crate::max_level() {\n $crate::__private_api::log(\n $logger,\n $crate::__private_api::format_args!($($arg)+),\n lvl,\n &($target, $crate::__private_api::module_path!(), $crate::__private_api::loc()),\n &[$(($crate::__log_key!($key), $crate::__log_value!($key $(:$capture)* = $($value)*))),+] as &[_],\n );\n }\n });\n\n // log!(logger: my_logger, target: \"my_target\", Level::Info, \"a {} event\", \"log\");\n (logger: $logger:expr, target: $target:expr, $lvl:expr, $($arg:tt)+) => ({\n let lvl = $lvl;\n if lvl <= $crate::STATIC_MAX_LEVEL && lvl <= $crate::max_level() {\n $crate::__private_api::log(\n $logger,\n $crate::__private_api::format_args!($($arg)+),\n lvl,\n &($target, $crate::__private_api::module_path!(), $crate::__private_api::loc()),\n (),\n );\n }\n });\n}\n\n/// Logs a message at the error level.\n///\n/// # Examples\n///\n/// ```\n/// use log::error;\n///\n/// # let my_logger = log::__private_api::GlobalLogger;\n/// let (err_info, port) = (\"No connection\", 22);\n///\n/// error!(\"Error: {err_info} on port {port}\");\n/// error!(target: \"app_events\", \"App Error: {err_info}, Port: {port}\");\n/// error!(logger: my_logger, \"App Error: {err_info}, Port: {port}\");\n/// ```\n#[macro_export]\n#[clippy::format_args]\nmacro_rules! error {\n // error!(logger: my_logger, target: \"my_target\", key1 = 42, key2 = true; \"a {} event\", \"log\")\n // error!(logger: my_logger, target: \"my_target\", \"a {} event\", \"log\")\n (logger: $logger:expr, target: $target:expr, $($arg:tt)+) => ({\n $crate::log!(logger: $crate::__log_logger!($logger), target: $target, $crate::Level::Error, $($arg)+)\n });\n\n // error!(logger: my_logger, key1 = 42, key2 = true; \"a {} event\", \"log\")\n // error!(logger: my_logger, \"a {} event\", \"log\")\n (logger: $logger:expr, $($arg:tt)+) => ({\n $crate::log!(logger: $crate::__log_logger!($logger), $crate::Level::Error, $($arg)+)\n });\n\n // error!(target: \"my_target\", key1 = 42, key2 = true; \"a {} event\", \"log\")\n // error!(target: \"my_target\", \"a {} event\", \"log\")\n (target: $target:expr, $($arg:tt)+) => ({\n $crate::log!(target: $target, $crate::Level::Error, $($arg)+)\n });\n\n // error!(\"a {} event\", \"log\")\n ($($arg:tt)+) => ($crate::log!($crate::Level::Error, $($arg)+))\n}\n\n/// Logs a message at the warn level.\n///\n/// # Examples\n///\n/// ```\n/// use log::warn;\n///\n/// # let my_logger = log::__private_api::GlobalLogger;\n/// let warn_description = \"Invalid Input\";\n///\n/// warn!(\"Warning! {warn_description}!\");\n/// warn!(target: \"input_events\", \"App received warning: {warn_description}\");\n/// warn!(logger: my_logger, \"App received warning: {warn_description}\");\n/// ```\n#[macro_export]\n#[clippy::format_args]\nmacro_rules! warn {\n // warn!(logger: my_logger, target: \"my_target\", key1 = 42, key2 = true; \"a {} event\", \"log\")\n // warn!(logger: my_logger, target: \"my_target\", \"a {} event\", \"log\")\n (logger: $logger:expr, target: $target:expr, $($arg:tt)+) => ({\n $crate::log!(logger: $crate::__log_logger!($logger), target: $target, $crate::Level::Warn, $($arg)+)\n });\n\n // warn!(logger: my_logger, key1 = 42, key2 = true; \"a {} event\", \"log\")\n // warn!(logger: my_logger, \"a {} event\", \"log\")\n (logger: $logger:expr, $($arg:tt)+) => ({\n $crate::log!(logger: $crate::__log_logger!($logger), $crate::Level::Warn, $($arg)+)\n });\n\n // warn!(target: \"my_target\", key1 = 42, key2 = true; \"a {} event\", \"log\")\n // warn!(target: \"my_target\", \"a {} event\", \"log\")\n (target: $target:expr, $($arg:tt)+) => ({\n $crate::log!(target: $target, $crate::Level::Warn, $($arg)+)\n });\n\n // warn!(\"a {} event\", \"log\")\n ($($arg:tt)+) => ($crate::log!($crate::Level::Warn, $($arg)+))\n}\n\n/// Logs a message at the info level.\n///\n/// # Examples\n///\n/// ```\n/// use log::info;\n///\n/// # let my_logger = log::__private_api::GlobalLogger;\n/// # struct Connection { port: u32, speed: f32 }\n/// let conn_info = Connection { port: 40, speed: 3.20 };\n///\n/// info!(\"Connected to port {} at {} Mb/s\", conn_info.port, conn_info.speed);\n/// info!(\n/// target: \"connection_events\",\n/// \"Successful connection, port: {}, speed: {}\",\n/// conn_info.port, conn_info.speed\n/// );\n/// info!(\n/// logger: my_logger,\n/// \"Successful connection, port: {}, speed: {}\",\n/// conn_info.port, conn_info.speed\n/// );\n/// ```\n#[macro_export]\n#[clippy::format_args]\nmacro_rules! info {\n // info!(logger: my_logger, target: \"my_target\", key1 = 42, key2 = true; \"a {} event\", \"log\")\n // info!(logger: my_logger, target: \"my_target\", \"a {} event\", \"log\")\n (logger: $logger:expr, target: $target:expr, $($arg:tt)+) => ({\n $crate::log!(logger: $crate::__log_logger!($logger), target: $target, $crate::Level::Info, $($arg)+)\n });\n\n // info!(logger: my_logger, key1 = 42, key2 = true; \"a {} event\", \"log\")\n // info!(logger: my_logger, \"a {} event\", \"log\")\n (logger: $logger:expr, $($arg:tt)+) => ({\n $crate::log!(logger: $crate::__log_logger!($logger), $crate::Level::Info, $($arg)+)\n });\n\n // info!(target: \"my_target\", key1 = 42, key2 = true; \"a {} event\", \"log\")\n // info!(target: \"my_target\", \"a {} event\", \"log\")\n (target: $target:expr, $($arg:tt)+) => ({\n $crate::log!(target: $target, $crate::Level::Info, $($arg)+)\n });\n\n // info!(\"a {} event\", \"log\")\n ($($arg:tt)+) => ($crate::log!($crate::Level::Info, $($arg)+))\n}\n\n/// Logs a message at the debug level.\n///\n/// # Examples\n///\n/// ```\n/// use log::debug;\n///\n/// # let my_logger = log::__private_api::GlobalLogger;\n/// # struct Position { x: f32, y: f32 }\n/// let pos = Position { x: 3.234, y: -1.223 };\n///\n/// debug!(\"New position: x: {}, y: {}\", pos.x, pos.y);\n/// debug!(target: \"app_events\", \"New position: x: {}, y: {}\", pos.x, pos.y);\n/// debug!(logger: my_logger, \"New position: x: {}, y: {}\", pos.x, pos.y);\n/// ```\n#[macro_export]\n#[clippy::format_args]\nmacro_rules! debug {\n // debug!(logger: my_logger, target: \"my_target\", key1 = 42, key2 = true; \"a {} event\", \"log\")\n // debug!(logger: my_logger, target: \"my_target\", \"a {} event\", \"log\")\n (logger: $logger:expr, target: $target:expr, $($arg:tt)+) => ({\n $crate::log!(logger: $crate::__log_logger!($logger), target: $target, $crate::Level::Debug, $($arg)+)\n });\n\n // debug!(logger: my_logger, key1 = 42, key2 = true; \"a {} event\", \"log\")\n // debug!(logger: my_logger, \"a {} event\", \"log\")\n (logger: $logger:expr, $($arg:tt)+) => ({\n $crate::log!(logger: $crate::__log_logger!($logger), $crate::Level::Debug, $($arg)+)\n });\n\n // debug!(target: \"my_target\", key1 = 42, key2 = true; \"a {} event\", \"log\")\n // debug!(target: \"my_target\", \"a {} event\", \"log\")\n (target: $target:expr, $($arg:tt)+) => ({\n $crate::log!(target: $target, $crate::Level::Debug, $($arg)+)\n });\n\n // debug!(\"a {} event\", \"log\")\n ($($arg:tt)+) => ($crate::log!($crate::Level::Debug, $($arg)+))\n}\n\n/// Logs a message at the trace level.\n///\n/// # Examples\n///\n/// ```\n/// use log::trace;\n///\n/// # let my_logger = log::__private_api::GlobalLogger;\n/// # struct Position { x: f32, y: f32 }\n/// let pos = Position { x: 3.234, y: -1.223 };\n///\n/// trace!(\"Position is: x: {}, y: {}\", pos.x, pos.y);\n/// trace!(target: \"app_events\", \"x is {} and y is {}\",\n/// if pos.x >= 0.0 { \"positive\" } else { \"negative\" },\n/// if pos.y >= 0.0 { \"positive\" } else { \"negative\" });\n/// trace!(logger: my_logger, \"x is {} and y is {}\",\n/// if pos.x >= 0.0 { \"positive\" } else { \"negative\" },\n/// if pos.y >= 0.0 { \"positive\" } else { \"negative\" });\n/// ```\n#[macro_export]\n#[clippy::format_args]\nmacro_rules! trace {\n // trace!(logger: my_logger, target: \"my_target\", key1 = 42, key2 = true; \"a {} event\", \"log\")\n // trace!(logger: my_logger, target: \"my_target\", \"a {} event\", \"log\")\n (logger: $logger:expr, target: $target:expr, $($arg:tt)+) => ({\n $crate::log!(logger: $crate::__log_logger!($logger), target: $target, $crate::Level::Trace, $($arg)+)\n });\n\n // trace!(logger: my_logger, key1 = 42, key2 = true; \"a {} event\", \"log\")\n // trace!(logger: my_logger, \"a {} event\", \"log\")\n (logger: $logger:expr, $($arg:tt)+) => ({\n $crate::log!(logger: $crate::__log_logger!($logger), $crate::Level::Trace, $($arg)+)\n });\n\n // trace!(target: \"my_target\", key1 = 42, key2 = true; \"a {} event\", \"log\")\n // trace!(target: \"my_target\", \"a {} event\", \"log\")\n (target: $target:expr, $($arg:tt)+) => ({\n $crate::log!(target: $target, $crate::Level::Trace, $($arg)+)\n });\n\n // trace!(\"a {} event\", \"log\")\n ($($arg:tt)+) => ($crate::log!($crate::Level::Trace, $($arg)+))\n}\n\n/// Determines if a message logged at the specified level in that module will\n/// be logged.\n///\n/// This can be used to avoid expensive computation of log message arguments if\n/// the message would be ignored anyway.\n///\n/// # Examples\n///\n/// ```\n/// use log::{debug, log_enabled, Level};\n///\n/// # struct Data { x: u32, y: u32 }\n/// # fn expensive_call() -> Data { Data { x: 0, y: 0 } }\n/// # let my_logger = log::__private_api::GlobalLogger;\n/// if log_enabled!(Level::Debug) {\n/// let data = expensive_call();\n/// debug!(\"expensive debug data: {} {}\", data.x, data.y);\n/// }\n///\n/// if log_enabled!(target: \"Global\", Level::Debug) {\n/// let data = expensive_call();\n/// debug!(target: \"Global\", \"expensive debug data: {} {}\", data.x, data.y);\n/// }\n///\n/// if log_enabled!(logger: my_logger, Level::Debug) {\n/// let data = expensive_call();\n/// debug!(target: \"Global\", \"expensive debug data: {} {}\", data.x, data.y);\n/// }\n/// ```\n///\n/// This macro accepts the same `target` and `logger` arguments as [`macro@log`].\n#[macro_export]\nmacro_rules! log_enabled {\n // log_enabled!(logger: my_logger, target: \"my_target\", Level::Info)\n (logger: $logger:expr, target: $target:expr, $lvl:expr) => ({\n $crate::__log_enabled!(logger: $crate::__log_logger!($logger), target: $target, $lvl)\n });\n\n // log_enabled!(logger: my_logger, Level::Info)\n (logger: $logger:expr, $lvl:expr) => ({\n $crate::__log_enabled!(logger: $crate::__log_logger!($logger), target: $crate::__private_api::module_path!(), $lvl)\n });\n\n // log_enabled!(target: \"my_target\", Level::Info)\n (target: $target:expr, $lvl:expr) => ({\n $crate::__log_enabled!(logger: $crate::__log_logger!(__log_global_logger), target: $target, $lvl)\n });\n\n // log_enabled!(Level::Info)\n ($lvl:expr) => ({\n $crate::__log_enabled!(logger: $crate::__log_logger!(__log_global_logger), target: $crate::__private_api::module_path!(), $lvl)\n });\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __log_enabled {\n // log_enabled!(logger: my_logger, target: \"my_target\", Level::Info)\n (logger: $logger:expr, target: $target:expr, $lvl:expr) => {{\n let lvl = $lvl;\n lvl <= $crate::STATIC_MAX_LEVEL\n && lvl <= $crate::max_level()\n && $crate::__private_api::enabled($logger, lvl, $target)\n }};\n}\n\n// Determine the logger to use, and whether to take it by-value or by reference\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __log_logger {\n (__log_global_logger) => {{\n $crate::__private_api::GlobalLogger\n }};\n\n ($logger:expr) => {{\n &($logger)\n }};\n}\n\n// These macros use a pattern of #[cfg]s to produce nicer error\n// messages when log features aren't available\n\n#[doc(hidden)]\n#[macro_export]\n#[cfg(feature = \"kv\")]\nmacro_rules! __log_key {\n // key1 = 42\n ($args:ident) => {\n $crate::__private_api::Key::from_str_static($crate::__private_api::stringify!($args))\n };\n // \"key1\" = 42\n ($args:literal) => {\n $crate::__private_api::Key::from_str_static($args)\n };\n // key = 42, where key is an expression\n ($args:expr) => {\n $crate::__private_api::Key::from_str($args)\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\n#[cfg(not(feature = \"kv\"))]\nmacro_rules! __log_key {\n ($($args:tt)*) => {\n compile_error!(\"key value support requires the `kv` feature of `log`\")\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\n#[cfg(feature = \"kv\")]\nmacro_rules! __log_value {\n // Entrypoint\n ($key:tt = $args:expr) => {\n $crate::__log_value!(($args):value)\n };\n ($key:tt :$capture:tt = $args:expr) => {\n $crate::__log_value!(($args):$capture)\n };\n ($key:ident =) => {\n $crate::__log_value!(($key):value)\n };\n ($key:ident :$capture:tt =) => {\n $crate::__log_value!(($key):$capture)\n };\n // ToValue\n (($args:expr):value) => {\n $crate::__private_api::capture_to_value(&&$args)\n };\n // Debug\n (($args:expr):?) => {\n $crate::__private_api::capture_debug(&&$args)\n };\n (($args:expr):debug) => {\n $crate::__private_api::capture_debug(&&$args)\n };\n // Display\n (($args:expr):%) => {\n $crate::__private_api::capture_display(&&$args)\n };\n (($args:expr):display) => {\n $crate::__private_api::capture_display(&&$args)\n };\n //Error\n (($args:expr):err) => {\n $crate::__log_value_error!($args)\n };\n // sval::Value\n (($args:expr):sval) => {\n $crate::__log_value_sval!($args)\n };\n // serde::Serialize\n (($args:expr):serde) => {\n $crate::__log_value_serde!($args)\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\n#[cfg(not(feature = \"kv\"))]\nmacro_rules! __log_value {\n ($($args:tt)*) => {\n compile_error!(\"key value support requires the `kv` feature of `log`\")\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\n#[cfg(feature = \"kv_sval\")]\nmacro_rules! __log_value_sval {\n ($args:expr) => {\n $crate::__private_api::capture_sval(&&$args)\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\n#[cfg(not(feature = \"kv_sval\"))]\nmacro_rules! __log_value_sval {\n ($args:expr) => {\n compile_error!(\"capturing values as `sval::Value` requires the `kv_sval` feature of `log`\")\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\n#[cfg(feature = \"kv_serde\")]\nmacro_rules! __log_value_serde {\n ($args:expr) => {\n $crate::__private_api::capture_serde(&&$args)\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\n#[cfg(not(feature = \"kv_serde\"))]\nmacro_rules! __log_value_serde {\n ($args:expr) => {\n compile_error!(\n \"capturing values as `serde::Serialize` requires the `kv_serde` feature of `log`\"\n )\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\n#[cfg(feature = \"kv_std\")]\nmacro_rules! __log_value_error {\n ($args:expr) => {\n $crate::__private_api::capture_error(&$args)\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\n#[cfg(not(feature = \"kv_std\"))]\nmacro_rules! __log_value_error {\n ($args:expr) => {\n compile_error!(\n \"capturing values as `std::error::Error` requires the `kv_std` feature of `log`\"\n )\n };\n}\n" }, { "package": "memchr@2.8.3", "sourcePath": "README.md", "sha256": "92a74aaffe011bdaa06fbc34a01686a6eba58ca1322e976759417a547fddf734", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "memchr\n======\nThis library provides heavily optimized routines for string search primitives.\n\n[![Build status](https://github.com/BurntSushi/memchr/workflows/ci/badge.svg)](https://github.com/BurntSushi/memchr/actions)\n[![Crates.io](https://img.shields.io/crates/v/memchr.svg)](https://crates.io/crates/memchr)\n\nDual-licensed under MIT or the [UNLICENSE](https://unlicense.org/).\n\n\n### Documentation\n\n[https://docs.rs/memchr](https://docs.rs/memchr)\n\n\n### Overview\n\n* The top-level module provides routines for searching for 1, 2 or 3 bytes\n in the forward or reverse direction. When searching for more than one byte,\n positions are considered a match if the byte at that position matches any\n of the bytes.\n* The `memmem` sub-module provides forward and reverse substring search\n routines.\n\nIn all such cases, routines operate on `&[u8]` without regard to encoding. This\nis exactly what you want when searching either UTF-8 or arbitrary bytes.\n\n### Compiling without the standard library\n\nmemchr links to the standard library by default, but you can disable the\n`std` feature if you want to use it in a `#![no_std]` crate:\n\n```toml\n[dependencies]\nmemchr = { version = \"2\", default-features = false }\n```\n\nOn `x86_64` platforms, when the `std` feature is disabled, the SSE2 accelerated\nimplementations will be used. When `std` is enabled, AVX2 accelerated\nimplementations will be used if the CPU is determined to support it at runtime.\n\nSIMD accelerated routines are also available on the `wasm32` and `aarch64`\ntargets. The `std` feature is not required to use them.\n\nWhen a SIMD version is not available, then this crate falls back to\n[SWAR](https://en.wikipedia.org/wiki/SWAR) techniques.\n\n### Minimum Rust version policy\n\nThis crate's minimum supported `rustc` version is `1.61.0`.\n\nThe current policy is that the minimum Rust version required to use this crate\ncan be increased in minor version updates. For example, if `crate 1.0` requires\nRust 1.20.0, then `crate 1.0.z` for all values of `z` will also require Rust\n1.20.0 or newer. However, `crate 1.y` for `y > 0` may require a newer minimum\nversion of Rust.\n\nIn general, this crate will be conservative with respect to the minimum\nsupported version of Rust.\n\n\n### Testing strategy\n\nGiven the complexity of the code in this crate, along with the pervasive use\nof `unsafe`, this crate has an extensive testing strategy. It combines multiple\napproaches:\n\n* Hand-written tests.\n* Exhaustive-style testing meant to exercise all possible branching and offset\n calculations.\n* Property based testing through [`quickcheck`](https://github.com/BurntSushi/quickcheck).\n* Fuzz testing through [`cargo fuzz`](https://github.com/rust-fuzz/cargo-fuzz).\n* A huge suite of benchmarks that are also run as tests. Benchmarks always\n confirm that the expected result occurs.\n\nImprovements to the testing infrastructure are very welcome.\n\n\n### Algorithms used\n\nAt time of writing, this crate's implementation of substring search actually\nhas a few different algorithms to choose from depending on the situation.\n\n* For very small haystacks,\n [Rabin-Karp](https://en.wikipedia.org/wiki/Rabin%E2%80%93Karp_algorithm)\n is used to reduce latency. Rabin-Karp has very small overhead and can often\n complete before other searchers have even been constructed.\n* For small needles, a variant of the\n [\"Generic SIMD\"](http://0x80.pl/articles/simd-strfind.html#algorithm-1-generic-simd)\n algorithm is used. Instead of using the first and last bytes, a heuristic is\n used to select bytes based on a background distribution of byte frequencies.\n* In all other cases,\n [Two-Way](https://en.wikipedia.org/wiki/Two-way_string-matching_algorithm)\n is used. If possible, a prefilter based on the \"Generic SIMD\" algorithm\n linked above is used to find candidates quickly. A dynamic heuristic is used\n to detect if the prefilter is ineffective, and if so, disables it.\n\n\n### Why is the standard library's substring search so much slower?\n\nWe'll start by establishing what the difference in performance actually\nis. There are two relevant benchmark classes to consider: `prebuilt` and\n`oneshot`. The `prebuilt` benchmarks are designed to measure---to the extent\npossible---search time only. That is, the benchmark first starts by building a\nsearcher and then only tracking the time for _using_ the searcher:\n\n```\n$ rebar rank benchmarks/record/x86_64/2023-08-26.csv --intersection -e memchr/memmem/prebuilt -e std/memmem/prebuilt\nEngine Version Geometric mean of speed ratios Benchmark count\n------ ------- ------------------------------ ---------------\nrust/memchr/memmem/prebuilt 2.5.0 1.03 53\nrust/std/memmem/prebuilt 1.73.0-nightly 180dffba1 6.50 53\n```\n\nConversely, the `oneshot` benchmark class measures the time it takes to both\nbuild the searcher _and_ use it:\n\n```\n$ rebar rank benchmarks/record/x86_64/2023-08-26.csv --intersection -e memchr/memmem/oneshot -e std/memmem/oneshot\nEngine Version Geometric mean of speed ratios Benchmark count\n------ ------- ------------------------------ ---------------\nrust/memchr/memmem/oneshot 2.5.0 1.04 53\nrust/std/memmem/oneshot 1.73.0-nightly 180dffba1 5.26 53\n```\n\n**NOTE:** Replace `rebar rank` with `rebar cmp` in the above commands to\nexplore the specific benchmarks and their differences.\n\nSo in both cases, this crate is quite a bit faster over a broad sampling of\nbenchmarks regardless of whether you measure only search time or search time\nplus construction time. The difference is a little smaller when you include\nconstruction time in your measurements.\n\nThese two different types of benchmark classes make for a nice segue into\none reason why the standard library's substring search can be slower: API\ndesign. In the standard library, the only APIs available to you require\none to re-construct the searcher for every search. While you can benefit\nfrom building a searcher once and iterating over all matches in a single\nstring, you cannot reuse that searcher to search other strings. This might\ncome up when, for example, searching a file one line at a time. You'll need\nto re-build the searcher for every line searched, and this can [really\nmatter][burntsushi-bstr-blog].\n\n**NOTE:** The `prebuilt` benchmark for the standard library can't actually\navoid measuring searcher construction at some level, because there is no API\nfor it. Instead, the benchmark consists of building the searcher once and then\nfinding all matches in a single string via an iterator. This tends to\napproximate a benchmark where searcher construction isn't measured, but it\nisn't perfect. While this means the comparison is not strictly\napples-to-apples, it does reflect what is maximally possible with the standard\nlibrary, and thus reflects the best that one could do in a real world scenario.\n\nWhile there is more to the story than just API design here, it's important to\npoint out that even if the standard library's substring search were a precise\nclone of this crate internally, it would still be at a disadvantage in some\nworkloads because of its API. (The same also applies to C's standard library\n`memmem` function. There is no way to amortize construction of the searcher.\nYou need to pay for it on every call.)\n\nThe other reason for the difference in performance is that\nthe standard library has trouble using SIMD. In particular, substring search\nis implemented in the `core` library, where platform specific code generally\ncan't exist. That's an issue because in order to utilize SIMD beyond SSE2\nwhile maintaining portable binaries, one needs to use [dynamic CPU feature\ndetection][dynamic-cpu], and that in turn requires platform specific code.\nWhile there is [an RFC for enabling target feature detection in\n`core`][core-feature], it doesn't yet exist.\n\nThe bottom line here is that `core`'s substring search implementation is\nlimited to making use of SSE2, but not AVX.\n\nStill though, this crate does accelerate substring search even when only SSE2\nis available. The standard library could therefore adopt the techniques in this\ncrate just for SSE2. The reason why that hasn't happened yet isn't totally\nclear to me. It likely needs a champion to push it through. The standard\nlibrary tends to be more conservative in these things. With that said, the\nstandard library does use some [SSE2 acceleration on `x86-64`][std-sse2] added\nin [this PR][std-sse2-pr]. However, at the time of writing, it is only used\nfor short needles and doesn't use the frequency based heuristics found in this\ncrate.\n\n**NOTE:** Another thing worth mentioning is that the standard library's\nsubstring search routine requires that both the needle and haystack have type\n`&str`. Unless you can assume that your data is valid UTF-8, building a `&str`\nwill come with the overhead of UTF-8 validation. This may in turn result in\noverall slower searching depending on your workload. In contrast, the `memchr`\ncrate permits both the needle and the haystack to have type `&[u8]`, where\n`&[u8]` can be created from a `&str` with zero cost. Therefore, the substring\nsearch in this crate is strictly more flexible than what the standard library\nprovides.\n\n[burntsushi-bstr-blog]: https://blog.burntsushi.net/bstr/#motivation-based-on-performance\n[dynamic-cpu]: https://doc.rust-lang.org/std/arch/index.html#dynamic-cpu-feature-detection\n[core-feature]: https://github.com/rust-lang/rfcs/pull/3469\n[std-sse2]: https://github.com/rust-lang/rust/blob/bf9229a2e366b4c311f059014a4aa08af16de5d8/library/core/src/str/pattern.rs#L1719-L1857\n[std-sse2-pr]: https://github.com/rust-lang/rust/pull/103779\n" }, { "package": "mimalloc@0.1.52", "sourcePath": "LICENSE.txt", "sha256": "a08554fab028af3af047e8588d32d019662c1b1bf07959a195a2f75fe340726b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Copyright 2019 Octavian Oncescu\n\nPermission is hereby granted, free of charge, to any\nperson obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without\nlimitation the rights to use, copy, modify, merge,\npublish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software\nis furnished to do so, subject to the following\nconditions:\n\nThe above copyright notice and this permission notice\nshall be included in all copies or substantial portions\nof the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF\nANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\nTO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\nSHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR\nIN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE." }, { "package": "mimalloc@0.1.52", "sourcePath": "src/lib.rs", "sha256": "f2f6e98264507d5cfa836a48e5b0fe220f3bcc3270a8f5fabeb4297898a27201", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 Octavian Oncescu\n\n#![no_std]\n\n//! A drop-in global allocator wrapper around the [mimalloc](https://github.com/microsoft/mimalloc) allocator.\n//! Mimalloc is a general purpose, performance oriented allocator built by Microsoft.\n//!\n//! ## Usage\n//! ```rust,ignore\n//! use mimalloc::MiMalloc;\n//!\n//! #[global_allocator]\n//! static GLOBAL: MiMalloc = MiMalloc;\n//! ```\n//!\n//! ## Usage with secure mode\n//! Using secure mode adds guard pages,\n//! randomized allocation, encrypted free lists, etc. The performance penalty is usually\n//! around 10% according to [mimalloc's](https://github.com/microsoft/mimalloc)\n//! own benchmarks.\n//!\n//! To enable secure mode, put in `Cargo.toml`:\n//! ```toml\n//! [dependencies]\n//! mimalloc = { version = \"*\", features = [\"secure\"] }\n//! ```\n\nextern crate libmimalloc_sys as ffi;\n\n#[cfg(feature = \"extended\")]\nmod extended;\n\nuse core::alloc::{GlobalAlloc, Layout};\nuse core::ffi::c_void;\nuse ffi::*;\n\n/// Drop-in mimalloc global allocator.\n///\n/// ## Usage\n/// ```rust,ignore\n/// use mimalloc::MiMalloc;\n///\n/// #[global_allocator]\n/// static GLOBAL: MiMalloc = MiMalloc;\n/// ```\npub struct MiMalloc;\n\nunsafe impl GlobalAlloc for MiMalloc {\n #[inline]\n unsafe fn alloc(&self, layout: Layout) -> *mut u8 {\n mi_malloc_aligned(layout.size(), layout.align()) as *mut u8\n }\n\n #[inline]\n unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {\n mi_zalloc_aligned(layout.size(), layout.align()) as *mut u8\n }\n\n #[inline]\n unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {\n mi_free(ptr as *mut c_void);\n }\n\n #[inline]\n unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {\n mi_realloc_aligned(ptr as *mut c_void, new_size, layout.align()) as *mut u8\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn it_frees_allocated_memory() {\n unsafe {\n let layout = Layout::from_size_align(8, 8).unwrap();\n let alloc = MiMalloc;\n\n let ptr = alloc.alloc(layout);\n alloc.dealloc(ptr, layout);\n }\n }\n\n #[test]\n fn it_frees_allocated_big_memory() {\n unsafe {\n let layout = Layout::from_size_align(1 << 20, 32).unwrap();\n let alloc = MiMalloc;\n\n let ptr = alloc.alloc(layout);\n alloc.dealloc(ptr, layout);\n }\n }\n\n #[test]\n fn it_frees_zero_allocated_memory() {\n unsafe {\n let layout = Layout::from_size_align(8, 8).unwrap();\n let alloc = MiMalloc;\n\n let ptr = alloc.alloc_zeroed(layout);\n alloc.dealloc(ptr, layout);\n }\n }\n\n #[test]\n fn it_frees_zero_allocated_big_memory() {\n unsafe {\n let layout = Layout::from_size_align(1 << 20, 32).unwrap();\n let alloc = MiMalloc;\n\n let ptr = alloc.alloc_zeroed(layout);\n alloc.dealloc(ptr, layout);\n }\n }\n\n #[test]\n fn it_frees_reallocated_memory() {\n unsafe {\n let layout = Layout::from_size_align(8, 8).unwrap();\n let alloc = MiMalloc;\n\n let ptr = alloc.alloc(layout);\n let ptr = alloc.realloc(ptr, layout, 16);\n alloc.dealloc(ptr, layout);\n }\n }\n\n #[test]\n fn it_frees_reallocated_big_memory() {\n unsafe {\n let layout = Layout::from_size_align(1 << 20, 32).unwrap();\n let alloc = MiMalloc;\n\n let ptr = alloc.alloc(layout);\n let ptr = alloc.realloc(ptr, layout, 2 << 20);\n alloc.dealloc(ptr, layout);\n }\n }\n}\n" }, { "package": "miniz_oxide@0.5.4", "sourcePath": "LICENSE-APACHE.md", "sha256": "0d542e0c8804e39aa7f37eb00da5a762149dc682d7829451287e11b938e94594", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n" }, { "package": "miniz_oxide@0.5.4", "sourcePath": "LICENSE-MIT.md", "sha256": "e190940e8ad3cdd4fca962a6508ed6865d589d314b1cb055f86000e124b88d8d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "MIT License\n\nCopyright (c) 2017 Frommi\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "package": "miniz_oxide@0.5.4", "sourcePath": "LICENSE-ZLIB.md", "sha256": "c89bcc058da12a0fb24402b8ea4542a21515dd1da2e8c67bba4ed9bd269f1c96", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Copyright (c) 2020 Frommi\n\nThis software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software.\n\nPermission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions:\n\n1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.\n\n2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.\n\n3. This notice may not be removed or altered from any source distribution.\n" }, { "package": "miniz_oxide@0.9.1", "sourcePath": "LICENSE-APACHE.md", "sha256": "0d542e0c8804e39aa7f37eb00da5a762149dc682d7829451287e11b938e94594", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n" }, { "package": "miniz_oxide@0.9.1", "sourcePath": "LICENSE-MIT.md", "sha256": "799e9ca9d179295ef372f25d3769cdda7d25bb2668add6a6a1e22d1e4c678b8d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "MIT License\n\nCopyright 2013-2014 RAD Game Tools and Valve Software\nCopyright 2010-2014 Rich Geldreich and Tenacious Software LLC\nCopyright (c) 2017 Frommi\nCopyright (c) 2017-2024 oyvindln\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "package": "miniz_oxide@0.9.1", "sourcePath": "LICENSE-ZLIB.md", "sha256": "0a54e647fe54104658b5e563c04c6f9edf251710e47bce692e0bd990a4ddaa39", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Copyright 2013-2014 RAD Game Tools and Valve Software\nCopyright 2010-2014 Rich Geldreich and Tenacious Software LLC\nCopyright (c) 2020 Frommi\nCopyright (c) 2017-2024 oyvindln\n\nThis software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software.\n\nPermission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions:\n\n1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.\n\n2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.\n\n3. This notice may not be removed or altered from any source distribution.\n" }, { "package": "once_cell_polyfill@1.70.2", "sourcePath": "README.md", "sha256": "e37bd89789ba9f0624e3a9c06595d6f45f3c87bfc69c5a1a32af91cdbfd0b456", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# once_cell_polyfill\n\n> Polyfill for `OnceCell` stdlib feature for use with older MSRVs\n\n[![Documentation](https://img.shields.io/badge/docs-master-blue.svg)][Documentation]\n![License](https://img.shields.io/crates/l/once_cell_polyfill.svg)\n[![Crates Status](https://img.shields.io/crates/v/once_cell_polyfill.svg)][Crates.io]\n\n## License\n\nLicensed under either of\n\n* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or )\n* MIT license ([LICENSE-MIT](LICENSE-MIT) or )\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally\nsubmitted for inclusion in the work by you, as defined in the Apache-2.0\nlicense, shall be dual-licensed as above, without any additional terms or\nconditions.\n\n[Crates.io]: https://crates.io/crates/once_cell_polyfill\n[Documentation]: https://docs.rs/once_cell_polyfill\n" }, { "package": "paste@1.0.15", "sourcePath": "README.md", "sha256": "cd8f304fa9cca9566dff0bb3492beae55f47f3972ddfb8f1fa38c3da1068b4a8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Macros for all your token pasting needs\n=======================================\n\n[\"github\"](https://github.com/dtolnay/paste)\n[\"crates.io\"](https://crates.io/crates/paste)\n[\"docs.rs\"](https://docs.rs/paste)\n[\"build](https://github.com/dtolnay/paste/actions?query=branch%3Amaster)\n\nThe nightly-only [`concat_idents!`] macro in the Rust standard library is\nnotoriously underpowered in that its concatenated identifiers can only refer to\nexisting items, they can never be used to define something new.\n\n[`concat_idents!`]: https://doc.rust-lang.org/std/macro.concat_idents.html\n\nThis crate provides a flexible way to paste together identifiers in a macro,\nincluding using pasted identifiers to define new items.\n\n```toml\n[dependencies]\npaste = \"1.0\"\n```\n\nThis approach works with any Rust compiler 1.31+.\n\n
\n\n## Pasting identifiers\n\nWithin the `paste!` macro, identifiers inside `[<`...`>]` are pasted together to\nform a single identifier.\n\n```rust\nuse paste::paste;\n\npaste! {\n // Defines a const called `QRST`.\n const []: &str = \"success!\";\n}\n\nfn main() {\n assert_eq!(\n paste! { [].len() },\n 8,\n );\n}\n```\n\n
\n\n## More elaborate example\n\nThe next example shows a macro that generates accessor methods for some struct\nfields. It demonstrates how you might find it useful to bundle a paste\ninvocation inside of a macro\\_rules macro.\n\n```rust\nuse paste::paste;\n\nmacro_rules! make_a_struct_and_getters {\n ($name:ident { $($field:ident),* }) => {\n // Define a struct. This expands to:\n //\n // pub struct S {\n // a: String,\n // b: String,\n // c: String,\n // }\n pub struct $name {\n $(\n $field: String,\n )*\n }\n\n // Build an impl block with getters. This expands to:\n //\n // impl S {\n // pub fn get_a(&self) -> &str { &self.a }\n // pub fn get_b(&self) -> &str { &self.b }\n // pub fn get_c(&self) -> &str { &self.c }\n // }\n paste! {\n impl $name {\n $(\n pub fn [](&self) -> &str {\n &self.$field\n }\n )*\n }\n }\n }\n}\n\nmake_a_struct_and_getters!(S { a, b, c });\n\nfn call_some_getters(s: &S) -> bool {\n s.get_a() == s.get_b() && s.get_c().is_empty()\n}\n```\n\n
\n\n## Case conversion\n\nUse `$var:lower` or `$var:upper` in the segment list to convert an interpolated\nsegment to lower- or uppercase as part of the paste. For example, `[]` would paste to `ld_bc_expr` if invoked with $reg=`Bc`.\n\nUse `$var:snake` to convert CamelCase input to snake\\_case.\nUse `$var:camel` to convert snake\\_case to CamelCase.\nThese compose, so for example `$var:snake:upper` would give you SCREAMING\\_CASE.\n\nThe precise Unicode conversions are as defined by [`str::to_lowercase`] and\n[`str::to_uppercase`].\n\n[`str::to_lowercase`]: https://doc.rust-lang.org/std/primitive.str.html#method.to_lowercase\n[`str::to_uppercase`]: https://doc.rust-lang.org/std/primitive.str.html#method.to_uppercase\n\n
\n\n## Pasting documentation strings\n\nWithin the `paste!` macro, arguments to a #\\[doc ...\\] attribute are implicitly\nconcatenated together to form a coherent documentation string.\n\n```rust\nuse paste::paste;\n\nmacro_rules! method_new {\n ($ret:ident) => {\n paste! {\n #[doc = \"Create a new `\" $ret \"` object.\"]\n pub fn new() -> $ret { todo!() }\n }\n };\n}\n\npub struct Paste {}\n\nmethod_new!(Paste); // expands to #[doc = \"Create a new `Paste` object\"]\n```\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this crate by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "pathfinder_color@0.5.0", "sourcePath": "src/lib.rs", "sha256": "211af4c221cb572c9b5c2bbaeae5ee63cf9ea840ce43d7dca85c222a1d38cadd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/color/src/lib.rs\n//\n// Copyright © 2020 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse pathfinder_simd::default::F32x4;\nuse std::f32::consts::PI;\nuse std::fmt::{self, Debug, Formatter};\nuse std::slice;\n\n// TODO(pcwalton): Maybe this should be a u32? Need to be aware of endianness issues if we do that.\n#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]\n#[repr(C)]\npub struct ColorU {\n pub r: u8,\n pub g: u8,\n pub b: u8,\n pub a: u8,\n}\n\nimpl ColorU {\n #[inline]\n pub fn new(r: u8, g: u8, b: u8, a: u8) -> ColorU {\n ColorU { r, g, b, a }\n }\n\n #[inline]\n pub fn transparent_black() -> ColorU {\n ColorU::from_u32(0)\n }\n\n #[inline]\n pub fn from_u32(rgba: u32) -> ColorU {\n ColorU {\n r: (rgba >> 24) as u8,\n g: ((rgba >> 16) & 0xff) as u8,\n b: ((rgba >> 8) & 0xff) as u8,\n a: (rgba & 0xff) as u8,\n }\n }\n\n #[inline]\n pub fn black() -> ColorU {\n ColorU {\n r: 0,\n g: 0,\n b: 0,\n a: 255,\n }\n }\n\n #[inline]\n pub fn white() -> ColorU {\n ColorU {\n r: 255,\n g: 255,\n b: 255,\n a: 255,\n }\n }\n\n #[inline]\n pub fn to_f32(&self) -> ColorF {\n let color = F32x4::new(self.r as f32, self.g as f32, self.b as f32, self.a as f32);\n ColorF(color * F32x4::splat(1.0 / 255.0))\n }\n\n #[inline]\n pub fn is_opaque(&self) -> bool {\n self.a == !0\n }\n\n #[inline]\n pub fn is_fully_transparent(&self) -> bool {\n self.a == 0\n }\n}\n\nimpl Debug for ColorU {\n fn fmt(&self, formatter: &mut Formatter) -> fmt::Result {\n if self.a == 255 {\n write!(formatter, \"#{:02x}{:02x}{:02x}\", self.r, self.g, self.b)\n } else {\n write!(\n formatter,\n \"rgba({}, {}, {}, {})\",\n self.r,\n self.g,\n self.b,\n self.a as f32 / 255.0\n )\n }\n }\n}\n\n#[derive(Clone, Copy, PartialEq, Default)]\npub struct ColorF(pub F32x4);\n\nimpl ColorF {\n // Constructors\n\n #[inline]\n pub fn new(r: f32, g: f32, b: f32, a: f32) -> ColorF {\n ColorF(F32x4::new(r, g, b, a))\n }\n\n #[inline]\n pub fn from_hsla(mut h: f32, s: f32, l: f32, a: f32) -> ColorF {\n // https://en.wikipedia.org/wiki/HSL_and_HSV#HSL_to_RGB\n\n // Make sure hue is always positive.\n h %= 2.0 * PI;\n if h < 0.0 {\n h += 2.0 * PI;\n }\n\n h *= 3.0 / PI;\n\n // Calculate chroma.\n let c = (1.0 - f32::abs(2.0 * l - 1.0)) * s;\n let xc = F32x4::new(c * (1.0 - f32::abs(h % 2.0 - 1.0)), c, 0.0, a);\n let rgba = match f32::ceil(h) as i32 {\n 1 => xc.yxzw(),\n 2 => xc.xyzw(),\n 3 => xc.zyxw(),\n 4 => xc.zxyw(),\n 5 => xc.xzyw(),\n 0 | 6 => xc.yzxw(),\n _ => xc.zzzw(),\n };\n let m = l - 0.5 * c;\n ColorF(rgba + F32x4::new(m, m, m, 0.0))\n }\n\n #[inline]\n pub fn from_hsl(h: f32, s: f32, l: f32) -> ColorF {\n ColorF::from_hsla(h, s, l, 1.0)\n }\n\n #[inline]\n pub fn transparent_black() -> ColorF {\n ColorF::default()\n }\n\n #[inline]\n pub fn black() -> ColorF {\n ColorF(F32x4::new(0.0, 0.0, 0.0, 1.0))\n }\n\n #[inline]\n pub fn white() -> ColorF {\n ColorF(F32x4::splat(1.0))\n }\n\n #[inline]\n pub fn to_u8(&self) -> ColorU {\n let color = (self.0 * F32x4::splat(255.0)).to_i32x4();\n ColorU { r: color[0] as u8, g: color[1] as u8, b: color[2] as u8, a: color[3] as u8 }\n }\n\n #[inline]\n pub fn lerp(&self, other: ColorF, t: f32) -> ColorF {\n ColorF(self.0 + (other.0 - self.0) * F32x4::splat(t))\n }\n\n #[inline]\n pub fn r(&self) -> f32 {\n self.0[0]\n }\n\n #[inline]\n pub fn g(&self) -> f32 {\n self.0[1]\n }\n\n #[inline]\n pub fn b(&self) -> f32 {\n self.0[2]\n }\n\n #[inline]\n pub fn a(&self) -> f32 {\n self.0[3]\n }\n\n #[inline]\n pub fn set_r(&mut self, r: f32) {\n self.0[0] = r;\n }\n\n #[inline]\n pub fn set_g(&mut self, g: f32) {\n self.0[1] = g;\n }\n\n #[inline]\n pub fn set_b(&mut self, b: f32) {\n self.0[2] = b;\n }\n\n #[inline]\n pub fn set_a(&mut self, a: f32) {\n self.0[3] = a;\n }\n}\n\nimpl Debug for ColorF {\n fn fmt(&self, formatter: &mut Formatter) -> fmt::Result {\n write!(\n formatter,\n \"rgba({}, {}, {}, {})\",\n self.r() * 255.0,\n self.g() * 255.0,\n self.b() * 255.0,\n self.a()\n )\n }\n}\n\n#[inline]\npub fn color_slice_to_u8_slice(slice: &[ColorU]) -> &[u8] {\n unsafe {\n slice::from_raw_parts(slice.as_ptr() as *const u8, slice.len() * 4)\n }\n}\n\n#[inline]\npub fn u8_slice_to_color_slice(slice: &[u8]) -> &[ColorU] {\n unsafe {\n assert_eq!(slice.len() % 4, 0);\n slice::from_raw_parts(slice.as_ptr() as *const ColorU, slice.len() / 4)\n }\n}\n\n// TODO(pcwalton): Do this without a copy?\n#[inline]\npub fn u8_vec_to_color_vec(buffer: Vec) -> Vec {\n u8_slice_to_color_slice(&buffer).to_vec()\n}\n\n/// A convenience method to construct a `ColorU` from an RGB triple.\n///\n/// Alpha is set to 255.\n#[inline]\npub fn rgbu(r: u8, g: u8, b: u8) -> ColorU {\n ColorU::new(r, g, b, 255)\n}\n\n/// A convenience method to construct a `ColorU` from an RGBA triple.\n#[inline]\npub fn rgbau(r: u8, g: u8, b: u8, a: u8) -> ColorU {\n ColorU::new(r, g, b, a)\n}\n\n/// A convenience method to construct a `ColorF` from an RGB triple.\n///\n/// Alpha is set to 1.0.\n#[inline]\npub fn rgbf(r: f32, g: f32, b: f32) -> ColorF {\n ColorF::new(r, g, b, 1.0)\n}\n\n/// A convenience method to construct a `ColorF` from an RGBA triple.\n#[inline]\npub fn rgbaf(r: f32, g: f32, b: f32, a: f32) -> ColorF {\n ColorF::new(r, g, b, a)\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/clip.rs", "sha256": "35deda1ded3a1f79ffb969742725d6cf61818af142cd0b8068af10a6d8efea87", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/clip.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::outline::{Contour, ContourIterFlags, PointFlags, PushSegmentFlags};\nuse crate::segment::{CubicSegment, Segment};\nuse arrayvec::ArrayVec;\nuse pathfinder_geometry::line_segment::LineSegment2F;\nuse pathfinder_geometry::rect::RectF;\nuse pathfinder_geometry::util::lerp;\nuse pathfinder_geometry::vector::{Vector2F, Vector4F};\nuse smallvec::SmallVec;\nuse std::fmt::Debug;\nuse std::mem;\n\n#[derive(Clone, Copy, Debug)]\nstruct Edge(LineSegment2F);\n\nimpl TEdge for Edge {\n #[inline]\n fn point_is_inside(&self, point: Vector2F) -> bool {\n let area = (self.0.to() - self.0.from()).det(point - self.0.from());\n debug!(\"point_is_inside({:?}, {:?}), area={}\", self, point, area);\n area >= 0.0\n }\n\n fn intersect_line_segment(&self, segment: LineSegment2F) -> ArrayVec<[f32; 3]> {\n let mut results = ArrayVec::new();\n if let Some(t) = segment.intersection_t(self.0) {\n if t >= 0.0 && t <= 1.0 {\n results.push(t);\n }\n }\n results\n }\n}\n\n#[derive(Clone, Copy, Debug)]\nenum AxisAlignedEdge {\n Left(f32),\n Top(f32),\n Right(f32),\n Bottom(f32),\n}\n\nimpl TEdge for AxisAlignedEdge {\n #[inline]\n fn point_is_inside(&self, point: Vector2F) -> bool {\n match *self {\n AxisAlignedEdge::Left(x) => point.x() >= x,\n AxisAlignedEdge::Top(y) => point.y() >= y,\n AxisAlignedEdge::Right(x) => point.x() <= x,\n AxisAlignedEdge::Bottom(y) => point.y() <= y,\n }\n }\n\n fn intersect_line_segment(&self, segment: LineSegment2F) -> ArrayVec<[f32; 3]> {\n let mut results = ArrayVec::new();\n let t = match *self {\n AxisAlignedEdge::Left(x) | AxisAlignedEdge::Right(x) => segment.solve_t_for_x(x),\n AxisAlignedEdge::Top(y) | AxisAlignedEdge::Bottom(y) => segment.solve_t_for_y(y),\n };\n if t >= 0.0 && t <= 1.0 {\n results.push(t);\n }\n results\n }\n}\n\ntrait TEdge: Debug {\n fn point_is_inside(&self, point: Vector2F) -> bool;\n fn intersect_line_segment(&self, segment: LineSegment2F) -> ArrayVec<[f32; 3]>;\n\n fn trivially_test_segment(&self, segment: &Segment) -> EdgeRelativeLocation {\n let from_inside = self.point_is_inside(segment.baseline.from());\n debug!(\n \"point {:?} inside {:?}: {:?}\",\n segment.baseline.from(),\n self,\n from_inside\n );\n if from_inside != self.point_is_inside(segment.baseline.to()) {\n return EdgeRelativeLocation::Intersecting;\n }\n if !segment.is_line() {\n if from_inside != self.point_is_inside(segment.ctrl.from()) {\n return EdgeRelativeLocation::Intersecting;\n }\n if !segment.is_quadratic() {\n if from_inside != self.point_is_inside(segment.ctrl.to()) {\n return EdgeRelativeLocation::Intersecting;\n }\n }\n }\n if from_inside {\n EdgeRelativeLocation::Inside\n } else {\n EdgeRelativeLocation::Outside\n }\n }\n\n fn intersect_segment(&self, segment: &Segment) -> ArrayVec<[f32; 3]> {\n if segment.is_line() {\n return self.intersect_line_segment(segment.baseline);\n }\n\n let mut segment = *segment;\n if segment.is_quadratic() {\n segment = segment.to_cubic();\n }\n\n let mut results = ArrayVec::new();\n let mut prev_t = 0.0;\n while !results.is_full() {\n if prev_t >= 1.0 {\n break;\n }\n let next_t = match self.intersect_cubic_segment(&segment, prev_t, 1.0) {\n None => break,\n Some(next_t) => next_t,\n };\n results.push(next_t);\n prev_t = next_t + EPSILON;\n }\n return results;\n\n const EPSILON: f32 = 0.0001;\n }\n\n fn intersect_cubic_segment(\n &self,\n segment: &Segment,\n mut t_min: f32,\n mut t_max: f32,\n ) -> Option {\n debug!(\n \"... intersect_cubic_segment({:?}, {:?}, t=({}, {}))\",\n self, segment, t_min, t_max\n );\n\n let mut segment = segment.as_cubic_segment().split_after(t_min);\n segment = segment\n .as_cubic_segment()\n .split_before(t_max / (1.0 - t_min));\n\n if !self.intersects_cubic_segment_hull(segment.as_cubic_segment()) {\n return None;\n }\n\n loop {\n let t_mid = lerp(t_min, t_max, 0.5);\n if t_max - t_min < 0.00001 {\n return Some(t_mid);\n }\n\n let (prev_segment, next_segment) = segment.as_cubic_segment().split(0.5);\n if self.intersects_cubic_segment_hull(prev_segment.as_cubic_segment()) {\n t_max = t_mid;\n segment = prev_segment;\n } else if self.intersects_cubic_segment_hull(next_segment.as_cubic_segment()) {\n t_min = t_mid;\n segment = next_segment;\n } else {\n return None;\n }\n }\n }\n\n fn intersects_cubic_segment_hull(&self, cubic_segment: CubicSegment) -> bool {\n let inside = self.point_is_inside(cubic_segment.0.baseline.from());\n inside != self.point_is_inside(cubic_segment.0.ctrl.from())\n || inside != self.point_is_inside(cubic_segment.0.ctrl.to())\n || inside != self.point_is_inside(cubic_segment.0.baseline.to())\n }\n}\n\ntrait ContourClipper\nwhere\n Self::Edge: TEdge + Debug,\n{\n type Edge;\n\n fn contour_mut(&mut self) -> &mut Contour;\n\n fn clip_against(&mut self, edge: Self::Edge) {\n // Fast path to avoid allocation in the no-clip case.\n match self.check_for_fast_clip(&edge) {\n FastClipResult::SlowPath => {}\n FastClipResult::AllInside => return,\n FastClipResult::AllOutside => {\n *self.contour_mut() = Contour::new();\n return;\n }\n }\n\n let input = self.contour_mut().take();\n for segment in input.iter(ContourIterFlags::empty()) {\n self.clip_segment_against(segment, &edge);\n }\n if input.is_closed() {\n self.contour_mut().close();\n }\n }\n\n fn clip_segment_against(&mut self, mut segment: Segment, edge: &Self::Edge) {\n // Easy cases.\n match edge.trivially_test_segment(&segment) {\n EdgeRelativeLocation::Outside => return,\n EdgeRelativeLocation::Inside => {\n debug!(\"trivial test inside, pushing segment\");\n self.push_segment(&segment);\n return;\n }\n EdgeRelativeLocation::Intersecting => {}\n }\n\n // We have a potential intersection.\n debug!(\"potential intersection: {:?} edge: {:?}\", segment, edge);\n let mut starts_inside = edge.point_is_inside(segment.baseline.from());\n let intersection_ts = edge.intersect_segment(&segment);\n let mut last_t = 0.0;\n debug!(\"... intersections: {:?}\", intersection_ts);\n for t in intersection_ts {\n let (before_split, after_split) = segment.split((t - last_t) / (1.0 - last_t));\n\n // Push the split segment if appropriate.\n debug!(\n \"... ... edge={:?} before_split={:?} t={:?} starts_inside={:?}\",\n edge, before_split, t, starts_inside\n );\n if starts_inside {\n debug!(\"... split segment case, pushing segment\");\n self.push_segment(&before_split);\n }\n\n // We've now transitioned from inside to outside or vice versa.\n starts_inside = !starts_inside;\n last_t = t;\n segment = after_split;\n }\n\n // No more intersections. Push the last segment if applicable.\n if starts_inside {\n debug!(\"... last segment case, pushing segment\");\n self.push_segment(&segment);\n }\n }\n\n fn push_segment(&mut self, segment: &Segment) {\n let contour = self.contour_mut();\n if let Some(last_position) = contour.last_position() {\n if last_position != segment.baseline.from() {\n // Add a line to join up segments.\n contour.push_point(segment.baseline.from(), PointFlags::empty(), true);\n }\n }\n\n contour.push_segment(segment, PushSegmentFlags::UPDATE_BOUNDS);\n }\n\n fn check_for_fast_clip(&mut self, edge: &Self::Edge) -> FastClipResult {\n let mut result = None;\n for segment in self.contour_mut().iter(ContourIterFlags::empty()) {\n let location = edge.trivially_test_segment(&segment);\n match (result, location) {\n (None, EdgeRelativeLocation::Outside) => {\n result = Some(FastClipResult::AllOutside);\n }\n (None, EdgeRelativeLocation::Inside) => {\n result = Some(FastClipResult::AllInside);\n }\n (Some(FastClipResult::AllInside), EdgeRelativeLocation::Inside)\n | (Some(FastClipResult::AllOutside), EdgeRelativeLocation::Outside) => {}\n (_, _) => return FastClipResult::SlowPath,\n }\n }\n result.unwrap_or(FastClipResult::AllOutside)\n }\n}\n\n#[derive(Clone, Copy)]\nenum FastClipResult {\n SlowPath,\n AllInside,\n AllOutside,\n}\n\n// General convex polygon clipping in 2D\n\npub(crate) struct ContourPolygonClipper {\n clip_polygon: SmallVec<[Vector2F; 4]>,\n contour: Contour,\n}\n\nimpl ContourClipper for ContourPolygonClipper {\n type Edge = Edge;\n\n #[inline]\n fn contour_mut(&mut self) -> &mut Contour {\n &mut self.contour\n }\n}\n\nimpl ContourPolygonClipper {\n #[inline]\n pub(crate) fn new(clip_polygon: &[Vector2F], contour: Contour) -> ContourPolygonClipper {\n ContourPolygonClipper {\n clip_polygon: SmallVec::from_slice(clip_polygon),\n contour,\n }\n }\n\n pub(crate) fn clip(mut self) -> Contour {\n // TODO(pcwalton): Maybe have a coarse circumscribed rect and use that for clipping?\n\n let clip_polygon = mem::replace(&mut self.clip_polygon, SmallVec::default());\n let mut prev = match clip_polygon.last() {\n None => return Contour::new(),\n Some(prev) => *prev,\n };\n for &next in &clip_polygon {\n self.clip_against(Edge(LineSegment2F::new(prev, next)));\n prev = next;\n }\n\n self.contour\n }\n}\n\n#[derive(PartialEq)]\nenum EdgeRelativeLocation {\n Intersecting,\n Inside,\n Outside,\n}\n\n// Fast axis-aligned box 2D clipping\n\npub(crate) struct ContourRectClipper {\n clip_rect: RectF,\n contour: Contour,\n}\n\nimpl ContourClipper for ContourRectClipper {\n type Edge = AxisAlignedEdge;\n\n #[inline]\n fn contour_mut(&mut self) -> &mut Contour {\n &mut self.contour\n }\n}\n\nimpl ContourRectClipper {\n #[inline]\n pub(crate) fn new(clip_rect: RectF, contour: Contour) -> ContourRectClipper {\n ContourRectClipper { clip_rect, contour }\n }\n\n pub(crate) fn clip(mut self) -> Contour {\n if self.clip_rect.contains_rect(self.contour.bounds()) {\n return self.contour;\n }\n\n self.clip_against(AxisAlignedEdge::Left(self.clip_rect.min_x()));\n self.clip_against(AxisAlignedEdge::Top(self.clip_rect.min_y()));\n self.clip_against(AxisAlignedEdge::Right(self.clip_rect.max_x()));\n self.clip_against(AxisAlignedEdge::Bottom(self.clip_rect.max_y()));\n\n self.contour\n }\n}\n\n// 3D quad clipping\n\npub struct PolygonClipper3D {\n subject: Vec,\n}\n\nimpl PolygonClipper3D {\n #[inline]\n pub fn new(subject: Vec) -> PolygonClipper3D {\n PolygonClipper3D { subject }\n }\n\n pub fn clip(mut self) -> Vec {\n // TODO(pcwalton): Fast path for completely contained polygon?\n\n debug!(\"before clipping against bottom: {:?}\", self.subject);\n self.clip_against(Edge3D::Bottom);\n debug!(\"before clipping against top: {:?}\", self.subject);\n self.clip_against(Edge3D::Top);\n debug!(\"before clipping against left: {:?}\", self.subject);\n self.clip_against(Edge3D::Left);\n debug!(\"before clipping against right: {:?}\", self.subject);\n self.clip_against(Edge3D::Right);\n debug!(\"before clipping against far: {:?}\", self.subject);\n self.clip_against(Edge3D::Far);\n debug!(\"before clipping against near: {:?}\", self.subject);\n self.clip_against(Edge3D::Near);\n debug!(\"after clipping: {:?}\", self.subject);\n\n self.subject\n }\n\n fn clip_against(&mut self, edge: Edge3D) {\n let input = mem::replace(&mut self.subject, vec![]);\n let mut prev = match input.last() {\n None => return,\n Some(point) => *point,\n };\n for next in input {\n if edge.point_is_inside(next) {\n if !edge.point_is_inside(prev) {\n self.subject.push(edge.line_intersection(prev, next));\n }\n self.subject.push(next);\n } else if edge.point_is_inside(prev) {\n self.subject.push(edge.line_intersection(prev, next));\n }\n prev = next;\n }\n }\n}\n\n#[derive(Clone, Copy, Debug)]\nenum Edge3D {\n Left,\n Right,\n Bottom,\n Top,\n Near,\n Far,\n}\n\nimpl Edge3D {\n #[inline]\n fn point_is_inside(self, point: Vector4F) -> bool {\n let w = point.w();\n match self {\n Edge3D::Left => point.x() >= -w,\n Edge3D::Right => point.x() <= w,\n Edge3D::Bottom => point.y() >= -w,\n Edge3D::Top => point.y() <= w,\n Edge3D::Near => point.z() >= -w,\n Edge3D::Far => point.z() <= w,\n }\n }\n\n // Blinn & Newell, \"Clipping using homogeneous coordinates\", SIGGRAPH 1978.\n fn line_intersection(self, prev: Vector4F, next: Vector4F) -> Vector4F {\n let (x0, x1) = match self {\n Edge3D::Left | Edge3D::Right => (prev.x(), next.x()),\n Edge3D::Bottom | Edge3D::Top => (prev.y(), next.y()),\n Edge3D::Near | Edge3D::Far => (prev.z(), next.z()),\n };\n let (w0, w1) = (prev.w(), next.w());\n let sign = match self {\n Edge3D::Left | Edge3D::Bottom | Edge3D::Near => -1.0,\n Edge3D::Right | Edge3D::Top | Edge3D::Far => 1.0,\n };\n let alpha = ((x0 - sign * w0) as f64) / ((sign * (w1 - w0) - (x1 - x0)) as f64);\n prev.lerp(next, alpha as f32)\n }\n}\n\n/// Coarse collision detection\n\n// Separating axis theorem. Requires that the polygon be convex.\npub(crate) fn rect_is_outside_polygon(rect: RectF, polygon_points: &[Vector2F]) -> bool {\n let mut outcode = Outcode::all();\n for point in polygon_points {\n if point.x() > rect.min_x() {\n outcode.remove(Outcode::LEFT);\n }\n if point.x() < rect.max_x() {\n outcode.remove(Outcode::RIGHT);\n }\n if point.y() > rect.min_y() {\n outcode.remove(Outcode::TOP);\n }\n if point.y() < rect.max_y() {\n outcode.remove(Outcode::BOTTOM);\n }\n }\n if !outcode.is_empty() {\n return true;\n }\n\n // FIXME(pcwalton): Check winding!\n let rect_points = [\n rect.origin(),\n rect.upper_right(),\n rect.lower_left(),\n rect.lower_right(),\n ];\n for (next_point_index, &next) in polygon_points.iter().enumerate() {\n let prev_point_index = if next_point_index == 0 {\n polygon_points.len() - 1\n } else {\n next_point_index - 1\n };\n let prev = polygon_points[prev_point_index];\n let polygon_edge_vector = next - prev;\n if rect_points\n .iter()\n .all(|&rect_point| polygon_edge_vector.det(rect_point - prev) < 0.0)\n {\n return true;\n }\n }\n\n false\n}\n\n// Edge equation method. Requires that the polygon be convex.\npub(crate) fn rect_is_inside_polygon(rect: RectF, polygon_points: &[Vector2F]) -> bool {\n // FIXME(pcwalton): Check winding!\n let rect_points = [\n rect.origin(),\n rect.upper_right(),\n rect.lower_left(),\n rect.lower_right(),\n ];\n for (next_point_index, &next) in polygon_points.iter().enumerate() {\n let prev_point_index = if next_point_index == 0 {\n polygon_points.len() - 1\n } else {\n next_point_index - 1\n };\n let prev = polygon_points[prev_point_index];\n let polygon_edge_vector = next - prev;\n for &rect_point in &rect_points {\n if polygon_edge_vector.det(rect_point - prev) < 0.0 {\n return false;\n }\n }\n }\n\n true\n}\n\nbitflags! {\n struct Outcode: u8 {\n const LEFT = 0x01;\n const RIGHT = 0x02;\n const TOP = 0x04;\n const BOTTOM = 0x08;\n }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/dash.rs", "sha256": "1546525d73d08c01e0cf2074920fa0baa081ac89ac310a658dfba7174be5d61d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/dash.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Line dashing support.\n\nuse crate::outline::{Contour, ContourIterFlags, Outline, PushSegmentFlags};\nuse std::mem;\n\nconst EPSILON: f32 = 0.0001;\n\npub struct OutlineDash<'a> {\n input: &'a Outline,\n output: Outline,\n state: DashState<'a>,\n}\n\nimpl<'a> OutlineDash<'a> {\n #[inline]\n pub fn new(input: &'a Outline, dashes: &'a [f32], offset: f32) -> OutlineDash<'a> {\n OutlineDash { input, output: Outline::new(), state: DashState::new(dashes, offset) }\n }\n\n pub fn dash(&mut self) {\n for contour in &self.input.contours {\n ContourDash::new(contour, &mut self.output, &mut self.state).dash()\n }\n }\n\n pub fn into_outline(mut self) -> Outline {\n if self.state.is_on() {\n self.output.push_contour(self.state.output);\n }\n self.output\n }\n}\n\nstruct ContourDash<'a, 'b, 'c> {\n input: &'a Contour,\n output: &'b mut Outline,\n state: &'c mut DashState<'a>,\n}\n\nimpl<'a, 'b, 'c> ContourDash<'a, 'b, 'c> {\n fn new(input: &'a Contour, output: &'b mut Outline, state: &'c mut DashState<'a>)\n -> ContourDash<'a, 'b, 'c> {\n ContourDash { input, output, state }\n }\n\n fn dash(&mut self) {\n let mut iterator = self.input.iter(ContourIterFlags::empty());\n let mut queued_segment = None;\n loop {\n if queued_segment.is_none() {\n match iterator.next() {\n None => break,\n Some(segment) => queued_segment = Some(segment),\n }\n }\n\n let mut current_segment = queued_segment.take().unwrap();\n let mut distance = self.state.distance_left;\n\n let t = current_segment.time_for_distance(distance);\n if t < 1.0 {\n let (prev_segment, next_segment) = current_segment.split(t);\n current_segment = prev_segment;\n queued_segment = Some(next_segment);\n } else {\n distance = current_segment.arc_length();\n }\n\n if self.state.is_on() {\n self.state.output.push_segment(¤t_segment, PushSegmentFlags::empty());\n }\n\n self.state.distance_left -= distance;\n if self.state.distance_left < EPSILON {\n if self.state.is_on() {\n self.output.push_contour(mem::replace(&mut self.state.output, Contour::new()));\n }\n\n self.state.current_dash_index += 1;\n if self.state.current_dash_index == self.state.dashes.len() {\n self.state.current_dash_index = 0;\n }\n\n self.state.distance_left = self.state.dashes[self.state.current_dash_index];\n }\n }\n }\n}\n\nstruct DashState<'a> {\n output: Contour,\n dashes: &'a [f32],\n current_dash_index: usize,\n distance_left: f32,\n}\n\nimpl<'a> DashState<'a> {\n fn new(dashes: &'a [f32], mut offset: f32) -> DashState<'a> {\n let total: f32 = dashes.iter().cloned().sum();\n offset %= total;\n\n let mut current_dash_index = 0;\n while current_dash_index < dashes.len() {\n let dash = dashes[current_dash_index];\n if offset < dash {\n break;\n }\n offset -= dash;\n current_dash_index += 1;\n }\n\n DashState {\n output: Contour::new(),\n dashes,\n current_dash_index,\n distance_left: offset,\n }\n }\n\n #[inline]\n fn is_on(&self) -> bool {\n self.current_dash_index % 2 == 0\n }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/dilation.rs", "sha256": "6da19ab4af3a7c3298d5c97fa9a220e0b13f9a1ef27f9ee1b14f2bc38c5673c0", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/dilation.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::orientation::Orientation;\nuse crate::outline::Contour;\nuse pathfinder_geometry::vector::{Vector2F, vec2f};\n\npub struct ContourDilator<'a> {\n contour: &'a mut Contour,\n amount: Vector2F,\n orientation: Orientation,\n}\n\nimpl<'a> ContourDilator<'a> {\n pub fn new(\n contour: &'a mut Contour,\n amount: Vector2F,\n orientation: Orientation,\n ) -> ContourDilator<'a> {\n ContourDilator {\n contour,\n amount,\n orientation,\n }\n }\n\n pub fn dilate(&mut self) {\n // Determine orientation.\n let scale = self.amount * (match self.orientation {\n Orientation::Ccw => vec2f( 1.0, -1.0),\n Orientation::Cw => vec2f(-1.0, 1.0),\n });\n\n // Find the starting and previous positions.\n let first_position = self.contour.position_of(0);\n let mut prev_point_index = 0;\n let mut prev_position;\n\n loop {\n prev_point_index = self.contour.prev_point_index_of(prev_point_index);\n prev_position = self.contour.position_of(prev_point_index);\n if prev_point_index == 0 || prev_position != first_position {\n break;\n }\n }\n\n // Initialize our loop.\n let first_point_index = self.contour.next_point_index_of(prev_point_index);\n let mut current_point_index = first_point_index;\n let mut position = first_position;\n\n let mut prev_vector = (position - prev_position).normalize();\n\n loop {\n // Find the next non-degenerate position.\n let mut next_point_index = current_point_index;\n let mut next_position;\n loop {\n next_point_index = self.contour.next_point_index_of(next_point_index);\n if next_point_index == first_point_index {\n next_position = first_position;\n break;\n }\n next_position = self.contour.position_of(next_point_index);\n if next_point_index == current_point_index || next_position != position {\n break;\n }\n }\n let next_vector = (next_position - position).normalize();\n\n debug!(\n \"prev={} cur={} next={}\",\n prev_point_index, current_point_index, next_point_index\n );\n\n // Calculate new position by moving the point by the bisector.\n let bisector = prev_vector.yx() + next_vector.yx();\n let bisector_length = bisector.length();\n let scaled_bisector = if bisector_length == 0.0 {\n Vector2F::zero()\n } else {\n bisector * scale * (1.0 / bisector_length)\n };\n let new_position = position - scaled_bisector;\n\n debug!(\n \"dilate(): prev={}({:?}) cur={}({:?}) next={}({:?}) bisector={:?}({:?}, {:?})\",\n prev_point_index,\n prev_position,\n current_point_index,\n position,\n next_point_index,\n next_position,\n bisector,\n bisector_length,\n scaled_bisector\n );\n\n // Update all points.\n let mut point_index = current_point_index;\n while point_index != next_point_index {\n self.contour.points[point_index as usize] = new_position;\n debug!(\"... updating {:?}\", point_index);\n point_index = self.contour.next_point_index_of(point_index);\n }\n\n // Check to see if we're done.\n if next_point_index == first_point_index {\n break;\n }\n\n // Continue.\n prev_vector = next_vector;\n position = next_position;\n current_point_index = next_point_index;\n }\n }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/effects.rs", "sha256": "3ddbee73eab09b742743081707c47a64e26d4fd28d484ff70a841f495f38d12c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/effects.rs\n//\n// Copyright © 2020 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Special effects that can be applied to layers.\n\nuse pathfinder_color::ColorF;\nuse pathfinder_geometry::line_segment::LineSegment2F;\nuse pathfinder_geometry::vector::Vector2F;\nuse pathfinder_simd::default::F32x2;\n\n/// This intentionally does not precisely match what Core Graphics does (a\n/// Lanczos function), because we don't want any ringing artefacts.\npub static DEFRINGING_KERNEL_CORE_GRAPHICS: DefringingKernel =\n DefringingKernel([0.033165660, 0.102074051, 0.221434336, 0.286651906]);\npub static DEFRINGING_KERNEL_FREETYPE: DefringingKernel =\n DefringingKernel([0.0, 0.031372549, 0.301960784, 0.337254902]);\n\n/// Should match macOS 10.13 High Sierra.\npub static STEM_DARKENING_FACTORS: [f32; 2] = [0.0121, 0.0121 * 1.25];\n\n/// Should match macOS 10.13 High Sierra.\npub const MAX_STEM_DARKENING_AMOUNT: [f32; 2] = [0.3, 0.3];\n\n/// This value is a subjective cutoff. Above this ppem value, no stem darkening is performed.\npub const MAX_STEM_DARKENING_PIXELS_PER_EM: f32 = 72.0;\n\n/// The shader that should be used when compositing this layer onto its destination.\n#[derive(Clone, Copy, PartialEq, Debug)]\npub enum Filter {\n /// No special filter.\n None,\n\n /// Converts a linear gradient to a radial one.\n RadialGradient {\n /// The line that the circles lie along.\n line: LineSegment2F,\n /// The radii of the circles at the two endpoints.\n radii: F32x2,\n /// The origin of the linearized gradient in the texture.\n uv_origin: Vector2F,\n },\n\n PatternFilter(PatternFilter),\n}\n\n/// Shaders applicable to patterns.\n#[derive(Clone, Copy, PartialEq, Debug)]\npub enum PatternFilter {\n /// Performs postprocessing operations useful for monochrome text.\n Text {\n /// The foreground color of the text.\n fg_color: ColorF,\n /// The background color of the text.\n bg_color: ColorF,\n /// The kernel used for defringing, if subpixel AA is enabled.\n defringing_kernel: Option,\n /// Whether gamma correction is used when compositing.\n ///\n /// If this is enabled, stem darkening is advised.\n gamma_correction: bool,\n },\n\n /// A blur operation in one direction, either horizontal or vertical.\n ///\n /// To produce a full Gaussian blur, perform two successive blur operations, one in each\n /// direction.\n Blur {\n direction: BlurDirection,\n sigma: f32,\n },\n}\n\n/// Blend modes that can be applied to individual paths.\n#[derive(Clone, Copy, PartialEq, Debug)]\npub enum BlendMode {\n // Porter-Duff, supported by GPU blender\n Clear,\n Copy,\n SrcIn,\n SrcOut,\n SrcOver,\n SrcAtop,\n DestIn,\n DestOut,\n DestOver,\n DestAtop,\n Xor,\n Lighter,\n\n // Others, unsupported by GPU blender\n Darken,\n Lighten,\n Multiply,\n Screen,\n HardLight,\n Overlay,\n ColorDodge,\n ColorBurn,\n SoftLight,\n Difference,\n Exclusion,\n Hue,\n Saturation,\n Color,\n Luminosity,\n}\n\n#[derive(Clone, Copy, PartialEq, Debug)]\npub struct DefringingKernel(pub [f32; 4]);\n\n#[derive(Clone, Copy, PartialEq, Debug)]\npub enum BlurDirection {\n X,\n Y,\n}\n\nimpl Default for BlendMode {\n #[inline]\n fn default() -> BlendMode {\n BlendMode::SrcOver\n }\n}\n\nimpl Default for Filter {\n #[inline]\n fn default() -> Filter {\n Filter::None\n }\n}\n\nimpl BlendMode {\n /// Whether the backdrop is irrelevant when applying this blend mode (i.e. destination blend\n /// factor is zero when source alpha is one).\n #[inline]\n pub fn occludes_backdrop(self) -> bool {\n match self {\n BlendMode::SrcOver | BlendMode::Clear => true,\n BlendMode::DestOver |\n BlendMode::DestOut |\n BlendMode::SrcAtop |\n BlendMode::Xor |\n BlendMode::Lighter |\n BlendMode::Lighten |\n BlendMode::Darken |\n BlendMode::Copy |\n BlendMode::SrcIn |\n BlendMode::DestIn |\n BlendMode::SrcOut |\n BlendMode::DestAtop |\n BlendMode::Multiply |\n BlendMode::Screen |\n BlendMode::HardLight |\n BlendMode::Overlay |\n BlendMode::ColorDodge |\n BlendMode::ColorBurn |\n BlendMode::SoftLight |\n BlendMode::Difference |\n BlendMode::Exclusion |\n BlendMode::Hue |\n BlendMode::Saturation |\n BlendMode::Color |\n BlendMode::Luminosity => false,\n }\n }\n\n /// True if this blend mode does not preserve destination areas outside the source.\n pub fn is_destructive(self) -> bool {\n match self {\n BlendMode::Clear |\n BlendMode::Copy |\n BlendMode::SrcIn |\n BlendMode::DestIn |\n BlendMode::SrcOut |\n BlendMode::DestAtop => true,\n BlendMode::SrcOver |\n BlendMode::DestOver |\n BlendMode::DestOut |\n BlendMode::SrcAtop |\n BlendMode::Xor |\n BlendMode::Lighter |\n BlendMode::Lighten |\n BlendMode::Darken |\n BlendMode::Multiply |\n BlendMode::Screen |\n BlendMode::HardLight |\n BlendMode::Overlay |\n BlendMode::ColorDodge |\n BlendMode::ColorBurn |\n BlendMode::SoftLight |\n BlendMode::Difference |\n BlendMode::Exclusion |\n BlendMode::Hue |\n BlendMode::Saturation |\n BlendMode::Color |\n BlendMode::Luminosity => false,\n }\n }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/fill.rs", "sha256": "9deb3e9a5059b44baae99054c7c4e154287080f640e3e4529154af65fc40e3ee", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/fill.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Fill rules.\n\n#[derive(Clone, Copy, PartialEq, Debug)]\npub enum FillRule {\n Winding,\n EvenOdd,\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/gradient.rs", "sha256": "87c7a30bd20213b7e65f53e5e842bec6a9b58695516d295253aaa5468e1daf48", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/gradient.rs\n//\n// Copyright © 2020 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::sorted_vector::SortedVector;\nuse crate::util;\nuse pathfinder_color::ColorU;\nuse pathfinder_geometry::line_segment::LineSegment2F;\nuse pathfinder_geometry::vector::Vector2F;\nuse pathfinder_geometry::util as geometry_util;\nuse pathfinder_simd::default::F32x2;\nuse std::cmp::{Ordering, PartialOrd};\nuse std::convert;\nuse std::hash::{Hash, Hasher};\nuse std::mem;\n\n#[derive(Clone, PartialEq, Debug)]\npub struct Gradient {\n /// The line this gradient runs along.\n /// \n /// If this is a radial gradient, this is the line that connects the two circles. It may have\n /// zero-length in the case of simple radial gradients.\n pub line: LineSegment2F,\n /// For radial gradients, the radii of the start and endpoints respectively. If this is a\n /// linear gradient, this is `None`.\n pub radii: Option,\n stops: SortedVector,\n}\n\n#[derive(Clone, Copy, PartialEq, PartialOrd, Debug)]\npub struct ColorStop {\n pub offset: f32,\n pub color: ColorU,\n}\n\nimpl Eq for Gradient {}\n\nimpl Hash for Gradient {\n fn hash(&self, state: &mut H) where H: Hasher {\n util::hash_line_segment(self.line, state);\n match self.radii {\n None => (0).hash(state),\n Some(radii) => {\n (1).hash(state);\n util::hash_f32(radii.x(), state);\n util::hash_f32(radii.y(), state);\n }\n }\n self.stops.hash(state);\n }\n}\n\nimpl Eq for ColorStop {}\n\nimpl Hash for ColorStop {\n fn hash(&self, state: &mut H) where H: Hasher {\n unsafe {\n self.color.hash(state);\n let offset = mem::transmute::(self.offset);\n offset.hash(state);\n }\n }\n}\n\nimpl Gradient {\n #[inline]\n pub fn linear(line: LineSegment2F) -> Gradient {\n Gradient { line, radii: None, stops: SortedVector::new() }\n }\n\n #[inline]\n pub fn linear_from_points(from: Vector2F, to: Vector2F) -> Gradient {\n Gradient::linear(LineSegment2F::new(from, to))\n }\n\n #[inline]\n pub fn radial(line: L, radii: F32x2) -> Gradient where L: RadialGradientLine {\n Gradient { line: line.to_line(), radii: Some(radii), stops: SortedVector::new() }\n }\n\n #[inline]\n pub fn add(&mut self, stop: ColorStop) {\n self.stops.push(stop);\n }\n\n /// A convenience method to add a color stop.\n #[inline]\n pub fn add_color_stop(&mut self, color: ColorU, offset: f32) {\n self.add(ColorStop::new(color, offset))\n }\n\n #[inline]\n pub fn line(&self) -> LineSegment2F {\n self.line\n }\n\n #[inline]\n pub fn set_line(&mut self, line: LineSegment2F) {\n self.line = line\n }\n\n #[inline]\n pub fn radii(&self) -> Option {\n self.radii\n }\n\n #[inline]\n pub fn set_radii(&mut self, radii: Option) {\n self.radii = radii\n }\n\n #[inline]\n pub fn stops(&self) -> &[ColorStop] {\n &self.stops.array\n }\n\n #[inline]\n pub fn stops_mut(&mut self) -> &mut [ColorStop] {\n &mut self.stops.array\n }\n\n pub fn sample(&self, mut t: f32) -> ColorU {\n if self.stops.is_empty() {\n return ColorU::transparent_black();\n }\n\n t = geometry_util::clamp(t, 0.0, 1.0);\n let last_index = self.stops.len() - 1;\n let upper_index = self.stops.binary_search_by(|stop| {\n stop.offset.partial_cmp(&t).unwrap_or(Ordering::Less)\n }).unwrap_or_else(convert::identity).min(last_index);\n let lower_index = if upper_index > 0 { upper_index - 1 } else { upper_index };\n\n let lower_stop = &self.stops.array[lower_index];\n let upper_stop = &self.stops.array[upper_index];\n\n let denom = upper_stop.offset - lower_stop.offset;\n if denom == 0.0 {\n return lower_stop.color;\n }\n\n lower_stop.color\n .to_f32()\n .lerp(upper_stop.color.to_f32(), (t - lower_stop.offset) / denom)\n .to_u8()\n }\n\n #[inline]\n pub fn is_opaque(&self) -> bool {\n self.stops.array.iter().all(|stop| stop.color.is_opaque())\n }\n\n #[inline]\n pub fn is_fully_transparent(&self) -> bool {\n self.stops.array.iter().all(|stop| stop.color.is_fully_transparent())\n }\n}\n\nimpl ColorStop {\n #[inline]\n pub fn new(color: ColorU, offset: f32) -> ColorStop {\n ColorStop { color, offset }\n }\n}\n\npub trait RadialGradientLine {\n fn to_line(self) -> LineSegment2F;\n}\n\nimpl RadialGradientLine for LineSegment2F {\n #[inline]\n fn to_line(self) -> LineSegment2F {\n self\n }\n}\n\nimpl RadialGradientLine for Vector2F {\n #[inline]\n fn to_line(self) -> LineSegment2F {\n LineSegment2F::new(self, self)\n }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/lib.rs", "sha256": "a6d51f9a74c053d4fe73b21876363e26a109125dfd77dd4a53101789cf700fee", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/lib.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Pathfinder's representation of a vector scene.\n//!\n//! This module also contains various path utilities.\n\n#[macro_use]\nextern crate bitflags;\n#[macro_use]\nextern crate log;\n\npub mod clip;\npub mod dash;\npub mod effects;\npub mod fill;\npub mod gradient;\npub mod orientation;\npub mod outline;\npub mod pattern;\npub mod render_target;\npub mod segment;\npub mod sorted_vector;\npub mod stroke;\npub mod transform;\n\nmod dilation;\nmod util;\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/orientation.rs", "sha256": "cc5b896f47629ace0ff760b4b923fc616e63101528f65030d914297b4c44d044", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/geometry/src/orientation.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::outline::Outline;\n\n#[derive(Clone, Copy, Debug, PartialEq)]\npub enum Orientation {\n Ccw = -1,\n Cw = 1,\n}\n\nimpl Orientation {\n /// This follows the FreeType algorithm.\n pub fn from_outline(outline: &Outline) -> Orientation {\n let mut area = 0.0;\n for contour in &outline.contours {\n let mut prev_position = match contour.last_position() {\n None => continue,\n Some(position) => position,\n };\n for &next_position in &contour.points {\n area += prev_position.det(next_position);\n prev_position = next_position;\n }\n }\n Orientation::from_area(area)\n }\n\n fn from_area(area: f32) -> Orientation {\n if area <= 0.0 {\n Orientation::Ccw\n } else {\n Orientation::Cw\n }\n }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/outline.rs", "sha256": "57eab3536a5a3a21fbb819342d3d6a1bf396d7da36de93e0db791aa3ae7665dc", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/outline.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! A compressed in-memory representation of paths.\n\nuse crate::clip::{self, ContourPolygonClipper, ContourRectClipper};\nuse crate::dilation::ContourDilator;\nuse crate::orientation::Orientation;\nuse crate::segment::{Segment, SegmentFlags, SegmentKind};\nuse pathfinder_geometry::line_segment::LineSegment2F;\nuse pathfinder_geometry::rect::RectF;\nuse pathfinder_geometry::transform2d::Transform2F;\nuse pathfinder_geometry::transform3d::Perspective;\nuse pathfinder_geometry::unit_vector::UnitVector;\nuse pathfinder_geometry::vector::{Vector2F, vec2f};\nuse std::f32::consts::PI;\nuse std::fmt::{self, Debug, Formatter};\nuse std::mem;\n\n#[derive(Clone)]\npub struct Outline {\n pub(crate) contours: Vec,\n pub(crate) bounds: RectF,\n}\n\n#[derive(Clone)]\npub struct Contour {\n pub(crate) points: Vec,\n pub(crate) flags: Vec,\n pub(crate) bounds: RectF,\n pub(crate) closed: bool,\n}\n\nbitflags! {\n pub struct PointFlags: u8 {\n const CONTROL_POINT_0 = 0x01;\n const CONTROL_POINT_1 = 0x02;\n }\n}\n\nbitflags! {\n pub struct PushSegmentFlags: u8 {\n const UPDATE_BOUNDS = 0x01;\n const INCLUDE_FROM_POINT = 0x02;\n }\n}\n\nimpl Outline {\n #[inline]\n pub fn new() -> Outline {\n Outline {\n contours: vec![],\n bounds: RectF::default(),\n }\n }\n\n #[inline]\n pub fn from_segments(segments: I) -> Outline\n where\n I: Iterator,\n {\n let mut outline = Outline::new();\n let mut current_contour = Contour::new();\n\n for segment in segments {\n if segment.flags.contains(SegmentFlags::FIRST_IN_SUBPATH) {\n if !current_contour.is_empty() {\n outline\n .contours\n .push(mem::replace(&mut current_contour, Contour::new()));\n }\n current_contour.push_point(segment.baseline.from(), PointFlags::empty(), true);\n }\n\n if segment.flags.contains(SegmentFlags::CLOSES_SUBPATH) {\n if !current_contour.is_empty() {\n current_contour.close();\n let contour = mem::replace(&mut current_contour, Contour::new());\n outline.push_contour(contour);\n }\n continue;\n }\n\n if segment.is_none() {\n continue;\n }\n\n if !segment.is_line() {\n current_contour.push_point(segment.ctrl.from(), PointFlags::CONTROL_POINT_0, true);\n if !segment.is_quadratic() {\n current_contour.push_point(\n segment.ctrl.to(),\n PointFlags::CONTROL_POINT_1,\n true,\n );\n }\n }\n\n current_contour.push_point(segment.baseline.to(), PointFlags::empty(), true);\n }\n\n outline.push_contour(current_contour);\n outline\n }\n\n #[inline]\n pub fn from_rect(rect: RectF) -> Outline {\n let mut outline = Outline::new();\n outline.push_contour(Contour::from_rect(rect));\n outline\n }\n\n #[inline]\n pub fn bounds(&self) -> RectF {\n self.bounds\n }\n\n #[inline]\n pub fn contours(&self) -> &[Contour] {\n &self.contours\n }\n\n #[inline]\n pub fn into_contours(self) -> Vec {\n self.contours\n }\n\n /// Removes all contours from this outline.\n #[inline]\n pub fn clear(&mut self) {\n self.contours.clear();\n self.bounds = RectF::default();\n }\n\n pub fn push_contour(&mut self, contour: Contour) {\n if contour.is_empty() {\n return;\n }\n\n if self.contours.is_empty() {\n self.bounds = contour.bounds;\n } else {\n self.bounds = self.bounds.union_rect(contour.bounds);\n }\n\n self.contours.push(contour);\n }\n\n pub fn pop_contour(&mut self) -> Option {\n let last_contour = self.contours.pop();\n\n let mut new_bounds = None;\n for contour in &mut self.contours {\n contour.update_bounds(&mut new_bounds);\n }\n self.bounds = new_bounds.unwrap_or_else(|| RectF::default());\n\n last_contour\n }\n\n pub fn transform(&mut self, transform: &Transform2F) {\n if transform.is_identity() {\n return;\n }\n\n let mut new_bounds = None;\n for contour in &mut self.contours {\n contour.transform(transform);\n contour.update_bounds(&mut new_bounds);\n }\n self.bounds = new_bounds.unwrap_or_else(|| RectF::default());\n }\n\n pub fn apply_perspective(&mut self, perspective: &Perspective) {\n let mut new_bounds = None;\n for contour in &mut self.contours {\n contour.apply_perspective(perspective);\n contour.update_bounds(&mut new_bounds);\n }\n self.bounds = new_bounds.unwrap_or_else(|| RectF::default());\n }\n\n pub fn dilate(&mut self, amount: Vector2F) {\n let orientation = Orientation::from_outline(self);\n self.contours\n .iter_mut()\n .for_each(|contour| contour.dilate(amount, orientation));\n self.bounds = self.bounds.dilate(amount);\n }\n\n pub fn prepare_for_tiling(&mut self, view_box: RectF) {\n self.contours\n .iter_mut()\n .for_each(|contour| contour.prepare_for_tiling(view_box));\n self.bounds = self\n .bounds\n .intersection(view_box)\n .unwrap_or_else(|| RectF::default());\n }\n\n pub fn is_outside_polygon(&self, clip_polygon: &[Vector2F]) -> bool {\n clip::rect_is_outside_polygon(self.bounds, clip_polygon)\n }\n\n fn is_inside_polygon(&self, clip_polygon: &[Vector2F]) -> bool {\n clip::rect_is_inside_polygon(self.bounds, clip_polygon)\n }\n\n pub fn clip_against_polygon(&mut self, clip_polygon: &[Vector2F]) {\n // Quick check.\n if self.is_inside_polygon(clip_polygon) {\n return;\n }\n\n for contour in mem::replace(&mut self.contours, vec![]) {\n self.push_contour(ContourPolygonClipper::new(clip_polygon, contour).clip());\n }\n }\n\n pub fn clip_against_rect(&mut self, clip_rect: RectF) {\n if clip_rect.contains_rect(self.bounds) {\n return;\n }\n\n for contour in mem::replace(&mut self.contours, vec![]) {\n self.push_contour(ContourRectClipper::new(clip_rect, contour).clip());\n }\n }\n\n #[inline]\n pub fn close_all_contours(&mut self) {\n self.contours.iter_mut().for_each(|contour| contour.close());\n }\n}\n\nimpl Debug for Outline {\n fn fmt(&self, formatter: &mut Formatter) -> fmt::Result {\n for (contour_index, contour) in self.contours.iter().enumerate() {\n if contour_index > 0 {\n write!(formatter, \" \")?;\n }\n contour.fmt(formatter)?;\n }\n Ok(())\n }\n}\n\nimpl Contour {\n #[inline]\n pub fn new() -> Contour {\n Contour {\n points: vec![],\n flags: vec![],\n bounds: RectF::default(),\n closed: false,\n }\n }\n\n #[inline]\n pub fn with_capacity(length: usize) -> Contour {\n Contour {\n points: Vec::with_capacity(length),\n flags: Vec::with_capacity(length),\n bounds: RectF::default(),\n closed: false,\n }\n }\n\n #[inline]\n pub fn from_rect(rect: RectF) -> Contour {\n let mut contour = Contour::new();\n contour.push_point(rect.origin(), PointFlags::empty(), false);\n contour.push_point(rect.upper_right(), PointFlags::empty(), false);\n contour.push_point(rect.lower_right(), PointFlags::empty(), false);\n contour.push_point(rect.lower_left(), PointFlags::empty(), false);\n contour.close();\n contour.bounds = rect;\n contour\n }\n\n // Replaces this contour with a new one, with arrays preallocated to match `self`.\n #[inline]\n pub(crate) fn take(&mut self) -> Contour {\n let length = self.len() as usize;\n mem::replace(\n self,\n Contour {\n points: Vec::with_capacity(length),\n flags: Vec::with_capacity(length),\n bounds: RectF::default(),\n closed: false,\n },\n )\n }\n\n /// restore self to the state of Contour::new(), but keep the points buffer allocated\n #[inline]\n pub fn clear(&mut self) {\n self.points.clear();\n self.flags.clear();\n self.bounds = RectF::default();\n self.closed = false;\n }\n\n #[inline]\n pub fn iter(&self, flags: ContourIterFlags) -> ContourIter {\n ContourIter {\n contour: self,\n index: 1,\n flags,\n }\n }\n\n #[inline]\n pub fn is_empty(&self) -> bool {\n self.points.is_empty()\n }\n\n #[inline]\n pub fn len(&self) -> u32 {\n self.points.len() as u32\n }\n\n #[inline]\n pub fn bounds(&self) -> RectF {\n self.bounds\n }\n\n #[inline]\n pub fn is_closed(&self) -> bool {\n self.closed\n }\n\n #[inline]\n pub fn position_of(&self, index: u32) -> Vector2F {\n self.points[index as usize]\n }\n\n #[inline]\n pub fn last_position(&self) -> Option {\n self.points.last().cloned()\n }\n\n #[inline]\n pub(crate) fn position_of_last(&self, index: u32) -> Vector2F {\n self.points[self.points.len() - index as usize]\n }\n\n #[inline]\n pub fn push_endpoint(&mut self, point: Vector2F) {\n self.push_point(point, PointFlags::empty(), true);\n }\n\n #[inline]\n pub fn push_quadratic(&mut self, ctrl: Vector2F, point: Vector2F) {\n self.push_point(ctrl, PointFlags::CONTROL_POINT_0, true);\n self.push_point(point, PointFlags::empty(), true);\n }\n\n #[inline]\n pub fn push_cubic(&mut self, ctrl0: Vector2F, ctrl1: Vector2F, point: Vector2F) {\n self.push_point(ctrl0, PointFlags::CONTROL_POINT_0, true);\n self.push_point(ctrl1, PointFlags::CONTROL_POINT_1, true);\n self.push_point(point, PointFlags::empty(), true);\n }\n\n #[inline]\n pub fn close(&mut self) {\n self.closed = true;\n }\n\n #[inline]\n pub(crate) fn push_point(&mut self,\n point: Vector2F,\n flags: PointFlags,\n update_bounds: bool) {\n debug_assert!(!point.x().is_nan() && !point.y().is_nan());\n\n if update_bounds {\n let first = self.is_empty();\n union_rect(&mut self.bounds, point, first);\n }\n\n self.points.push(point);\n self.flags.push(flags);\n }\n\n #[inline]\n pub(crate) fn push_segment(&mut self, segment: &Segment, flags: PushSegmentFlags) {\n if segment.is_none() {\n return;\n }\n\n let update_bounds = flags.contains(PushSegmentFlags::UPDATE_BOUNDS);\n self.push_point(segment.baseline.from(), PointFlags::empty(), update_bounds);\n\n if !segment.is_line() {\n self.push_point(\n segment.ctrl.from(),\n PointFlags::CONTROL_POINT_0,\n update_bounds,\n );\n if !segment.is_quadratic() {\n self.push_point(\n segment.ctrl.to(),\n PointFlags::CONTROL_POINT_1,\n update_bounds,\n );\n }\n }\n\n self.push_point(segment.baseline.to(), PointFlags::empty(), update_bounds);\n }\n\n pub fn push_arc(&mut self,\n transform: &Transform2F,\n start_angle: f32,\n end_angle: f32,\n direction: ArcDirection) {\n if end_angle - start_angle >= PI * 2.0 {\n self.push_ellipse(transform);\n } else {\n let start = vec2f(start_angle.cos(), start_angle.sin());\n let end = vec2f(end_angle.cos(), end_angle.sin());\n self.push_arc_from_unit_chord(transform, LineSegment2F::new(start, end), direction);\n }\n }\n\n pub fn push_arc_from_unit_chord(&mut self,\n transform: &Transform2F,\n mut chord: LineSegment2F,\n direction: ArcDirection) {\n let mut direction_transform = Transform2F::default();\n if direction == ArcDirection::CCW {\n chord *= vec2f(1.0, -1.0);\n direction_transform = Transform2F::from_scale(vec2f(1.0, -1.0));\n }\n\n let (mut vector, end_vector) = (UnitVector(chord.from()), UnitVector(chord.to()));\n for segment_index in 0..4 {\n debug!(\"push_arc_from_unit_chord(): loop segment index {}\", segment_index);\n\n let mut sweep_vector = end_vector.rev_rotate_by(vector);\n let last = sweep_vector.0.x() >= -EPSILON && sweep_vector.0.y() >= -EPSILON;\n debug!(\"... end_vector={:?} vector={:?} sweep_vector={:?} last={:?}\",\n end_vector,\n vector,\n sweep_vector,\n last);\n\n let mut segment;\n if !last {\n sweep_vector = UnitVector(vec2f(0.0, 1.0));\n segment = Segment::quarter_circle_arc();\n } else {\n segment = Segment::arc_from_cos(sweep_vector.0.x());\n }\n\n let half_sweep_vector = sweep_vector.halve_angle();\n let rotation = Transform2F::from_rotation_vector(half_sweep_vector.rotate_by(vector));\n segment = segment.transform(&(*transform * direction_transform * rotation));\n\n let mut push_segment_flags = PushSegmentFlags::UPDATE_BOUNDS;\n if segment_index == 0 {\n push_segment_flags.insert(PushSegmentFlags::INCLUDE_FROM_POINT);\n }\n self.push_segment(&segment, push_segment_flags);\n\n if last {\n break;\n }\n\n vector = vector.rotate_by(sweep_vector);\n }\n\n const EPSILON: f32 = 0.001;\n }\n\n pub fn push_ellipse(&mut self, transform: &Transform2F) {\n let segment = Segment::quarter_circle_arc();\n let mut rotation;\n self.push_segment(&segment.transform(transform),\n PushSegmentFlags::UPDATE_BOUNDS | PushSegmentFlags::INCLUDE_FROM_POINT);\n rotation = Transform2F::from_rotation_vector(UnitVector(vec2f( 0.0, 1.0)));\n self.push_segment(&segment.transform(&(*transform * rotation)),\n PushSegmentFlags::UPDATE_BOUNDS);\n rotation = Transform2F::from_rotation_vector(UnitVector(vec2f(-1.0, 0.0)));\n self.push_segment(&segment.transform(&(*transform * rotation)),\n PushSegmentFlags::UPDATE_BOUNDS);\n rotation = Transform2F::from_rotation_vector(UnitVector(vec2f( 0.0, -1.0)));\n self.push_segment(&segment.transform(&(*transform * rotation)),\n PushSegmentFlags::UPDATE_BOUNDS);\n }\n\n #[inline]\n pub fn segment_after(&self, point_index: u32) -> Segment {\n debug_assert!(self.point_is_endpoint(point_index));\n\n let mut segment = Segment::none();\n segment.baseline.set_from(self.position_of(point_index));\n\n let point1_index = self.add_to_point_index(point_index, 1);\n if self.point_is_endpoint(point1_index) {\n segment.baseline.set_to(self.position_of(point1_index));\n segment.kind = SegmentKind::Line;\n } else {\n segment.ctrl.set_from(self.position_of(point1_index));\n\n let point2_index = self.add_to_point_index(point_index, 2);\n if self.point_is_endpoint(point2_index) {\n segment.baseline.set_to(self.position_of(point2_index));\n segment.kind = SegmentKind::Quadratic;\n } else {\n segment.ctrl.set_to(self.position_of(point2_index));\n segment.kind = SegmentKind::Cubic;\n\n let point3_index = self.add_to_point_index(point_index, 3);\n segment.baseline.set_to(self.position_of(point3_index));\n }\n }\n\n segment\n }\n\n #[inline]\n pub fn hull_segment_after(&self, prev_point_index: u32) -> LineSegment2F {\n let next_point_index = self.next_point_index_of(prev_point_index);\n LineSegment2F::new(\n self.points[prev_point_index as usize],\n self.points[next_point_index as usize],\n )\n }\n\n #[inline]\n pub fn point_is_endpoint(&self, point_index: u32) -> bool {\n !self.flags[point_index as usize]\n .intersects(PointFlags::CONTROL_POINT_0 | PointFlags::CONTROL_POINT_1)\n }\n\n #[inline]\n pub fn add_to_point_index(&self, point_index: u32, addend: u32) -> u32 {\n let (index, limit) = (point_index + addend, self.len());\n if index >= limit {\n index - limit\n } else {\n index\n }\n }\n\n #[inline]\n pub fn point_is_logically_above(&self, a: u32, b: u32) -> bool {\n let (a_y, b_y) = (self.points[a as usize].y(), self.points[b as usize].y());\n a_y < b_y || (a_y == b_y && a < b)\n }\n\n #[inline]\n pub fn prev_endpoint_index_of(&self, mut point_index: u32) -> u32 {\n loop {\n point_index = self.prev_point_index_of(point_index);\n if self.point_is_endpoint(point_index) {\n return point_index;\n }\n }\n }\n\n #[inline]\n pub fn next_endpoint_index_of(&self, mut point_index: u32) -> u32 {\n loop {\n point_index = self.next_point_index_of(point_index);\n if self.point_is_endpoint(point_index) {\n return point_index;\n }\n }\n }\n\n #[inline]\n pub fn prev_point_index_of(&self, point_index: u32) -> u32 {\n if point_index == 0 {\n self.len() - 1\n } else {\n point_index - 1\n }\n }\n\n #[inline]\n pub fn next_point_index_of(&self, point_index: u32) -> u32 {\n if point_index == self.len() - 1 {\n 0\n } else {\n point_index + 1\n }\n }\n\n pub fn transform(&mut self, transform: &Transform2F) {\n if transform.is_identity() {\n return;\n }\n\n for (point_index, point) in self.points.iter_mut().enumerate() {\n *point = *transform * *point;\n union_rect(&mut self.bounds, *point, point_index == 0);\n }\n }\n\n pub fn apply_perspective(&mut self, perspective: &Perspective) {\n for (point_index, point) in self.points.iter_mut().enumerate() {\n *point = *perspective * *point;\n union_rect(&mut self.bounds, *point, point_index == 0);\n }\n }\n\n pub fn dilate(&mut self, amount: Vector2F, orientation: Orientation) {\n ContourDilator::new(self, amount, orientation).dilate();\n self.bounds = self.bounds.dilate(amount);\n }\n\n fn prepare_for_tiling(&mut self, view_box: RectF) {\n // Snap points to the view box bounds. This mops up floating point error from the clipping\n // process.\n let (mut last_endpoint_index, mut contour_is_monotonic) = (None, true);\n for point_index in 0..(self.points.len() as u32) {\n if contour_is_monotonic {\n if self.point_is_endpoint(point_index) {\n if let Some(last_endpoint_index) = last_endpoint_index {\n if !self.curve_with_endpoints_is_monotonic(last_endpoint_index,\n point_index) {\n contour_is_monotonic = false;\n }\n }\n last_endpoint_index = Some(point_index);\n }\n }\n }\n\n // Convert to monotonic, if necessary.\n if !contour_is_monotonic {\n self.make_monotonic();\n }\n\n // Update bounds.\n self.bounds = self\n .bounds\n .intersection(view_box)\n .unwrap_or_else(|| RectF::default());\n }\n\n fn make_monotonic(&mut self) {\n debug!(\"--- make_monotonic() ---\");\n\n let contour = self.take();\n self.bounds = contour.bounds;\n\n let mut last_endpoint_index = None;\n let input_point_count = contour.points.len() as u32;\n for point_index in 0..(input_point_count + 1) {\n if point_index < input_point_count && !contour.point_is_endpoint(point_index) {\n continue;\n }\n\n if let Some(last_endpoint_index) = last_endpoint_index {\n let position_index = if point_index == input_point_count {\n 0\n } else {\n point_index\n };\n let baseline = LineSegment2F::new(\n contour.points[last_endpoint_index as usize],\n contour.points[position_index as usize],\n );\n let point_count = point_index - last_endpoint_index + 1;\n if point_count == 3 {\n let ctrl_point_index = last_endpoint_index as usize + 1;\n let ctrl_position = &contour.points[ctrl_point_index];\n handle_cubic(\n self,\n &Segment::quadratic(baseline, *ctrl_position).to_cubic(),\n );\n } else if point_count == 4 {\n let first_ctrl_point_index = last_endpoint_index as usize + 1;\n let ctrl_position_0 = &contour.points[first_ctrl_point_index + 0];\n let ctrl_position_1 = &contour.points[first_ctrl_point_index + 1];\n let ctrl = LineSegment2F::new(*ctrl_position_0, *ctrl_position_1);\n handle_cubic(self, &Segment::cubic(baseline, ctrl));\n }\n\n self.push_point(\n contour.points[position_index as usize],\n PointFlags::empty(),\n false,\n );\n }\n\n last_endpoint_index = Some(point_index);\n }\n\n fn handle_cubic(contour: &mut Contour, segment: &Segment) {\n debug!(\"handle_cubic({:?})\", segment);\n\n match segment.as_cubic_segment().y_extrema() {\n (Some(t0), Some(t1)) => {\n let (segments_01, segment_2) = segment.as_cubic_segment().split(t1);\n let (segment_0, segment_1) = segments_01.as_cubic_segment().split(t0 / t1);\n contour.push_segment(&segment_0, PushSegmentFlags::empty());\n contour.push_segment(&segment_1, PushSegmentFlags::empty());\n contour.push_segment(&segment_2, PushSegmentFlags::empty());\n }\n (Some(t0), None) | (None, Some(t0)) => {\n let (segment_0, segment_1) = segment.as_cubic_segment().split(t0);\n contour.push_segment(&segment_0, PushSegmentFlags::empty());\n contour.push_segment(&segment_1, PushSegmentFlags::empty());\n }\n (None, None) => contour.push_segment(segment, PushSegmentFlags::empty()),\n }\n }\n }\n\n fn curve_with_endpoints_is_monotonic(\n &self,\n start_endpoint_index: u32,\n end_endpoint_index: u32,\n ) -> bool {\n let start_position = self.points[start_endpoint_index as usize];\n let end_position = self.points[end_endpoint_index as usize];\n\n if start_position.x() <= end_position.x() {\n for point_index in start_endpoint_index..end_endpoint_index {\n if self.points[point_index as usize].x() > self.points[point_index as usize + 1].x()\n {\n return false;\n }\n }\n } else {\n for point_index in start_endpoint_index..end_endpoint_index {\n if self.points[point_index as usize].x() < self.points[point_index as usize + 1].x()\n {\n return false;\n }\n }\n }\n\n if start_position.y() <= end_position.y() {\n for point_index in start_endpoint_index..end_endpoint_index {\n if self.points[point_index as usize].y() > self.points[point_index as usize + 1].y()\n {\n return false;\n }\n }\n } else {\n for point_index in start_endpoint_index..end_endpoint_index {\n if self.points[point_index as usize].y() < self.points[point_index as usize + 1].y()\n {\n return false;\n }\n }\n }\n\n true\n }\n\n // Use this function to keep bounds up to date when mutating paths. See `Outline::transform()`\n // for an example of use.\n pub(crate) fn update_bounds(&self, bounds: &mut Option) {\n *bounds = Some(match *bounds {\n None => self.bounds,\n Some(bounds) => bounds.union_rect(self.bounds),\n })\n }\n}\n\nimpl Debug for Contour {\n fn fmt(&self, formatter: &mut Formatter) -> fmt::Result {\n for (segment_index, segment) in self.iter(ContourIterFlags::IGNORE_CLOSE_SEGMENT)\n .enumerate() {\n if segment_index == 0 {\n write!(\n formatter,\n \"M {} {}\",\n segment.baseline.from_x(),\n segment.baseline.from_y()\n )?;\n }\n\n match segment.kind {\n SegmentKind::None => {}\n SegmentKind::Line => {\n write!(\n formatter,\n \" L {} {}\",\n segment.baseline.to_x(),\n segment.baseline.to_y()\n )?;\n }\n SegmentKind::Quadratic => {\n write!(\n formatter,\n \" Q {} {} {} {}\",\n segment.ctrl.from_x(),\n segment.ctrl.from_y(),\n segment.baseline.to_x(),\n segment.baseline.to_y()\n )?;\n }\n SegmentKind::Cubic => {\n write!(\n formatter,\n \" C {} {} {} {} {} {}\",\n segment.ctrl.from_x(),\n segment.ctrl.from_y(),\n segment.ctrl.to_x(),\n segment.ctrl.to_y(),\n segment.baseline.to_x(),\n segment.baseline.to_y()\n )?;\n }\n }\n }\n\n if self.closed {\n write!(formatter, \" z\")?;\n }\n\n Ok(())\n }\n}\n\n#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]\npub struct PointIndex(u32);\n\nimpl PointIndex {\n #[inline]\n pub fn new(contour: u32, point: u32) -> PointIndex {\n debug_assert!(contour <= 0xfff);\n debug_assert!(point <= 0x000f_ffff);\n PointIndex((contour << 20) | point)\n }\n\n #[inline]\n pub fn contour(self) -> u32 {\n self.0 >> 20\n }\n\n #[inline]\n pub fn point(self) -> u32 {\n self.0 & 0x000f_ffff\n }\n}\n\npub struct ContourIter<'a> {\n contour: &'a Contour,\n index: u32,\n flags: ContourIterFlags,\n}\n\nimpl<'a> Iterator for ContourIter<'a> {\n type Item = Segment;\n\n #[inline]\n fn next(&mut self) -> Option {\n let contour = self.contour;\n\n let include_close_segment = self.contour.closed &&\n !self.flags.contains(ContourIterFlags::IGNORE_CLOSE_SEGMENT);\n if (self.index == contour.len() && !include_close_segment) ||\n self.index == contour.len() + 1 {\n return None;\n }\n\n let point0_index = self.index - 1;\n let point0 = contour.position_of(point0_index);\n if self.index == contour.len() {\n let point1 = contour.position_of(0);\n self.index += 1;\n return Some(Segment::line(LineSegment2F::new(point0, point1)));\n }\n\n let point1_index = self.index;\n self.index += 1;\n let point1 = contour.position_of(point1_index);\n if contour.point_is_endpoint(point1_index) {\n return Some(Segment::line(LineSegment2F::new(point0, point1)));\n }\n\n let point2_index = self.index;\n let point2 = contour.position_of(point2_index);\n self.index += 1;\n if contour.point_is_endpoint(point2_index) {\n return Some(Segment::quadratic(LineSegment2F::new(point0, point2), point1));\n }\n\n let point3_index = self.index;\n let point3 = contour.position_of(point3_index);\n self.index += 1;\n debug_assert!(contour.point_is_endpoint(point3_index));\n return Some(Segment::cubic(\n LineSegment2F::new(point0, point3),\n LineSegment2F::new(point1, point2),\n ));\n }\n}\n\n#[derive(Clone, Copy, Debug, PartialEq)]\npub enum ArcDirection {\n CW,\n CCW,\n}\n\nbitflags! {\n pub struct ContourIterFlags: u8 {\n const IGNORE_CLOSE_SEGMENT = 1;\n }\n}\n\n#[inline]\npub(crate) fn union_rect(bounds: &mut RectF, new_point: Vector2F, first: bool) {\n if first {\n *bounds = RectF::from_points(new_point, new_point);\n } else {\n *bounds = bounds.union_point(new_point)\n }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/pattern.rs", "sha256": "2924a04df9d0fcd124feda7b785cf87961e2f8881b74ed47a7d91d3f6d596b0c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/pattern.rs\n//\n// Copyright © 2020 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Raster image patterns.\n\nuse crate::effects::PatternFilter;\nuse crate::render_target::RenderTargetId;\nuse crate::util;\nuse pathfinder_color::{self as color, ColorU};\nuse pathfinder_geometry::transform2d::Transform2F;\nuse pathfinder_geometry::vector::{Vector2I, vec2i};\nuse std::collections::hash_map::DefaultHasher;\nuse std::fmt::{self, Debug, Formatter};\nuse std::hash::{Hash, Hasher};\nuse std::sync::Arc;\n\n#[cfg(feature = \"pf-image\")]\nuse image::RgbaImage;\n\n/// A raster image pattern.\n#[derive(Clone, PartialEq, Debug)]\npub struct Pattern {\n source: PatternSource,\n transform: Transform2F,\n filter: Option,\n flags: PatternFlags,\n}\n\n#[derive(Clone, PartialEq, Eq, Hash, Debug)]\npub enum PatternSource {\n Image(Image),\n RenderTarget {\n id: RenderTargetId,\n size: Vector2I,\n }\n}\n\n/// RGBA, non-premultiplied.\n// FIXME(pcwalton): Hash the pixel contents so that we don't have to compare every pixel!\n// TODO(pcwalton): Should the pixels be premultiplied?\n#[derive(Clone, PartialEq, Eq)]\npub struct Image {\n size: Vector2I,\n pixels: Arc>,\n pixels_hash: u64,\n is_opaque: bool,\n}\n\nbitflags! {\n pub struct PatternFlags: u8 {\n const REPEAT_X = 0x01;\n const REPEAT_Y = 0x02;\n const NO_SMOOTHING = 0x04;\n }\n}\n\nimpl Pattern {\n #[inline]\n fn from_source(source: PatternSource) -> Pattern {\n Pattern {\n source,\n transform: Transform2F::default(),\n filter: None,\n flags: PatternFlags::empty(),\n }\n }\n\n #[inline]\n pub fn from_image(image: Image) -> Pattern {\n Pattern::from_source(PatternSource::Image(image))\n }\n\n #[inline]\n pub fn from_render_target(id: RenderTargetId, size: Vector2I) -> Pattern {\n Pattern::from_source(PatternSource::RenderTarget { id, size })\n }\n\n #[inline]\n pub fn transform(&self) -> Transform2F {\n self.transform\n }\n\n #[inline]\n pub fn apply_transform(&mut self, transform: Transform2F) {\n self.transform = transform * self.transform;\n }\n\n #[inline]\n pub fn size(&self) -> Vector2I {\n match self.source {\n PatternSource::Image(ref image) => image.size(),\n PatternSource::RenderTarget { size, .. } => size,\n }\n }\n\n #[inline]\n pub fn filter(&self) -> Option {\n self.filter\n }\n\n #[inline]\n pub fn set_filter(&mut self, filter: Option) {\n self.filter = filter;\n }\n\n #[inline]\n pub fn repeat_x(&self) -> bool {\n self.flags.contains(PatternFlags::REPEAT_X)\n }\n\n #[inline]\n pub fn set_repeat_x(&mut self, repeat_x: bool) {\n self.flags.set(PatternFlags::REPEAT_X, repeat_x);\n }\n\n #[inline]\n pub fn repeat_y(&self) -> bool {\n self.flags.contains(PatternFlags::REPEAT_Y)\n }\n\n #[inline]\n pub fn set_repeat_y(&mut self, repeat_y: bool) {\n self.flags.set(PatternFlags::REPEAT_Y, repeat_y);\n }\n\n #[inline]\n pub fn smoothing_enabled(&self) -> bool {\n !self.flags.contains(PatternFlags::NO_SMOOTHING)\n }\n\n #[inline]\n pub fn set_smoothing_enabled(&mut self, enable: bool) {\n self.flags.set(PatternFlags::NO_SMOOTHING, !enable);\n }\n\n #[inline]\n pub fn is_opaque(&self) -> bool {\n self.source.is_opaque()\n }\n\n #[inline]\n pub fn source(&self) -> &PatternSource {\n &self.source\n }\n}\n\nimpl Image {\n #[inline]\n pub fn new(size: Vector2I, pixels: Arc>) -> Image {\n assert_eq!(size.x() as usize * size.y() as usize, pixels.len());\n let is_opaque = pixels.iter().all(|pixel| pixel.is_opaque());\n\n let mut pixels_hasher = DefaultHasher::new();\n pixels.hash(&mut pixels_hasher);\n let pixels_hash = pixels_hasher.finish();\n\n Image { size, pixels, pixels_hash, is_opaque }\n }\n\n #[cfg(feature = \"pf-image\")]\n pub fn from_image_buffer(image_buffer: RgbaImage) -> Image {\n let (width, height) = image_buffer.dimensions();\n let pixels = color::u8_vec_to_color_vec(image_buffer.into_raw());\n Image::new(vec2i(width as i32, height as i32), Arc::new(pixels))\n }\n\n #[inline]\n pub fn size(&self) -> Vector2I {\n self.size\n }\n\n #[inline]\n pub fn pixels(&self) -> &Arc> {\n &self.pixels\n }\n\n #[inline]\n pub fn is_opaque(&self) -> bool {\n self.is_opaque\n }\n}\n\nimpl PatternSource {\n #[inline]\n pub fn is_opaque(&self) -> bool {\n match *self {\n PatternSource::Image(ref image) => image.is_opaque(),\n PatternSource::RenderTarget { .. } => {\n // TODO(pcwalton): Maybe do something smarter here?\n false\n }\n }\n }\n}\n\nimpl Debug for Image {\n #[inline]\n fn fmt(&self, formatter: &mut Formatter) -> fmt::Result {\n write!(formatter, \"(image {}×{} px)\", self.size.x(), self.size.y())\n }\n}\n\nimpl Hash for Image {\n fn hash(&self, hasher: &mut H) where H: Hasher {\n self.size.hash(hasher);\n self.pixels_hash.hash(hasher);\n self.is_opaque.hash(hasher);\n }\n}\n\nimpl Eq for Pattern {}\n\nimpl Hash for Pattern {\n fn hash(&self, state: &mut H) where H: Hasher {\n self.source.hash(state);\n util::hash_transform2f(self.transform, state);\n self.flags.hash(state);\n }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/render_target.rs", "sha256": "4c4351303466b11b11c9e6bf73faec138db7a054e27e3ffcfc6eb108cd95a370", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/render_target.rs\n//\n// Copyright © 2020 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Render targets.\n\n#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]\npub struct RenderTargetId {\n pub scene: u32,\n pub render_target: u32,\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/segment.rs", "sha256": "8e119e2dc90d839b0577f7de34c20dfd5789d287e55b7ac824e24406e99f0da1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/segment.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Line or curve segments, optimized with SIMD.\n\nuse pathfinder_geometry::line_segment::LineSegment2F;\nuse pathfinder_geometry::transform2d::Transform2F;\nuse pathfinder_geometry::util::{self, EPSILON};\nuse pathfinder_geometry::vector::{Vector2F, vec2f};\nuse pathfinder_simd::default::F32x4;\nuse std::f32::consts::SQRT_2;\n\nconst MAX_NEWTON_ITERATIONS: u32 = 32;\n\n#[derive(Clone, Copy, Debug, PartialEq)]\npub struct Segment {\n pub baseline: LineSegment2F,\n pub ctrl: LineSegment2F,\n pub kind: SegmentKind,\n pub flags: SegmentFlags,\n}\n\nimpl Segment {\n #[inline]\n pub fn none() -> Segment {\n Segment {\n baseline: LineSegment2F::default(),\n ctrl: LineSegment2F::default(),\n kind: SegmentKind::None,\n flags: SegmentFlags::empty(),\n }\n }\n\n #[inline]\n pub fn line(line: LineSegment2F) -> Segment {\n Segment {\n baseline: line,\n ctrl: LineSegment2F::default(),\n kind: SegmentKind::Line,\n flags: SegmentFlags::empty(),\n }\n }\n\n #[inline]\n pub fn quadratic(baseline: LineSegment2F, ctrl: Vector2F) -> Segment {\n Segment {\n baseline,\n ctrl: LineSegment2F::new(ctrl, Vector2F::zero()),\n kind: SegmentKind::Quadratic,\n flags: SegmentFlags::empty(),\n }\n }\n\n #[inline]\n pub fn cubic(baseline: LineSegment2F, ctrl: LineSegment2F) -> Segment {\n Segment {\n baseline,\n ctrl,\n kind: SegmentKind::Cubic,\n flags: SegmentFlags::empty(),\n }\n }\n\n /// Approximates an unit-length arc with a cubic Bézier curve.\n ///\n /// The maximum supported sweep angle is π/2 (i.e. 90°).\n pub fn arc(sweep_angle: f32) -> Segment {\n Segment::arc_from_cos(f32::cos(sweep_angle))\n }\n\n /// Approximates an unit-length arc with a cubic Bézier curve, given the cosine of the sweep\n /// angle.\n ///\n /// The maximum supported sweep angle is π/2 (i.e. 90°).\n pub fn arc_from_cos(cos_sweep_angle: f32) -> Segment {\n // Richard A. DeVeneza, \"How to determine the control points of a Bézier curve that\n // approximates a small arc\", 2004.\n //\n // https://www.tinaja.com/glib/bezcirc2.pdf\n if cos_sweep_angle >= 1.0 - EPSILON {\n return Segment::line(LineSegment2F::new(vec2f(1.0, 0.0), vec2f(1.0, 0.0)));\n }\n\n let term = F32x4::new(cos_sweep_angle, -cos_sweep_angle,\n cos_sweep_angle, -cos_sweep_angle);\n let signs = F32x4::new(1.0, -1.0, 1.0, 1.0);\n let p3p0 = ((F32x4::splat(1.0) + term) * F32x4::splat(0.5)).sqrt() * signs;\n let (p0x, p0y) = (p3p0.z(), p3p0.w());\n let (p1x, p1y) = (4.0 - p0x, (1.0 - p0x) * (3.0 - p0x) / p0y);\n let p2p1 = F32x4::new(p1x, -p1y, p1x, p1y) * F32x4::splat(1.0 / 3.0);\n return Segment::cubic(LineSegment2F(p3p0), LineSegment2F(p2p1));\n }\n\n #[inline]\n pub fn quarter_circle_arc() -> Segment {\n let p0 = Vector2F::splat(SQRT_2 * 0.5);\n let p1 = vec2f(-SQRT_2 / 6.0 + 4.0 / 3.0, 7.0 * SQRT_2 / 6.0 - 4.0 / 3.0);\n let flip = vec2f(1.0, -1.0);\n let (p2, p3) = (p1 * flip, p0 * flip);\n Segment::cubic(LineSegment2F::new(p3, p0), LineSegment2F::new(p2, p1))\n }\n\n #[inline]\n pub fn as_line_segment(&self) -> LineSegment2F {\n debug_assert!(self.is_line());\n self.baseline\n }\n\n #[inline]\n pub fn is_none(&self) -> bool {\n self.kind == SegmentKind::None\n }\n\n #[inline]\n pub fn is_line(&self) -> bool {\n self.kind == SegmentKind::Line\n }\n\n #[inline]\n pub fn is_quadratic(&self) -> bool {\n self.kind == SegmentKind::Quadratic\n }\n\n #[inline]\n pub fn is_cubic(&self) -> bool {\n self.kind == SegmentKind::Cubic\n }\n\n #[inline]\n pub fn as_cubic_segment(&self) -> CubicSegment {\n debug_assert!(self.is_cubic());\n CubicSegment(self)\n }\n\n // FIXME(pcwalton): We should basically never use this function.\n // FIXME(pcwalton): Handle lines!\n #[inline]\n pub fn to_cubic(&self) -> Segment {\n if self.is_cubic() {\n return *self;\n }\n\n let mut new_segment = *self;\n let p1_2 = self.ctrl.from() + self.ctrl.from();\n new_segment.ctrl = LineSegment2F::new(self.baseline.from() + p1_2,\n p1_2 + self.baseline.to()) * (1.0 / 3.0);\n new_segment.kind = SegmentKind::Cubic;\n new_segment\n }\n\n #[inline]\n pub fn is_monotonic(&self) -> bool {\n // FIXME(pcwalton): Don't degree elevate!\n match self.kind {\n SegmentKind::None | SegmentKind::Line => true,\n SegmentKind::Quadratic => self.to_cubic().as_cubic_segment().is_monotonic(),\n SegmentKind::Cubic => self.as_cubic_segment().is_monotonic(),\n }\n }\n\n #[inline]\n pub fn reversed(&self) -> Segment {\n Segment {\n baseline: self.baseline.reversed(),\n ctrl: if self.is_quadratic() {\n self.ctrl\n } else {\n self.ctrl.reversed()\n },\n kind: self.kind,\n flags: self.flags,\n }\n }\n\n // Reverses if necessary so that the from point is above the to point. Calling this method\n // again will undo the transformation.\n #[inline]\n pub fn orient(&self, y_winding: i32) -> Segment {\n if y_winding >= 0 {\n *self\n } else {\n self.reversed()\n }\n }\n\n #[inline]\n pub fn is_tiny(&self) -> bool {\n const EPSILON: f32 = 0.0001;\n self.baseline.square_length() < EPSILON\n }\n\n #[inline]\n pub fn split(&self, t: f32) -> (Segment, Segment) {\n // FIXME(pcwalton): Don't degree elevate!\n if self.is_line() {\n let (before, after) = self.as_line_segment().split(t);\n (Segment::line(before), Segment::line(after))\n } else {\n self.to_cubic().as_cubic_segment().split(t)\n }\n }\n\n #[inline]\n pub fn sample(self, t: f32) -> Vector2F {\n // FIXME(pcwalton): Don't degree elevate!\n if self.is_line() {\n self.as_line_segment().sample(t)\n } else {\n self.to_cubic().as_cubic_segment().sample(t)\n }\n }\n\n #[inline]\n pub fn transform(self, transform: &Transform2F) -> Segment {\n Segment {\n baseline: *transform * self.baseline,\n ctrl: *transform * self.ctrl,\n kind: self.kind,\n flags: self.flags,\n }\n }\n\n pub fn arc_length(&self) -> f32 {\n // FIXME(pcwalton)\n self.baseline.vector().length()\n }\n\n pub fn time_for_distance(&self, distance: f32) -> f32 {\n // FIXME(pcwalton)\n distance / self.arc_length()\n }\n}\n\n#[derive(Clone, Copy, Debug, PartialEq)]\n#[repr(u8)]\npub enum SegmentKind {\n None,\n Line,\n Quadratic,\n Cubic,\n}\n\nbitflags! {\n pub struct SegmentFlags: u8 {\n const FIRST_IN_SUBPATH = 0x01;\n const CLOSES_SUBPATH = 0x02;\n }\n}\n\n#[derive(Clone, Copy, Debug)]\npub struct CubicSegment<'s>(pub &'s Segment);\n\nimpl<'s> CubicSegment<'s> {\n // See Kaspar Fischer, \"Piecewise Linear Approximation of Bézier Curves\", 2000.\n #[inline]\n pub fn is_flat(self, tolerance: f32) -> bool {\n let mut uv = F32x4::splat(3.0) * self.0.ctrl.0\n - self.0.baseline.0\n - self.0.baseline.0\n - self.0.baseline.reversed().0;\n uv = uv * uv;\n uv = uv.max(uv.zwxy());\n uv[0] + uv[1] <= 16.0 * tolerance * tolerance\n }\n\n #[inline]\n pub fn split(self, t: f32) -> (Segment, Segment) {\n let (baseline0, ctrl0, baseline1, ctrl1);\n if t <= 0.0 {\n let from = &self.0.baseline.from();\n baseline0 = LineSegment2F::new(*from, *from);\n ctrl0 = LineSegment2F::new(*from, *from);\n baseline1 = self.0.baseline;\n ctrl1 = self.0.ctrl;\n } else if t >= 1.0 {\n let to = &self.0.baseline.to();\n baseline0 = self.0.baseline;\n ctrl0 = self.0.ctrl;\n baseline1 = LineSegment2F::new(*to, *to);\n ctrl1 = LineSegment2F::new(*to, *to);\n } else {\n let tttt = F32x4::splat(t);\n\n let (p0p3, p1p2) = (self.0.baseline.0, self.0.ctrl.0);\n let p0p1 = p0p3.concat_xy_xy(p1p2);\n\n // p01 = lerp(p0, p1, t), p12 = lerp(p1, p2, t), p23 = lerp(p2, p3, t)\n let p01p12 = p0p1 + tttt * (p1p2 - p0p1);\n let pxxp23 = p1p2 + tttt * (p0p3 - p1p2);\n let p12p23 = p01p12.concat_zw_zw(pxxp23);\n\n // p012 = lerp(p01, p12, t), p123 = lerp(p12, p23, t)\n let p012p123 = p01p12 + tttt * (p12p23 - p01p12);\n let p123 = p012p123.zwzw();\n\n // p0123 = lerp(p012, p123, t)\n let p0123 = p012p123 + tttt * (p123 - p012p123);\n\n baseline0 = LineSegment2F(p0p3.concat_xy_xy(p0123));\n ctrl0 = LineSegment2F(p01p12.concat_xy_xy(p012p123));\n baseline1 = LineSegment2F(p0123.concat_xy_zw(p0p3));\n ctrl1 = LineSegment2F(p012p123.concat_zw_zw(p12p23));\n }\n\n (\n Segment {\n baseline: baseline0,\n ctrl: ctrl0,\n kind: SegmentKind::Cubic,\n flags: self.0.flags & SegmentFlags::FIRST_IN_SUBPATH,\n },\n Segment {\n baseline: baseline1,\n ctrl: ctrl1,\n kind: SegmentKind::Cubic,\n flags: self.0.flags & SegmentFlags::CLOSES_SUBPATH,\n },\n )\n }\n\n #[inline]\n pub fn split_before(self, t: f32) -> Segment {\n self.split(t).0\n }\n\n #[inline]\n pub fn split_after(self, t: f32) -> Segment {\n self.split(t).1\n }\n\n // FIXME(pcwalton): Use Horner's method!\n #[inline]\n pub fn sample(self, t: f32) -> Vector2F {\n self.split(t).0.baseline.to()\n }\n\n #[inline]\n pub fn is_monotonic(self) -> bool {\n // TODO(pcwalton): Optimize this.\n let (p0, p3) = (self.0.baseline.from_y(), self.0.baseline.to_y());\n let (p1, p2) = (self.0.ctrl.from_y(), self.0.ctrl.to_y());\n (p0 <= p1 && p1 <= p2 && p2 <= p3) || (p0 >= p1 && p1 >= p2 && p2 >= p3)\n }\n\n #[inline]\n pub fn y_extrema(self) -> (Option, Option) {\n if self.is_monotonic() {\n return (None, None);\n }\n\n let p0p1p2p3 = F32x4::new(\n self.0.baseline.from_y(),\n self.0.ctrl.from_y(),\n self.0.ctrl.to_y(),\n self.0.baseline.to_y(),\n );\n\n let pxp0p1p2 = p0p1p2p3.wxyz();\n let pxv0v1v2 = p0p1p2p3 - pxp0p1p2;\n let (v0, v1, v2) = (pxv0v1v2[1], pxv0v1v2[2], pxv0v1v2[3]);\n\n let (t0, t1);\n let (v0_to_v1, v2_to_v1) = (v0 - v1, v2 - v1);\n let denom = v0_to_v1 + v2_to_v1;\n\n if util::approx_eq(denom, 0.0) {\n // Let's not divide by zero (issue #146). Fall back to Newton's method.\n // FIXME(pcwalton): Can we have two roots here?\n let mut t = 0.5;\n for _ in 0..MAX_NEWTON_ITERATIONS {\n let dydt = 3.0 * ((denom * t - v0_to_v1 - v0_to_v1) * t + v0);\n if f32::abs(dydt) <= EPSILON {\n break\n }\n let d2ydt2 = 6.0 * (denom * t - v0_to_v1);\n t -= dydt / d2ydt2;\n }\n t0 = t;\n t1 = 0.0;\n debug!(\"... t=(newton) {}\", t);\n } else {\n // Algebraically compute the values for t.\n let discrim = f32::sqrt(v1 * v1 - v0 * v2);\n let denom_recip = 1.0 / denom;\n\n t0 = (v0_to_v1 + discrim) * denom_recip;\n t1 = (v0_to_v1 - discrim) * denom_recip;\n\n debug!(\"... t=({} +/- {})/{} t0={} t1={}\", v0_to_v1, discrim, denom, t0, t1);\n }\n\n return match (\n t0 > EPSILON && t0 < 1.0 - EPSILON,\n t1 > EPSILON && t1 < 1.0 - EPSILON,\n ) {\n (false, false) => (None, None),\n (true, false) => (Some(t0), None),\n (false, true) => (Some(t1), None),\n (true, true) => (Some(f32::min(t0, t1)), Some(f32::max(t0, t1))),\n };\n }\n\n #[inline]\n pub fn min_x(&self) -> f32 {\n f32::min(self.0.baseline.min_x(), self.0.ctrl.min_x())\n }\n #[inline]\n pub fn min_y(&self) -> f32 {\n f32::min(self.0.baseline.min_y(), self.0.ctrl.min_y())\n }\n #[inline]\n pub fn max_x(&self) -> f32 {\n f32::max(self.0.baseline.max_x(), self.0.ctrl.max_x())\n }\n #[inline]\n pub fn max_y(&self) -> f32 {\n f32::max(self.0.baseline.max_y(), self.0.ctrl.max_y())\n }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/sorted_vector.rs", "sha256": "a4ce0e1da35c7c1cfc0ccbae16d32fd5b447152bd1a95450ffecb2207d861ba0", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/sorted_vector.rs\n//\n// Copyright © 2020 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! A vector that maintains sorted order with insertion sort.\n\nuse std::cmp::Ordering;\nuse std::convert;\n\n#[derive(Clone, PartialEq, Eq, Hash, Debug)]\npub struct SortedVector\nwhere\n T: PartialOrd,\n{\n pub array: Vec,\n}\n\nimpl SortedVector\nwhere\n T: PartialOrd,\n{\n #[inline]\n pub fn new() -> SortedVector {\n SortedVector { array: vec![] }\n }\n\n #[inline]\n pub fn push(&mut self, value: T) {\n let index = self.binary_search_by(|other| {\n other.partial_cmp(&value).unwrap_or(Ordering::Less)\n }).unwrap_or_else(convert::identity);\n self.array.insert(index, value);\n }\n\n #[inline]\n pub fn peek(&self) -> Option<&T> {\n self.array.last()\n }\n\n #[inline]\n pub fn pop(&mut self) -> Option {\n self.array.pop()\n }\n\n #[inline]\n pub fn clear(&mut self) {\n self.array.clear()\n }\n\n #[allow(dead_code)]\n #[inline]\n pub fn is_empty(&self) -> bool {\n self.array.is_empty()\n }\n\n #[inline]\n pub fn len(&self) -> usize {\n self.array.len()\n }\n\n #[inline]\n pub fn binary_search_by<'a, F>(&'a self, f: F) -> Result\n where F: FnMut(&'a T) -> Ordering {\n self.array.binary_search_by(f)\n }\n}\n\n#[cfg(test)]\nmod test {\n use crate::sorted_vector::SortedVector;\n use quickcheck;\n\n #[test]\n fn test_sorted_vec() {\n quickcheck::quickcheck(prop_sorted_vec as fn(Vec) -> bool);\n\n fn prop_sorted_vec(mut values: Vec) -> bool {\n let mut sorted_vec = SortedVector::new();\n for &value in &values {\n sorted_vec.push(value)\n }\n\n values.sort();\n let mut results = Vec::with_capacity(values.len());\n while !sorted_vec.is_empty() {\n results.push(sorted_vec.pop().unwrap());\n }\n results.reverse();\n assert_eq!(&values, &results);\n\n true\n }\n }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/stroke.rs", "sha256": "25450d1680bee0b0b8848ffb42665cef1f0523ec6c879bcd3df01e4a26d206c9", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/stroke.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Utilities for converting path strokes to fills.\n\nuse crate::outline::{ArcDirection, Contour, ContourIterFlags, Outline, PushSegmentFlags};\nuse crate::segment::Segment;\nuse pathfinder_geometry::line_segment::LineSegment2F;\nuse pathfinder_geometry::rect::RectF;\nuse pathfinder_geometry::transform2d::Transform2F;\nuse pathfinder_geometry::util::EPSILON;\nuse pathfinder_geometry::vector::{Vector2F, vec2f};\nuse std::f32;\n\nconst TOLERANCE: f32 = 0.01;\n\npub struct OutlineStrokeToFill<'a> {\n input: &'a Outline,\n output: Outline,\n style: StrokeStyle,\n}\n\n#[derive(Clone, Copy, Debug, PartialEq)]\npub struct StrokeStyle {\n pub line_width: f32,\n pub line_cap: LineCap,\n pub line_join: LineJoin,\n}\n\n#[derive(Clone, Copy, Debug, PartialEq)]\npub enum LineCap {\n Butt,\n Square,\n Round,\n}\n\n#[derive(Clone, Copy, Debug, PartialEq)]\npub enum LineJoin {\n Miter(f32),\n Bevel,\n Round,\n}\n\nimpl<'a> OutlineStrokeToFill<'a> {\n #[inline]\n pub fn new(input: &Outline, style: StrokeStyle) -> OutlineStrokeToFill {\n OutlineStrokeToFill { input, output: Outline::new(), style }\n }\n\n pub fn offset(&mut self) {\n let mut new_contours = vec![];\n for input in &self.input.contours {\n let closed = input.closed;\n let mut stroker = ContourStrokeToFill::new(input,\n Contour::new(),\n self.style.line_width * 0.5,\n self.style.line_join);\n\n stroker.offset_forward();\n if closed {\n self.push_stroked_contour(&mut new_contours, stroker, true);\n stroker = ContourStrokeToFill::new(input,\n Contour::new(),\n self.style.line_width * 0.5,\n self.style.line_join);\n } else {\n self.add_cap(&mut stroker.output);\n }\n\n stroker.offset_backward();\n if !closed {\n self.add_cap(&mut stroker.output);\n }\n\n self.push_stroked_contour(&mut new_contours, stroker, closed);\n }\n\n let mut new_bounds = None;\n new_contours.iter().for_each(|contour| contour.update_bounds(&mut new_bounds));\n\n self.output.contours = new_contours;\n self.output.bounds = new_bounds.unwrap_or_else(|| RectF::default());\n }\n\n #[inline]\n pub fn into_outline(self) -> Outline {\n self.output\n }\n\n fn push_stroked_contour(&mut self,\n new_contours: &mut Vec,\n mut stroker: ContourStrokeToFill,\n closed: bool) {\n // Add join if necessary.\n if closed && stroker.output.might_need_join(self.style.line_join) {\n let (p1, p0) = (stroker.output.position_of(1), stroker.output.position_of(0));\n let final_segment = LineSegment2F::new(p1, p0);\n stroker.output.add_join(self.style.line_width * 0.5,\n self.style.line_join,\n stroker.input.position_of(0),\n final_segment);\n }\n\n stroker.output.closed = true;\n new_contours.push(stroker.output);\n }\n\n fn add_cap(&mut self, contour: &mut Contour) {\n if self.style.line_cap == LineCap::Butt || contour.len() < 2 {\n return\n }\n\n let width = self.style.line_width;\n let p1 = contour.position_of_last(1);\n\n // Determine the ending gradient.\n let mut p0;\n let mut p0_index = contour.len() - 2;\n loop {\n p0 = contour.position_of(p0_index);\n if (p1 - p0).square_length() > EPSILON {\n break;\n }\n if p0_index == 0 {\n return;\n }\n p0_index -= 1;\n }\n let gradient = (p1 - p0).normalize();\n\n match self.style.line_cap {\n LineCap::Butt => unreachable!(),\n\n LineCap::Square => {\n let offset = gradient * (width * 0.5);\n\n let p2 = p1 + offset;\n let p3 = p2 + gradient.yx() * vec2f(-width, width);\n let p4 = p3 - offset;\n\n contour.push_endpoint(p2);\n contour.push_endpoint(p3);\n contour.push_endpoint(p4);\n }\n\n LineCap::Round => {\n let scale = width * 0.5;\n let offset = gradient.yx() * vec2f(-1.0, 1.0);\n let translation = p1 + offset * (width * 0.5);\n let transform = Transform2F::from_scale(scale).translate(translation);\n let chord = LineSegment2F::new(-offset, offset);\n contour.push_arc_from_unit_chord(&transform, chord, ArcDirection::CW);\n }\n }\n }\n}\n\nstruct ContourStrokeToFill<'a> {\n input: &'a Contour,\n output: Contour,\n radius: f32,\n join: LineJoin,\n}\n\nimpl<'a> ContourStrokeToFill<'a> {\n #[inline]\n fn new(input: &Contour, output: Contour, radius: f32, join: LineJoin) -> ContourStrokeToFill {\n ContourStrokeToFill { input, output, radius, join }\n }\n\n fn offset_forward(&mut self) {\n for (segment_index, segment) in self.input.iter(ContourIterFlags::empty()).enumerate() {\n // FIXME(pcwalton): We negate the radius here so that round end caps can be drawn\n // clockwise. Of course, we should just implement anticlockwise arcs to begin with...\n let join = if segment_index == 0 { LineJoin::Bevel } else { self.join };\n segment.offset(-self.radius, join, &mut self.output);\n }\n }\n\n fn offset_backward(&mut self) {\n let mut segments: Vec<_> = self\n .input\n .iter(ContourIterFlags::empty())\n .map(|segment| segment.reversed())\n .collect();\n segments.reverse();\n for (segment_index, segment) in segments.iter().enumerate() {\n // FIXME(pcwalton): We negate the radius here so that round end caps can be drawn\n // clockwise. Of course, we should just implement anticlockwise arcs to begin with...\n let join = if segment_index == 0 { LineJoin::Bevel } else { self.join };\n segment.offset(-self.radius, join, &mut self.output);\n }\n }\n}\n\ntrait Offset {\n fn offset(&self, distance: f32, join: LineJoin, contour: &mut Contour);\n fn add_to_contour(&self,\n distance: f32,\n join: LineJoin,\n join_point: Vector2F,\n contour: &mut Contour);\n fn offset_once(&self, distance: f32) -> Self;\n fn error_is_within_tolerance(&self, other: &Segment, distance: f32) -> bool;\n}\n\nimpl Offset for Segment {\n fn offset(&self, distance: f32, join: LineJoin, contour: &mut Contour) {\n let join_point = self.baseline.from();\n if self.baseline.square_length() < TOLERANCE * TOLERANCE {\n self.add_to_contour(distance, join, join_point, contour);\n return;\n }\n\n let candidate = self.offset_once(distance);\n if self.error_is_within_tolerance(&candidate, distance) {\n candidate.add_to_contour(distance, join, join_point, contour);\n return;\n }\n\n debug!(\"--- SPLITTING ---\");\n debug!(\"... PRE-SPLIT: {:?}\", self);\n let (before, after) = self.split(0.5);\n debug!(\"... AFTER-SPLIT: {:?} {:?}\", before, after);\n before.offset(distance, join, contour);\n after.offset(distance, join, contour);\n }\n\n fn add_to_contour(&self,\n distance: f32,\n join: LineJoin,\n join_point: Vector2F,\n contour: &mut Contour) {\n // Add join if necessary.\n if contour.might_need_join(join) {\n let p3 = self.baseline.from();\n let p4 = if self.is_line() {\n self.baseline.to()\n } else {\n // NB: If you change the representation of quadratic curves, you will need to\n // change this.\n self.ctrl.from()\n };\n\n contour.add_join(distance, join, join_point, LineSegment2F::new(p4, p3));\n }\n\n // Push segment.\n let flags = PushSegmentFlags::UPDATE_BOUNDS | PushSegmentFlags::INCLUDE_FROM_POINT;\n contour.push_segment(self, flags);\n }\n\n fn offset_once(&self, distance: f32) -> Segment {\n if self.is_line() {\n return Segment::line(self.baseline.offset(distance));\n }\n\n if self.is_quadratic() {\n let mut segment_0 = LineSegment2F::new(self.baseline.from(), self.ctrl.from());\n let mut segment_1 = LineSegment2F::new(self.ctrl.from(), self.baseline.to());\n segment_0 = segment_0.offset(distance);\n segment_1 = segment_1.offset(distance);\n let ctrl = match segment_0.intersection_t(segment_1) {\n Some(t) => segment_0.sample(t),\n None => segment_0.to().lerp(segment_1.from(), 0.5),\n };\n let baseline = LineSegment2F::new(segment_0.from(), segment_1.to());\n return Segment::quadratic(baseline, ctrl);\n }\n\n debug_assert!(self.is_cubic());\n\n if self.baseline.from() == self.ctrl.from() {\n let mut segment_0 = LineSegment2F::new(self.baseline.from(), self.ctrl.to());\n let mut segment_1 = LineSegment2F::new(self.ctrl.to(), self.baseline.to());\n segment_0 = segment_0.offset(distance);\n segment_1 = segment_1.offset(distance);\n let ctrl = match segment_0.intersection_t(segment_1) {\n Some(t) => segment_0.sample(t),\n None => segment_0.to().lerp(segment_1.from(), 0.5),\n };\n let baseline = LineSegment2F::new(segment_0.from(), segment_1.to());\n let ctrl = LineSegment2F::new(segment_0.from(), ctrl);\n return Segment::cubic(baseline, ctrl);\n }\n\n if self.ctrl.to() == self.baseline.to() {\n let mut segment_0 = LineSegment2F::new(self.baseline.from(), self.ctrl.from());\n let mut segment_1 = LineSegment2F::new(self.ctrl.from(), self.baseline.to());\n segment_0 = segment_0.offset(distance);\n segment_1 = segment_1.offset(distance);\n let ctrl = match segment_0.intersection_t(segment_1) {\n Some(t) => segment_0.sample(t),\n None => segment_0.to().lerp(segment_1.from(), 0.5),\n };\n let baseline = LineSegment2F::new(segment_0.from(), segment_1.to());\n let ctrl = LineSegment2F::new(ctrl, segment_1.to());\n return Segment::cubic(baseline, ctrl);\n }\n\n let mut segment_0 = LineSegment2F::new(self.baseline.from(), self.ctrl.from());\n let mut segment_1 = LineSegment2F::new(self.ctrl.from(), self.ctrl.to());\n let mut segment_2 = LineSegment2F::new(self.ctrl.to(), self.baseline.to());\n segment_0 = segment_0.offset(distance);\n segment_1 = segment_1.offset(distance);\n segment_2 = segment_2.offset(distance);\n let (ctrl_0, ctrl_1) = match (\n segment_0.intersection_t(segment_1),\n segment_1.intersection_t(segment_2),\n ) {\n (Some(t0), Some(t1)) => (segment_0.sample(t0), segment_1.sample(t1)),\n _ => (\n segment_0.to().lerp(segment_1.from(), 0.5),\n segment_1.to().lerp(segment_2.from(), 0.5),\n ),\n };\n let baseline = LineSegment2F::new(segment_0.from(), segment_2.to());\n let ctrl = LineSegment2F::new(ctrl_0, ctrl_1);\n Segment::cubic(baseline, ctrl)\n }\n\n fn error_is_within_tolerance(&self, other: &Segment, distance: f32) -> bool {\n let (mut min, mut max) = (\n f32::abs(distance) - TOLERANCE,\n f32::abs(distance) + TOLERANCE,\n );\n min = if min <= 0.0 { 0.0 } else { min * min };\n max = if max <= 0.0 { 0.0 } else { max * max };\n\n for t_num in 0..(SAMPLE_COUNT + 1) {\n let t = t_num as f32 / SAMPLE_COUNT as f32;\n // FIXME(pcwalton): Use signed distance!\n let (this_p, other_p) = (self.sample(t), other.sample(t));\n let vector = this_p - other_p;\n let square_distance = vector.square_length();\n debug!(\n \"this_p={:?} other_p={:?} vector={:?} sqdist={:?} min={:?} max={:?}\",\n this_p, other_p, vector, square_distance, min, max\n );\n if square_distance < min || square_distance > max {\n return false;\n }\n }\n\n return true;\n\n const SAMPLE_COUNT: u32 = 16;\n }\n}\n\nimpl Contour {\n fn might_need_join(&self, join: LineJoin) -> bool {\n if self.len() < 2 {\n false\n } else {\n match join {\n LineJoin::Miter(_) | LineJoin::Round => true,\n LineJoin::Bevel => false,\n }\n }\n }\n\n fn add_join(&mut self,\n distance: f32,\n join: LineJoin,\n join_point: Vector2F,\n next_tangent: LineSegment2F) {\n let (p0, p1) = (self.position_of_last(2), self.position_of_last(1));\n let prev_tangent = LineSegment2F::new(p0, p1);\n\n if prev_tangent.square_length() < EPSILON || next_tangent.square_length() < EPSILON {\n return;\n }\n\n match join {\n LineJoin::Bevel => {}\n LineJoin::Miter(miter_limit) => {\n if let Some(prev_tangent_t) = prev_tangent.intersection_t(next_tangent) {\n if prev_tangent_t < -EPSILON {\n return;\n }\n let miter_endpoint = prev_tangent.sample(prev_tangent_t);\n let threshold = miter_limit * distance;\n if (miter_endpoint - join_point).square_length() > threshold * threshold {\n return;\n }\n self.push_endpoint(miter_endpoint);\n }\n }\n LineJoin::Round => {\n let scale = distance.abs();\n let transform = Transform2F::from_scale(scale).translate(join_point);\n let chord_from = (prev_tangent.to() - join_point).normalize();\n let chord_to = (next_tangent.to() - join_point).normalize();\n let chord = LineSegment2F::new(chord_from, chord_to);\n self.push_arc_from_unit_chord(&transform, chord, ArcDirection::CW);\n }\n }\n }\n}\n\nimpl Default for StrokeStyle {\n #[inline]\n fn default() -> StrokeStyle {\n StrokeStyle {\n line_width: 1.0,\n line_cap: LineCap::default(),\n line_join: LineJoin::default(),\n }\n }\n}\n\nimpl Default for LineCap {\n #[inline]\n fn default() -> LineCap { LineCap::Butt }\n}\n\nimpl Default for LineJoin {\n #[inline]\n fn default() -> LineJoin { LineJoin::Miter(10.0) }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/transform.rs", "sha256": "4eb2c22dbfee166ba857a5173ef4df8428f36cd2ddd2786b9bcd34c8251db414", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/transform.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Utilities for transforming paths.\n\nuse crate::segment::Segment;\nuse pathfinder_geometry::transform2d::Transform2F;\nuse pathfinder_geometry::transform3d::Perspective;\n\n/// Transforms a path with a SIMD 2D transform.\npub struct Transform2FPathIter\nwhere\n I: Iterator,\n{\n iter: I,\n transform: Transform2F,\n}\n\nimpl Iterator for Transform2FPathIter\nwhere\n I: Iterator,\n{\n type Item = Segment;\n\n #[inline]\n fn next(&mut self) -> Option {\n // TODO(pcwalton): Can we go faster by transforming an entire line segment with SIMD?\n let mut segment = self.iter.next()?;\n if !segment.is_none() {\n segment.baseline.set_from(self.transform * segment.baseline.from());\n segment.baseline.set_to(self.transform * segment.baseline.to());\n if !segment.is_line() {\n segment.ctrl.set_from(self.transform * segment.ctrl.from());\n if !segment.is_quadratic() {\n segment.ctrl.set_to(self.transform * segment.ctrl.to());\n }\n }\n }\n Some(segment)\n }\n}\n\nimpl Transform2FPathIter\nwhere\n I: Iterator,\n{\n #[inline]\n pub fn new(iter: I, transform: &Transform2F) -> Transform2FPathIter {\n Transform2FPathIter {\n iter,\n transform: *transform,\n }\n }\n}\n\n/// Transforms a path with a perspective projection.\npub struct PerspectivePathIter\nwhere\n I: Iterator,\n{\n iter: I,\n perspective: Perspective,\n}\n\nimpl Iterator for PerspectivePathIter\nwhere\n I: Iterator,\n{\n type Item = Segment;\n\n #[inline]\n fn next(&mut self) -> Option {\n let mut segment = self.iter.next()?;\n if !segment.is_none() {\n segment.baseline.set_from(self.perspective * segment.baseline.from());\n segment.baseline.set_to(self.perspective * segment.baseline.to());\n if !segment.is_line() {\n segment.ctrl.set_from(self.perspective * segment.ctrl.from());\n if !segment.is_quadratic() {\n segment.ctrl.set_to(self.perspective * segment.ctrl.to());\n }\n }\n }\n Some(segment)\n }\n}\n\nimpl PerspectivePathIter\nwhere\n I: Iterator,\n{\n #[inline]\n pub fn new(iter: I, perspective: &Perspective) -> PerspectivePathIter {\n PerspectivePathIter {\n iter,\n perspective: *perspective,\n }\n }\n}\n" }, { "package": "pathfinder_content@0.5.0", "sourcePath": "src/util.rs", "sha256": "fed2ffe6642fad99f27905840920427d73138fccfd3fad0393326a56aa76ff19", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/content/src/util.rs\n//\n// Copyright © 2020 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Miscellaneous utilities.\n\nuse pathfinder_geometry::line_segment::LineSegment2F;\nuse pathfinder_geometry::transform2d::Transform2F;\nuse pathfinder_simd::default::{F32x2, F32x4};\nuse std::hash::{Hash, Hasher};\nuse std::mem;\n\npub(crate) fn hash_line_segment(line_segment: LineSegment2F, state: &mut H) where H: Hasher {\n hash_f32x4(line_segment.0, state);\n}\n\npub(crate) fn hash_transform2f(transform: Transform2F, state: &mut H) where H: Hasher {\n hash_f32x4(transform.matrix.0, state);\n hash_f32x2(transform.vector.0, state);\n}\n\npub(crate) fn hash_f32(value: f32, state: &mut H) where H: Hasher {\n unsafe {\n let data: u32 = mem::transmute::(value);\n data.hash(state);\n }\n}\n\npub(crate) fn hash_f32x2(vector: F32x2, state: &mut H) where H: Hasher {\n unsafe {\n let data: [u32; 2] = mem::transmute::(vector);\n data.hash(state);\n }\n}\n\npub(crate) fn hash_f32x4(vector: F32x4, state: &mut H) where H: Hasher {\n unsafe {\n let data: [u32; 4] = mem::transmute::(vector);\n data.hash(state);\n }\n}\n" }, { "package": "pathfinder_geometry@0.5.1", "sourcePath": "src/angle.rs", "sha256": "dce9c08facbaceccb920035477ac53ffc2798247b42b2ddef9338f2e35ab9c80", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/geometry/src/angle.rs\n//\n// Copyright © 2020 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Angle utilities.\n\nuse std::f32::consts::PI;\n\n#[inline]\npub fn angle_from_degrees(degrees: f32) -> f32 {\n const SCALE: f32 = 2.0 * PI / 360.0;\n degrees * SCALE\n}\n" }, { "package": "pathfinder_geometry@0.5.1", "sourcePath": "src/lib.rs", "sha256": "cedfb2d85e682e9b9e327a169347bf0cebecd52668e1ed91a3af05ec91b42090", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/geometry/src/lib.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Basic geometry and linear algebra primitives, optimized with SIMD.\n\npub mod angle;\npub mod line_segment;\npub mod rect;\npub mod transform2d;\npub mod transform3d;\npub mod unit_vector;\npub mod util;\npub mod vector;\n" }, { "package": "pathfinder_geometry@0.5.1", "sourcePath": "src/line_segment.rs", "sha256": "4af21598890aef0ffec10abd5460c0416cb1bb56b003fb499f0cd75fc68177da", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/geometry/src/basic/line_segment.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Line segment types, optimized with SIMD.\n\nuse crate::transform2d::Matrix2x2F;\nuse crate::vector::{Vector2F, vec2f};\nuse crate::util;\nuse pathfinder_simd::default::F32x4;\nuse std::ops::{Add, Mul, MulAssign, Sub};\n\n#[derive(Clone, Copy, Debug, PartialEq, Default)]\npub struct LineSegment2F(pub F32x4);\n\nimpl LineSegment2F {\n #[inline]\n pub fn new(from: Vector2F, to: Vector2F) -> LineSegment2F {\n LineSegment2F(from.0.concat_xy_xy(to.0))\n }\n\n #[inline]\n pub fn from(self) -> Vector2F {\n Vector2F(self.0.xy())\n }\n\n #[inline]\n pub fn to(self) -> Vector2F {\n Vector2F(self.0.zw())\n }\n\n #[inline]\n pub fn set_from(&mut self, point: Vector2F) {\n self.0 = point.0.to_f32x4().concat_xy_zw(self.0)\n }\n\n #[inline]\n pub fn set_to(&mut self, point: Vector2F) {\n self.0 = self.0.concat_xy_xy(point.0.to_f32x4())\n }\n\n #[allow(clippy::wrong_self_convention)]\n #[inline]\n pub fn from_x(self) -> f32 {\n self.0[0]\n }\n\n #[allow(clippy::wrong_self_convention)]\n #[inline]\n pub fn from_y(self) -> f32 {\n self.0[1]\n }\n\n #[inline]\n pub fn to_x(self) -> f32 {\n self.0[2]\n }\n\n #[inline]\n pub fn to_y(self) -> f32 {\n self.0[3]\n }\n\n #[inline]\n pub fn set_from_x(&mut self, x: f32) {\n self.0[0] = x\n }\n\n #[inline]\n pub fn set_from_y(&mut self, y: f32) {\n self.0[1] = y\n }\n\n #[inline]\n pub fn set_to_x(&mut self, x: f32) {\n self.0[2] = x\n }\n\n #[inline]\n pub fn set_to_y(&mut self, y: f32) {\n self.0[3] = y\n }\n\n #[inline]\n pub fn split(self, t: f32) -> (LineSegment2F, LineSegment2F) {\n debug_assert!(t >= 0.0 && t <= 1.0);\n let (from_from, to_to) = (self.0.xyxy(), self.0.zwzw());\n let d_d = to_to - from_from;\n let mid_mid = from_from + d_d * F32x4::splat(t);\n (\n LineSegment2F(from_from.concat_xy_xy(mid_mid)),\n LineSegment2F(mid_mid.concat_xy_xy(to_to)),\n )\n }\n\n // Returns the left segment first, followed by the right segment.\n #[inline]\n pub fn split_at_x(self, x: f32) -> (LineSegment2F, LineSegment2F) {\n let (min_part, max_part) = self.split(self.solve_t_for_x(x));\n if min_part.from_x() < max_part.from_x() {\n (min_part, max_part)\n } else {\n (max_part, min_part)\n }\n }\n\n // Returns the upper segment first, followed by the lower segment.\n #[inline]\n pub fn split_at_y(self, y: f32) -> (LineSegment2F, LineSegment2F) {\n let (min_part, max_part) = self.split(self.solve_t_for_y(y));\n\n // Make sure we compare `from_y` and `to_y` to properly handle the case in which one of the\n // two segments is zero-length.\n if min_part.from_y() < max_part.to_y() {\n (min_part, max_part)\n } else {\n (max_part, min_part)\n }\n }\n\n #[inline]\n pub fn solve_t_for_x(self, x: f32) -> f32 {\n (x - self.from_x()) / (self.to_x() - self.from_x())\n }\n\n #[inline]\n pub fn solve_t_for_y(self, y: f32) -> f32 {\n (y - self.from_y()) / (self.to_y() - self.from_y())\n }\n\n #[inline]\n pub fn solve_x_for_y(self, y: f32) -> f32 {\n util::lerp(self.from_x(), self.to_x(), self.solve_t_for_y(y))\n }\n\n #[inline]\n pub fn solve_y_for_x(self, x: f32) -> f32 {\n util::lerp(self.from_y(), self.to_y(), self.solve_t_for_x(x))\n }\n\n #[inline]\n pub fn reversed(self) -> LineSegment2F {\n LineSegment2F(self.0.zwxy())\n }\n\n #[inline]\n pub fn upper_point(self) -> Vector2F {\n if self.from_y() < self.to_y() {\n self.from()\n } else {\n self.to()\n }\n }\n\n #[inline]\n pub fn min_x(self) -> f32 {\n f32::min(self.from_x(), self.to_x())\n }\n\n #[inline]\n pub fn max_x(self) -> f32 {\n f32::max(self.from_x(), self.to_x())\n }\n\n #[inline]\n pub fn min_y(self) -> f32 {\n f32::min(self.from_y(), self.to_y())\n }\n\n #[inline]\n pub fn max_y(self) -> f32 {\n f32::max(self.from_y(), self.to_y())\n }\n\n #[inline]\n pub fn y_winding(self) -> i32 {\n if self.from_y() < self.to_y() {\n 1\n } else {\n -1\n }\n }\n\n // Reverses if necessary so that the from point is above the to point. Calling this method\n // again will undo the transformation.\n #[inline]\n pub fn orient(self, y_winding: i32) -> LineSegment2F {\n if y_winding >= 0 {\n self\n } else {\n self.reversed()\n }\n }\n\n // TODO(pcwalton): Optimize with SIMD.\n #[inline]\n pub fn square_length(self) -> f32 {\n let (dx, dy) = (self.to_x() - self.from_x(), self.to_y() - self.from_y());\n dx * dx + dy * dy\n }\n\n #[inline]\n pub fn vector(self) -> Vector2F {\n self.to() - self.from()\n }\n\n // http://www.cs.swan.ac.uk/~cssimon/line_intersection.html\n pub fn intersection_t(self, other: LineSegment2F) -> Option {\n let p0p1 = self.vector();\n let matrix = Matrix2x2F(other.vector().0.concat_xy_xy((-p0p1).0));\n if f32::abs(matrix.det()) < EPSILON {\n return None;\n }\n return Some((matrix.inverse() * (self.from() - other.from())).y());\n\n const EPSILON: f32 = 0.0001;\n }\n\n #[inline]\n pub fn sample(self, t: f32) -> Vector2F {\n self.from() + self.vector() * t\n }\n\n #[inline]\n pub fn midpoint(self) -> Vector2F {\n self.sample(0.5)\n }\n\n #[inline]\n pub fn offset(self, distance: f32) -> LineSegment2F {\n if self.is_zero_length() {\n self\n } else {\n self + self.vector().yx().normalize() * vec2f(-distance, distance)\n }\n }\n\n #[inline]\n pub fn is_zero_length(self) -> bool {\n self.vector().is_zero()\n }\n}\n\nimpl Add for LineSegment2F {\n type Output = LineSegment2F;\n #[inline]\n fn add(self, point: Vector2F) -> LineSegment2F {\n LineSegment2F(self.0 + point.0.to_f32x4().xyxy())\n }\n}\n\nimpl Sub for LineSegment2F {\n type Output = LineSegment2F;\n #[inline]\n fn sub(self, point: Vector2F) -> LineSegment2F {\n LineSegment2F(self.0 - point.0.to_f32x4().xyxy())\n }\n}\n\nimpl Mul for LineSegment2F {\n type Output = LineSegment2F;\n #[inline]\n fn mul(self, factors: Vector2F) -> LineSegment2F {\n LineSegment2F(self.0 * factors.0.to_f32x4().xyxy())\n }\n}\n\nimpl Mul for LineSegment2F {\n type Output = LineSegment2F;\n #[inline]\n fn mul(self, factor: f32) -> LineSegment2F {\n LineSegment2F(self.0 * F32x4::splat(factor))\n }\n}\n\nimpl MulAssign for LineSegment2F {\n #[inline]\n fn mul_assign(&mut self, factors: Vector2F) {\n *self = *self * factors\n }\n}\n\n#[derive(Clone, Copy, Debug, Default)]\n#[repr(C)]\npub struct LineSegmentU4 {\n pub from: u8,\n pub to: u8,\n}\n\n#[derive(Clone, Copy, Debug, Default)]\n#[repr(C)]\npub struct LineSegmentU8 {\n pub from_x: u8,\n pub from_y: u8,\n pub to_x: u8,\n pub to_y: u8,\n}\n" }, { "package": "pathfinder_geometry@0.5.1", "sourcePath": "src/rect.rs", "sha256": "68d48ddd9934ac11f0ed0e81c2cc6ede9271528dc318c3dd49cc9c9f51cb15ca", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/geometry/src/basic/rect.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! 2D axis-aligned rectangles, optimized with SIMD.\n\nuse crate::vector::{IntoVector2F, Vector2F, Vector2I};\nuse pathfinder_simd::default::{F32x4, I32x4};\nuse std::ops::{Add, Mul, Sub};\n\n#[derive(Clone, Copy, Debug, PartialEq, Default)]\npub struct RectF(pub F32x4);\n\nimpl RectF {\n #[inline]\n pub fn new(origin: Vector2F, size: Vector2F) -> RectF {\n RectF(origin.0.concat_xy_xy(origin.0 + size.0))\n }\n\n #[inline]\n pub fn from_points(origin: Vector2F, lower_right: Vector2F) -> RectF {\n RectF(origin.0.concat_xy_xy(lower_right.0))\n }\n\n // Accessors\n\n #[inline]\n pub fn origin(self) -> Vector2F {\n Vector2F(self.0.xy())\n }\n\n #[inline]\n pub fn size(self) -> Vector2F {\n Vector2F(self.0.zw() - self.0.xy())\n }\n\n #[inline]\n pub fn origin_x(self) -> f32 {\n self.0.x()\n }\n\n #[inline]\n pub fn origin_y(self) -> f32 {\n self.0.y()\n }\n\n #[inline]\n pub fn width(self) -> f32 {\n self.0.z() - self.0.x()\n }\n\n #[inline]\n pub fn height(self) -> f32 {\n self.0.w() - self.0.y()\n }\n\n #[inline]\n pub fn upper_right(self) -> Vector2F {\n Vector2F(self.0.zy())\n }\n\n #[inline]\n pub fn lower_left(self) -> Vector2F {\n Vector2F(self.0.xw())\n }\n\n #[inline]\n pub fn lower_right(self) -> Vector2F {\n Vector2F(self.0.zw())\n }\n\n // Mutators\n\n #[inline]\n pub fn set_origin_x(&mut self, x: f32) {\n self.0.set_x(x)\n }\n\n #[inline]\n pub fn set_origin_y(&mut self, y: f32) {\n self.0.set_y(y)\n }\n\n #[inline]\n pub fn contains_point(self, point: Vector2F) -> bool {\n // self.origin <= point && point <= self.lower_right\n let point = point.0.to_f32x4();\n self.0.concat_xy_xy(point).packed_le(point.concat_xy_zw(self.0)).all_true()\n }\n\n #[inline]\n pub fn contains_rect(self, other: RectF) -> bool {\n // self.origin <= other.origin && other.lower_right <= self.lower_right\n self.0.concat_xy_zw(other.0).packed_le(other.0.concat_xy_zw(self.0)).all_true()\n }\n\n #[inline]\n pub fn is_empty(self) -> bool {\n self.origin() == self.lower_right()\n }\n\n #[inline]\n pub fn union_point(self, point: Vector2F) -> RectF {\n RectF::from_points(self.origin().min(point), self.lower_right().max(point))\n }\n\n #[inline]\n pub fn union_rect(self, other: RectF) -> RectF {\n RectF::from_points(\n self.origin().min(other.origin()),\n self.lower_right().max(other.lower_right()),\n )\n }\n\n #[inline]\n pub fn intersects(self, other: RectF) -> bool {\n // self.origin < other.lower_right && other.origin < self.lower_right\n self.0.concat_xy_xy(other.0).packed_lt(other.0.concat_zw_zw(self.0)).all_true()\n }\n\n #[inline]\n pub fn intersection(self, other: RectF) -> Option {\n if !self.intersects(other) {\n None\n } else {\n Some(RectF::from_points(\n self.origin().max(other.origin()),\n self.lower_right().min(other.lower_right()),\n ))\n }\n }\n\n #[inline]\n pub fn min_x(self) -> f32 {\n self.0[0]\n }\n\n #[inline]\n pub fn min_y(self) -> f32 {\n self.0[1]\n }\n\n #[inline]\n pub fn max_x(self) -> f32 {\n self.0[2]\n }\n\n #[inline]\n pub fn max_y(self) -> f32 {\n self.0[3]\n }\n\n #[inline]\n pub fn center(self) -> Vector2F {\n self.origin() + self.size() * 0.5\n }\n\n /// Rounds all points to the nearest integer.\n #[inline]\n pub fn round(self) -> RectF {\n RectF(self.0.to_i32x4().to_f32x4())\n }\n\n #[inline]\n pub fn round_out(self) -> RectF {\n RectF::from_points(self.origin().floor(), self.lower_right().ceil())\n }\n\n #[inline]\n pub fn dilate(self, amount: A) -> RectF where A: IntoVector2F {\n let amount = amount.into_vector_2f();\n RectF::from_points(self.origin() - amount, self.lower_right() + amount)\n }\n\n #[inline]\n pub fn contract(self, amount: A) -> RectF where A: IntoVector2F {\n let amount = amount.into_vector_2f();\n RectF::from_points(self.origin() + amount, self.lower_right() - amount)\n }\n\n #[inline]\n pub fn to_i32(&self) -> RectI {\n RectI(self.0.to_i32x4())\n }\n}\n\nimpl Add for RectF {\n type Output = RectF;\n #[inline]\n fn add(self, other: Vector2F) -> RectF {\n RectF::new(self.origin() + other, self.size())\n }\n}\n\nimpl Add for RectF {\n type Output = RectF;\n #[inline]\n fn add(self, other: f32) -> RectF {\n RectF::new(self.origin() + other, self.size())\n }\n}\n\nimpl Mul for RectF {\n type Output = RectF;\n #[inline]\n fn mul(self, factors: Vector2F) -> RectF {\n RectF(self.0 * factors.0.concat_xy_xy(factors.0))\n }\n}\n\nimpl Mul for RectF {\n type Output = RectF;\n #[inline]\n fn mul(self, factor: f32) -> RectF {\n RectF(self.0 * F32x4::splat(factor))\n }\n}\n\nimpl Sub for RectF {\n type Output = RectF;\n #[inline]\n fn sub(self, other: Vector2F) -> RectF {\n RectF::new(self.origin() - other, self.size())\n }\n}\n\nimpl Sub for RectF {\n type Output = RectF;\n #[inline]\n fn sub(self, other: f32) -> RectF {\n RectF::new(self.origin() - other, self.size())\n }\n}\n\n/// NB: The origin is inclusive, while the lower right point is exclusive.\n#[derive(Clone, Copy, Debug, PartialEq, Default)]\npub struct RectI(pub I32x4);\n\nimpl RectI {\n #[inline]\n pub fn new(origin: Vector2I, size: Vector2I) -> RectI {\n RectI(origin.0.concat_xy_xy(origin.0 + size.0))\n }\n\n #[inline]\n pub fn from_points(origin: Vector2I, lower_right: Vector2I) -> RectI {\n RectI(origin.0.concat_xy_xy(lower_right.0))\n }\n\n // Accessors\n\n #[inline]\n pub fn origin(&self) -> Vector2I {\n Vector2I(self.0.xy())\n }\n\n #[inline]\n pub fn size(&self) -> Vector2I {\n Vector2I(self.0.zw() - self.0.xy())\n }\n\n #[inline]\n pub fn origin_x(self) -> i32 {\n self.0.x()\n }\n\n #[inline]\n pub fn origin_y(self) -> i32 {\n self.0.y()\n }\n\n #[inline]\n pub fn width(self) -> i32 {\n self.0.z() - self.0.x()\n }\n\n #[inline]\n pub fn height(self) -> i32 {\n self.0.w() - self.0.y()\n }\n\n #[inline]\n pub fn upper_right(&self) -> Vector2I {\n Vector2I(self.0.zy())\n }\n\n #[inline]\n pub fn lower_left(&self) -> Vector2I {\n Vector2I(self.0.xw())\n }\n\n #[inline]\n pub fn lower_right(&self) -> Vector2I {\n Vector2I(self.0.zw())\n }\n\n #[inline]\n pub fn scale(self, factor: i32) -> RectI {\n RectI(self.0 * I32x4::splat(factor))\n }\n\n #[inline]\n pub fn scale_xy(self, factors: Vector2I) -> RectI {\n RectI(self.0 * factors.0.concat_xy_xy(factors.0))\n }\n\n #[inline]\n pub fn min_x(self) -> i32 {\n self.0[0]\n }\n\n #[inline]\n pub fn min_y(self) -> i32 {\n self.0[1]\n }\n\n #[inline]\n pub fn max_x(self) -> i32 {\n self.0[2]\n }\n\n #[inline]\n pub fn max_y(self) -> i32 {\n self.0[3]\n }\n\n #[inline]\n pub fn intersects(self, other: RectI) -> bool {\n // self.origin < other.lower_right && other.origin < self.lower_right\n self.0.concat_xy_xy(other.0).packed_lt(other.0.concat_zw_zw(self.0)).all_true()\n }\n\n #[inline]\n pub fn intersection(self, other: RectI) -> Option {\n if !self.intersects(other) {\n None\n } else {\n Some(RectI::from_points(\n self.origin().max(other.origin()),\n self.lower_right().min(other.lower_right()),\n ))\n }\n }\n\n #[inline]\n pub fn contains_point(&self, point: Vector2I) -> bool {\n // self.origin <= point && point <= self.lower_right - 1\n let lower_right = self.lower_right() - 1;\n self.origin()\n .0\n .concat_xy_xy(point.0)\n .packed_le(point.0.concat_xy_xy(lower_right.0))\n .all_true()\n }\n\n #[inline]\n pub fn contract(self, amount: Vector2I) -> RectI {\n RectI::from_points(self.origin() + amount, self.lower_right() - amount)\n }\n\n #[inline]\n pub fn to_f32(&self) -> RectF {\n RectF(self.0.to_f32x4())\n }\n}\n" }, { "package": "pathfinder_geometry@0.5.1", "sourcePath": "src/transform2d.rs", "sha256": "36514cf158f7bd2118a1e2f2586e439f6484d61f0505919e8005fb733e3c917c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/geometry/src/basic/transform2d.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! 2D affine transforms.\n\nuse crate::line_segment::LineSegment2F;\nuse crate::rect::RectF;\nuse crate::transform3d::Transform4F;\nuse crate::unit_vector::UnitVector;\nuse crate::vector::{IntoVector2F, Vector2F, vec2f};\nuse pathfinder_simd::default::F32x4;\nuse std::ops::{Mul, MulAssign, Sub};\n\n/// A 2x2 matrix, optimized with SIMD, in column-major order.\n#[derive(Clone, Copy, Debug, PartialEq)]\npub struct Matrix2x2F(pub F32x4);\n\nimpl Default for Matrix2x2F {\n #[inline]\n fn default() -> Matrix2x2F {\n Self::from_scale(1.0)\n }\n}\n\nimpl Matrix2x2F {\n #[inline]\n pub fn from_scale(scale: S) -> Matrix2x2F where S: IntoVector2F {\n let scale = scale.into_vector_2f();\n Matrix2x2F(F32x4::new(scale.x(), 0.0, 0.0, scale.y()))\n }\n\n #[inline]\n pub fn from_rotation(theta: f32) -> Matrix2x2F {\n Matrix2x2F::from_rotation_vector(UnitVector::from_angle(theta))\n }\n\n #[inline]\n pub fn from_rotation_vector(vector: UnitVector) -> Matrix2x2F {\n Matrix2x2F((vector.0).0.to_f32x4().xyyx() * F32x4::new(1.0, 1.0, -1.0, 1.0))\n }\n\n #[inline]\n pub fn row_major(m00: f32, m01: f32, m10: f32, m11: f32) -> Matrix2x2F {\n Matrix2x2F(F32x4::new(m00, m10, m01, m11))\n }\n\n #[inline]\n pub fn entrywise_mul(&self, other: &Matrix2x2F) -> Matrix2x2F {\n Matrix2x2F(self.0 * other.0)\n }\n\n #[inline]\n pub fn adjugate(&self) -> Matrix2x2F {\n Matrix2x2F(self.0.wyzx() * F32x4::new(1.0, -1.0, -1.0, 1.0))\n }\n\n #[inline]\n pub fn det(&self) -> f32 {\n self.0[0] * self.0[3] - self.0[2] * self.0[1]\n }\n\n #[inline]\n pub fn inverse(&self) -> Matrix2x2F {\n Matrix2x2F(F32x4::splat(1.0 / self.det()) * self.adjugate().0)\n }\n\n #[inline]\n pub fn scale(&self, factor: f32) -> Matrix2x2F {\n Matrix2x2F(self.0 * F32x4::splat(factor))\n }\n\n /// Extracts the scale from this matrix.\n #[inline]\n pub fn extract_scale(&self) -> Vector2F {\n let squared = self.0 * self.0;\n Vector2F((squared.xy() + squared.zw()).sqrt())\n }\n\n #[inline]\n pub fn m11(&self) -> f32 {\n self.0[0]\n }\n\n #[inline]\n pub fn m21(&self) -> f32 {\n self.0[1]\n }\n\n #[inline]\n pub fn m12(&self) -> f32 {\n self.0[2]\n }\n\n #[inline]\n pub fn m22(&self) -> f32 {\n self.0[3]\n }\n}\n\nimpl Sub for Matrix2x2F {\n type Output = Matrix2x2F;\n #[inline]\n fn sub(self, other: Matrix2x2F) -> Matrix2x2F {\n Matrix2x2F(self.0 - other.0)\n }\n}\n\nimpl Mul for Matrix2x2F {\n type Output = Matrix2x2F;\n #[inline]\n fn mul(self, other: Matrix2x2F) -> Matrix2x2F {\n Matrix2x2F(self.0.xyxy() * other.0.xxzz() + self.0.zwzw() * other.0.yyww())\n }\n}\n\nimpl Mul for Matrix2x2F {\n type Output = Vector2F;\n #[inline]\n fn mul(self, vector: Vector2F) -> Vector2F {\n let halves = self.0 * vector.0.to_f32x4().xxyy();\n Vector2F(halves.xy() + halves.zw())\n }\n}\n\n/// An affine transform, optimized with SIMD.\n#[derive(Clone, Copy, Debug, PartialEq)]\npub struct Transform2F {\n pub matrix: Matrix2x2F,\n pub vector: Vector2F,\n}\n\nimpl Default for Transform2F {\n #[inline]\n fn default() -> Transform2F {\n Self::from_scale(vec2f(1.0, 1.0))\n }\n}\n\nimpl Transform2F {\n #[inline]\n pub fn from_scale(scale: S) -> Transform2F where S: IntoVector2F {\n let scale = scale.into_vector_2f();\n Transform2F {\n matrix: Matrix2x2F::from_scale(scale),\n vector: Vector2F::zero(),\n }\n }\n\n #[inline]\n pub fn from_rotation(theta: f32) -> Transform2F {\n Transform2F {\n matrix: Matrix2x2F::from_rotation(theta),\n vector: Vector2F::zero(),\n }\n }\n\n #[inline]\n pub fn from_rotation_vector(vector: UnitVector) -> Transform2F {\n Transform2F {\n matrix: Matrix2x2F::from_rotation_vector(vector),\n vector: Vector2F::zero(),\n }\n }\n\n #[inline]\n pub fn from_translation(vector: Vector2F) -> Transform2F {\n Transform2F { matrix: Matrix2x2F::default(), vector }\n }\n\n #[inline]\n pub fn from_scale_rotation_translation(scale: S, theta: f32, translation: Vector2F)\n -> Transform2F where S: IntoVector2F {\n let scale = scale.into_vector_2f();\n let rotation = Transform2F::from_rotation(theta);\n let translation = Transform2F::from_translation(translation);\n Transform2F::from_scale(scale) * rotation * translation\n }\n\n #[inline]\n pub fn row_major(m11: f32, m12: f32, m21: f32, m22: f32, m31: f32, m32: f32) -> Transform2F {\n Transform2F {\n matrix: Matrix2x2F::row_major(m11, m12, m21, m22),\n vector: Vector2F::new(m31, m32),\n }\n }\n\n // TODO(pcwalton): Optimize better with SIMD.\n #[inline]\n pub fn to_3d(&self) -> Transform4F {\n Transform4F::row_major(\n self.matrix.0[0],\n self.matrix.0[1],\n 0.0,\n self.vector.x(),\n self.matrix.0[2],\n self.matrix.0[3],\n 0.0,\n self.vector.y(),\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n )\n }\n\n #[inline]\n pub fn is_identity(&self) -> bool {\n *self == Transform2F::default()\n }\n\n /// Extracts the scale from this matrix.\n #[inline]\n pub fn extract_scale(&self) -> Vector2F {\n self.matrix.extract_scale()\n }\n\n #[inline]\n pub fn m11(&self) -> f32 {\n self.matrix.m11()\n }\n #[inline]\n pub fn m21(&self) -> f32 {\n self.matrix.m21()\n }\n #[inline]\n pub fn m12(&self) -> f32 {\n self.matrix.m12()\n }\n #[inline]\n pub fn m22(&self) -> f32 {\n self.matrix.m22()\n }\n #[inline]\n pub fn m31(&self) -> f32 {\n self.vector.x()\n }\n #[inline]\n pub fn m32(&self) -> f32 {\n self.vector.y()\n }\n\n #[inline]\n pub fn translate(&self, vector: Vector2F) -> Transform2F {\n Transform2F::from_translation(vector) * *self\n }\n\n #[inline]\n pub fn rotate(&self, theta: f32) -> Transform2F {\n Transform2F::from_rotation(theta) * *self\n }\n\n #[inline]\n pub fn scale(&self, scale: S) -> Transform2F where S: IntoVector2F {\n let scale = scale.into_vector_2f();\n Transform2F::from_scale(scale) * *self\n }\n\n /// Returns the translation part of this matrix.\n ///\n /// This decomposition assumes that scale, rotation, and translation are applied in that order.\n #[inline]\n pub fn translation(&self) -> Vector2F {\n self.vector\n }\n\n /// Returns the rotation angle of this matrix.\n ///\n /// This decomposition assumes that scale, rotation, and translation are applied in that order.\n #[inline]\n pub fn rotation(&self) -> f32 {\n f32::atan2(self.m21(), self.m11())\n }\n\n /// Returns the scale factor of this matrix.\n ///\n /// This decomposition assumes that scale, rotation, and translation are applied in that order.\n #[inline]\n pub fn scale_factor(&self) -> f32 {\n Vector2F(self.matrix.0.zw()).length()\n }\n\n #[inline]\n pub fn inverse(&self) -> Transform2F {\n let matrix_inv = self.matrix.inverse();\n let vector_inv = -(matrix_inv * self.vector);\n Transform2F { matrix: matrix_inv, vector: vector_inv }\n }\n}\n\nimpl Mul for Transform2F {\n type Output = Transform2F;\n #[inline]\n fn mul(self, other: Transform2F) -> Transform2F {\n Transform2F {\n matrix: self.matrix * other.matrix,\n vector: self * other.vector,\n }\n }\n}\n\nimpl Mul for Transform2F {\n type Output = Vector2F;\n #[inline]\n fn mul(self, vector: Vector2F) -> Vector2F {\n self.matrix * vector + self.vector\n }\n}\n\nimpl Mul for Transform2F {\n type Output = LineSegment2F;\n #[inline]\n fn mul(self, line_segment: LineSegment2F) -> LineSegment2F {\n LineSegment2F::new(self * line_segment.from(), self * line_segment.to())\n }\n}\n\nimpl Mul for Transform2F {\n type Output = RectF;\n #[inline]\n fn mul(self, rect: RectF) -> RectF {\n let (upper_left, upper_right) = (self * rect.origin(), self * rect.upper_right());\n let (lower_left, lower_right) = (self * rect.lower_left(), self * rect.lower_right());\n let min_point = upper_left.min(upper_right).min(lower_left).min(lower_right);\n let max_point = upper_left.max(upper_right).max(lower_left).max(lower_right);\n RectF::from_points(min_point, max_point)\n }\n}\n\nimpl MulAssign for Transform2F {\n #[inline]\n fn mul_assign(&mut self, other: Transform2F) {\n *self = *self * other\n }\n}\n" }, { "package": "pathfinder_geometry@0.5.1", "sourcePath": "src/transform3d.rs", "sha256": "d42aa24d02570376c429256f24891f489bd71ef257d9691f80b4fb6528e09275", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/geometry/src/basic/transform3d.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! 3D transforms that can be applied to paths.\n\nuse crate::rect::RectF;\nuse crate::transform2d::Matrix2x2F;\nuse crate::vector::{Vector2F, Vector2I, Vector3F, Vector4F};\nuse pathfinder_simd::default::F32x4;\nuse std::ops::{Add, Mul, MulAssign, Neg};\n\n/// An transform, optimized with SIMD.\n///\n/// In column-major order.\n#[derive(Clone, Copy, Debug, PartialEq)]\n#[repr(C)]\npub struct Transform4F {\n pub c0: F32x4,\n pub c1: F32x4,\n pub c2: F32x4,\n pub c3: F32x4,\n}\n\nimpl Default for Transform4F {\n #[inline]\n fn default() -> Transform4F {\n Transform4F {\n c0: F32x4::new(1.0, 0.0, 0.0, 0.0),\n c1: F32x4::new(0.0, 1.0, 0.0, 0.0),\n c2: F32x4::new(0.0, 0.0, 1.0, 0.0),\n c3: F32x4::new(0.0, 0.0, 0.0, 1.0),\n }\n }\n}\n\nimpl Transform4F {\n #[inline]\n pub fn row_major(\n m00: f32,\n m01: f32,\n m02: f32,\n m03: f32,\n m10: f32,\n m11: f32,\n m12: f32,\n m13: f32,\n m20: f32,\n m21: f32,\n m22: f32,\n m23: f32,\n m30: f32,\n m31: f32,\n m32: f32,\n m33: f32,\n ) -> Transform4F {\n Transform4F {\n c0: F32x4::new(m00, m10, m20, m30),\n c1: F32x4::new(m01, m11, m21, m31),\n c2: F32x4::new(m02, m12, m22, m32),\n c3: F32x4::new(m03, m13, m23, m33),\n }\n }\n\n #[inline]\n pub fn from_scale(scale: Vector4F) -> Transform4F {\n Transform4F {\n c0: F32x4::new(scale.x(), 0.0, 0.0, 0.0),\n c1: F32x4::new(0.0, scale.y(), 0.0, 0.0),\n c2: F32x4::new(0.0, 0.0, scale.z(), 0.0),\n c3: F32x4::new(0.0, 0.0, 0.0, 1.0),\n }\n }\n\n #[inline]\n pub fn from_uniform_scale(factor: f32) -> Transform4F {\n Transform4F::from_scale(Vector4F::splat(factor))\n }\n\n #[inline]\n pub fn from_translation(mut translation: Vector4F) -> Transform4F {\n translation.set_w(1.0);\n Transform4F { c3: translation.0, ..Transform4F::default() }\n }\n\n // TODO(pcwalton): Optimize.\n pub fn from_rotation(yaw: f32, pitch: f32, roll: f32) -> Transform4F {\n let (cos_b, sin_b) = (yaw.cos(), yaw.sin());\n let (cos_c, sin_c) = (pitch.cos(), pitch.sin());\n let (cos_a, sin_a) = (roll.cos(), roll.sin());\n let m00 = cos_a * cos_b;\n let m01 = cos_a * sin_b * sin_c - sin_a * cos_c;\n let m02 = cos_a * sin_b * cos_c + sin_a * sin_c;\n let m10 = sin_a * cos_b;\n let m11 = sin_a * sin_b * sin_c + cos_a * cos_c;\n let m12 = sin_a * sin_b * cos_c - cos_a * sin_c;\n let m20 = -sin_b;\n let m21 = cos_b * sin_c;\n let m22 = cos_b * cos_c;\n Transform4F::row_major(\n m00, m01, m02, 0.0, m10, m11, m12, 0.0, m20, m21, m22, 0.0, 0.0, 0.0, 0.0, 1.0,\n )\n }\n\n /// Creates a rotation matrix from the given quaternion.\n ///\n /// The quaternion is expected to be packed into a SIMD type (x, y, z, w) corresponding to\n /// x + yi + zj + wk.\n pub fn from_rotation_quaternion(q: F32x4) -> Transform4F {\n // TODO(pcwalton): Optimize better with more shuffles.\n let (mut sq, mut w, mut xy_xz_yz) = (q * q, q.wwww() * q, q.xxyy() * q.yzzy());\n sq += sq;\n w += w;\n xy_xz_yz += xy_xz_yz;\n let diag = F32x4::splat(1.0) - (sq.yxxy() + sq.zzyy());\n let (wx2, wy2, wz2) = (w.x(), w.y(), w.z());\n let (xy2, xz2, yz2) = (xy_xz_yz.x(), xy_xz_yz.y(), xy_xz_yz.z());\n Transform4F::row_major(\n diag.x(),\n xy2 - wz2,\n xz2 + wy2,\n 0.0,\n xy2 + wz2,\n diag.y(),\n yz2 - wx2,\n 0.0,\n xz2 - wy2,\n yz2 + wx2,\n diag.z(),\n 0.0,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n )\n }\n\n /// Just like `glOrtho()`.\n #[inline]\n pub fn from_ortho(\n left: f32,\n right: f32,\n bottom: f32,\n top: f32,\n near_val: f32,\n far_val: f32,\n ) -> Transform4F {\n let x_inv = 1.0 / (right - left);\n let y_inv = 1.0 / (top - bottom);\n let z_inv = 1.0 / (far_val - near_val);\n let tx = -(right + left) * x_inv;\n let ty = -(top + bottom) * y_inv;\n let tz = -(far_val + near_val) * z_inv;\n Transform4F::row_major(\n 2.0 * x_inv,\n 0.0,\n 0.0,\n tx,\n 0.0,\n 2.0 * y_inv,\n 0.0,\n ty,\n 0.0,\n 0.0,\n -2.0 * z_inv,\n tz,\n 0.0,\n 0.0,\n 0.0,\n 1.0,\n )\n }\n\n /// Linearly interpolate between transforms\n pub fn lerp(&self, weight: f32, other: &Transform4F) -> Transform4F {\n let c0 = self.c0 * F32x4::splat(weight) + other.c0 * F32x4::splat(1.0 - weight);\n let c1 = self.c1 * F32x4::splat(weight) + other.c1 * F32x4::splat(1.0 - weight);\n let c2 = self.c2 * F32x4::splat(weight) + other.c2 * F32x4::splat(1.0 - weight);\n let c3 = self.c3 * F32x4::splat(weight) + other.c3 * F32x4::splat(1.0 - weight);\n Transform4F { c0, c1, c2, c3 }\n }\n\n /// Just like `gluPerspective()`.\n #[inline]\n pub fn from_perspective(fov_y: f32, aspect: f32, z_near: f32, z_far: f32) -> Transform4F {\n let f = 1.0 / (fov_y * 0.5).tan();\n let z_denom = 1.0 / (z_near - z_far);\n let m00 = f / aspect;\n let m11 = f;\n let m22 = (z_far + z_near) * z_denom;\n let m23 = 2.0 * z_far * z_near * z_denom;\n let m32 = -1.0;\n Transform4F::row_major(\n m00, 0.0, 0.0, 0.0, 0.0, m11, 0.0, 0.0, 0.0, 0.0, m22, m23, 0.0, 0.0, m32, 0.0,\n )\n }\n\n /// Just like `gluLookAt()`.\n #[inline]\n pub fn looking_at(eye: Vector3F, center: Vector3F, mut up: Vector3F) -> Transform4F {\n let f = (center - eye).normalize();\n up = up.normalize();\n let s = f.cross(up);\n let u = s.normalize().cross(f);\n let minus_f = -f;\n\n // TODO(pcwalton): Use SIMD. This needs a matrix transpose:\n // https://fgiesen.wordpress.com/2013/07/09/simd-transposes-1/\n let transform = Transform4F::row_major(s.x(), s.y(), s.z(), 0.0,\n u.x(), u.y(), u.z(), 0.0,\n minus_f.x(), minus_f.y(), minus_f.z(), 0.0,\n 0.0, 0.0, 0.0, 1.0) *\n Transform4F::from_translation((-eye).to_4d());\n transform\n }\n\n // +- -+\n // | A B |\n // | C D |\n // +- -+\n #[inline]\n pub fn from_submatrices(\n a: Matrix2x2F,\n b: Matrix2x2F,\n c: Matrix2x2F,\n d: Matrix2x2F,\n ) -> Transform4F {\n Transform4F {\n c0: a.0.concat_xy_xy(c.0),\n c1: a.0.concat_zw_zw(c.0),\n c2: b.0.concat_xy_xy(d.0),\n c3: b.0.concat_zw_zw(d.0),\n }\n }\n\n #[inline]\n pub fn rotate(&self, yaw: f32, pitch: f32, roll: f32) -> Transform4F {\n Transform4F::from_rotation(yaw, pitch, roll) * *self\n }\n\n #[inline]\n pub fn scale(&self, scale: Vector4F) -> Transform4F {\n Transform4F::from_scale(scale) * *self\n }\n\n #[inline]\n pub fn uniform_scale(&self, scale: f32) -> Transform4F {\n Transform4F::from_uniform_scale(scale) * *self\n }\n\n #[inline]\n pub fn translate(&self, translation: Vector4F) -> Transform4F {\n Transform4F::from_translation(translation) * *self\n }\n\n #[inline]\n pub fn upper_left(&self) -> Matrix2x2F {\n Matrix2x2F(self.c0.concat_xy_xy(self.c1))\n }\n\n #[inline]\n pub fn upper_right(&self) -> Matrix2x2F {\n Matrix2x2F(self.c2.concat_xy_xy(self.c3))\n }\n\n #[inline]\n pub fn lower_left(&self) -> Matrix2x2F {\n Matrix2x2F(self.c0.concat_zw_zw(self.c1))\n }\n\n #[inline]\n pub fn lower_right(&self) -> Matrix2x2F {\n Matrix2x2F(self.c2.concat_zw_zw(self.c3))\n }\n\n // https://en.wikipedia.org/wiki/Invertible_matrix#Blockwise_inversion\n //\n // If A is the upper left submatrix of this matrix, this method assumes that A and the Schur\n // complement of A are invertible.\n pub fn inverse(&self) -> Transform4F {\n // Extract submatrices.\n let (a, b) = (self.upper_left(), self.upper_right());\n let (c, d) = (self.lower_left(), self.lower_right());\n\n // Compute temporary matrices.\n let a_inv = a.inverse();\n let x = c * a_inv;\n let y = (d - x * b).inverse();\n let z = a_inv * b;\n\n // Compute new submatrices.\n let (a_new, b_new) = (a_inv + z * y * x, -z * y);\n let (c_new, d_new) = (-y * x, y);\n\n // Construct inverse.\n Transform4F::from_submatrices(a_new, b_new, c_new, d_new)\n }\n\n pub fn approx_eq(&self, other: &Transform4F, epsilon: f32) -> bool {\n self.c0.approx_eq(other.c0, epsilon)\n && self.c1.approx_eq(other.c1, epsilon)\n && self.c2.approx_eq(other.c2, epsilon)\n && self.c3.approx_eq(other.c3, epsilon)\n }\n\n #[inline]\n pub fn as_ptr(&self) -> *const f32 {\n (&self.c0) as *const F32x4 as *const f32\n }\n\n #[inline]\n pub fn to_columns(&self) -> [F32x4; 4] {\n [self.c0, self.c1, self.c2, self.c3]\n }\n}\n\nimpl Mul for Transform4F {\n type Output = Transform4F;\n\n // https://stackoverflow.com/a/18508113\n #[inline]\n fn mul(self, other: Transform4F) -> Transform4F {\n return Transform4F {\n c0: mul_col(&self, other.c0),\n c1: mul_col(&self, other.c1),\n c2: mul_col(&self, other.c2),\n c3: mul_col(&self, other.c3),\n };\n\n #[inline]\n fn mul_col(a: &Transform4F, b_col: F32x4) -> F32x4 {\n a.c0 * b_col.xxxx() + a.c1 * b_col.yyyy() + a.c2 * b_col.zzzz() + a.c3 * b_col.wwww()\n }\n }\n}\n\nimpl Mul for Transform4F {\n type Output = Vector4F;\n\n #[inline]\n fn mul(self, vector: Vector4F) -> Vector4F {\n let term0 = self.c0 * F32x4::splat(vector.x());\n let term1 = self.c1 * F32x4::splat(vector.y());\n let term2 = self.c2 * F32x4::splat(vector.z());\n let term3 = self.c3 * F32x4::splat(vector.w());\n Vector4F(term0 + term1 + term2 + term3)\n }\n}\n\nimpl MulAssign for Transform4F {\n fn mul_assign(&mut self, other: Transform4F) {\n *self = *self * other\n }\n}\n\nimpl Add for Matrix2x2F {\n type Output = Matrix2x2F;\n #[inline]\n fn add(self, other: Matrix2x2F) -> Matrix2x2F {\n Matrix2x2F(self.0 + other.0)\n }\n}\n\nimpl Neg for Matrix2x2F {\n type Output = Matrix2x2F;\n #[inline]\n fn neg(self) -> Matrix2x2F {\n Matrix2x2F(-self.0)\n }\n}\n\n#[derive(Clone, Copy, Debug)]\npub struct Perspective {\n pub transform: Transform4F,\n pub window_size: Vector2I,\n}\n\nimpl Perspective {\n #[inline]\n pub fn new(transform: &Transform4F, window_size: Vector2I) -> Perspective {\n Perspective {\n transform: *transform,\n window_size,\n }\n }\n}\n\nimpl Mul for Perspective {\n type Output = Perspective;\n #[inline]\n fn mul(self, other: Transform4F) -> Perspective {\n Perspective {\n transform: self.transform * other,\n window_size: self.window_size,\n }\n }\n}\n\nimpl Mul for Perspective {\n type Output = Vector2F;\n #[inline]\n fn mul(self, vector: Vector2F) -> Vector2F {\n let point = (self.transform * vector.to_4d()).to_2d() * Vector2F::new(1.0, -1.0);\n (point + 1.0) * self.window_size.to_f32() * 0.5\n }\n}\n\nimpl Mul for Perspective {\n type Output = RectF;\n #[inline]\n fn mul(self, rect: RectF) -> RectF {\n let (upper_left, upper_right) = (self * rect.origin(), self * rect.upper_right());\n let (lower_left, lower_right) = (self * rect.lower_left(), self * rect.lower_right());\n let min_point = upper_left.min(upper_right).min(lower_left).min(lower_right);\n let max_point = upper_left.max(upper_right).max(lower_left).max(lower_right);\n RectF::from_points(min_point, max_point)\n }\n}\n\n#[cfg(test)]\nmod test {\n use crate::vector::Vector4F;\n use crate::transform3d::Transform4F;\n\n #[test]\n fn test_post_mul() {\n let a = Transform4F::row_major(\n 3.0, 1.0, 4.0, 5.0, 9.0, 2.0, 6.0, 5.0, 3.0, 5.0, 8.0, 9.0, 7.0, 9.0, 3.0, 2.0,\n );\n let b = Transform4F::row_major(\n 3.0, 8.0, 4.0, 6.0, 2.0, 6.0, 4.0, 3.0, 3.0, 8.0, 3.0, 2.0, 7.0, 9.0, 5.0, 0.0,\n );\n let c = Transform4F::row_major(\n 58.0, 107.0, 53.0, 29.0, 84.0, 177.0, 87.0, 72.0, 106.0, 199.0, 101.0, 49.0, 62.0,\n 152.0, 83.0, 75.0,\n );\n assert_eq!(a * b, c);\n }\n\n #[test]\n fn test_pre_mul() {\n let a = Transform4F::row_major(\n 3.0, 1.0, 4.0, 5.0, 9.0, 2.0, 6.0, 5.0, 3.0, 5.0, 8.0, 9.0, 7.0, 9.0, 3.0, 2.0,\n );\n let b = Transform4F::row_major(\n 3.0, 8.0, 4.0, 6.0, 2.0, 6.0, 4.0, 3.0, 3.0, 8.0, 3.0, 2.0, 7.0, 9.0, 5.0, 0.0,\n );\n let c = Transform4F::row_major(\n 135.0, 93.0, 110.0, 103.0, 93.0, 61.0, 85.0, 82.0, 104.0, 52.0, 90.0, 86.0, 117.0,\n 50.0, 122.0, 125.0,\n );\n assert_eq!(b * a, c);\n }\n\n #[test]\n fn test_transform_point() {\n let a = Transform4F::row_major(\n 3.0, 1.0, 4.0, 5.0, 9.0, 2.0, 6.0, 5.0, 3.0, 5.0, 8.0, 9.0, 7.0, 9.0, 3.0, 2.0,\n );\n let p = Vector4F::new(3.0, 8.0, 4.0, 6.0);\n let q = Vector4F::new(63.0, 97.0, 135.0, 117.0);\n assert_eq!(a * p, q);\n }\n\n #[test]\n fn test_inverse() {\n // Random matrix.\n let m = Transform4F::row_major(\n 0.86277982, 0.15986552, 0.90739898, 0.60066808, 0.17386167, 0.016353, 0.8535783,\n 0.12969608, 0.0946466, 0.43248631, 0.63480505, 0.08154603, 0.50305436, 0.48359687,\n 0.51057162, 0.24812012,\n );\n let p0 = Vector4F::new(0.95536648, 0.80633691, 0.16357357, 0.5477598);\n let p1 = m * p0;\n let m_inv = m.inverse();\n let m_inv_exp = Transform4F::row_major(\n -2.47290136,\n 3.48865688,\n -6.12298336,\n 6.17536696,\n 0.00124033357,\n -1.72561993,\n 2.16876606,\n 0.186227748,\n -0.375021729,\n 1.53883017,\n -0.0558194403,\n 0.121857058,\n 5.78300323,\n -6.87635769,\n 8.30196620,\n -9.10374060,\n );\n assert!(m_inv.approx_eq(&m_inv_exp, 0.0001));\n let p2 = m_inv * p1;\n assert!(p0.approx_eq(p2, 0.0001));\n }\n}\n" }, { "package": "pathfinder_geometry@0.5.1", "sourcePath": "src/unit_vector.rs", "sha256": "a6c30c98aaf859492746696b9d67999322b18117fbe891b53f38ccfca5b28a5f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/geometry/src/unit_vector.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! A utility module that allows unit vectors to be treated like angles.\n\nuse crate::vector::Vector2F;\nuse pathfinder_simd::default::F32x2;\n\n#[derive(Clone, Copy, Debug)]\npub struct UnitVector(pub Vector2F);\n\nimpl UnitVector {\n #[inline]\n pub fn from_angle(theta: f32) -> UnitVector {\n UnitVector(Vector2F::new(theta.cos(), theta.sin()))\n }\n\n /// Angle addition formula.\n #[inline]\n pub fn rotate_by(&self, other: UnitVector) -> UnitVector {\n let products = (self.0).0.to_f32x4().xyyx() * (other.0).0.to_f32x4().xyxy();\n UnitVector(Vector2F::new(products[0] - products[1], products[2] + products[3]))\n }\n\n /// Angle subtraction formula.\n #[inline]\n pub fn rev_rotate_by(&self, other: UnitVector) -> UnitVector {\n let products = (self.0).0.to_f32x4().xyyx() * (other.0).0.to_f32x4().xyxy();\n UnitVector(Vector2F::new(products[0] + products[1], products[2] - products[3]))\n }\n\n /// Half angle formula.\n #[inline]\n pub fn halve_angle(&self) -> UnitVector {\n let x = self.0.x();\n let term = F32x2::new(x, -x);\n UnitVector(Vector2F((F32x2::splat(0.5) * (F32x2::splat(1.0) + term)).max(F32x2::default())\n .sqrt()))\n }\n}\n" }, { "package": "pathfinder_geometry@0.5.1", "sourcePath": "src/util.rs", "sha256": "33a05481be71f5cf41128eeeb3e378248249d0f6f30ebc057e4c05f11344a0e5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/geometry/src/util.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Various utilities.\n\nuse std::f32;\n\npub const EPSILON: f32 = 0.001;\n\n/// Approximate equality.\n#[inline]\npub fn approx_eq(a: f32, b: f32) -> bool {\n f32::abs(a - b) <= EPSILON\n}\n\n/// Linear interpolation.\n#[inline]\npub fn lerp(a: f32, b: f32, t: f32) -> f32 {\n a + (b - a) * t\n}\n\n/// Clamping.\n#[inline]\npub fn clamp(x: f32, min_val: f32, max_val: f32) -> f32 {\n f32::min(max_val, f32::max(min_val, x))\n}\n\n/// Divides `a` by `b`, rounding up.\n#[inline]\npub fn alignup_i32(a: i32, b: i32) -> i32 {\n (a + b - 1) / b\n}\n" }, { "package": "pathfinder_geometry@0.5.1", "sourcePath": "src/vector.rs", "sha256": "cfcaad0394c5489a412bcdbbbe6928e1875d4a422eb5a7da1a3e415fdbc0790c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/geometry/src/basic/point.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! A SIMD-optimized point type.\n\nuse pathfinder_simd::default::{F32x2, F32x4, I32x2};\nuse std::hash::{Hash, Hasher};\nuse std::ops::{Add, AddAssign, Div, Mul, MulAssign, Neg, Sub, SubAssign};\n\n/// 2D points with 32-bit floating point coordinates.\n#[derive(Clone, Copy, Debug, Default)]\npub struct Vector2F(pub F32x2);\n\nimpl Vector2F {\n #[inline]\n pub fn new(x: f32, y: f32) -> Vector2F {\n Vector2F(F32x2::new(x, y))\n }\n\n #[inline]\n pub fn splat(value: f32) -> Vector2F {\n Vector2F(F32x2::splat(value))\n }\n\n #[inline]\n pub fn zero() -> Vector2F {\n Vector2F::default()\n }\n\n #[inline]\n pub fn to_3d(self) -> Vector3F {\n Vector3F(self.0.to_f32x4().concat_xy_zw(F32x4::new(0.0, 0.0, 0.0, 0.0)))\n }\n\n #[inline]\n pub fn to_4d(self) -> Vector4F {\n Vector4F(self.0.to_f32x4().concat_xy_zw(F32x4::new(0.0, 0.0, 0.0, 1.0)))\n }\n\n #[inline]\n pub fn x(self) -> f32 {\n self.0[0]\n }\n\n #[inline]\n pub fn y(self) -> f32 {\n self.0[1]\n }\n\n #[inline]\n pub fn set_x(&mut self, x: f32) {\n self.0[0] = x;\n }\n\n #[inline]\n pub fn set_y(&mut self, y: f32) {\n self.0[1] = y;\n }\n\n #[inline]\n pub fn min(self, other: Vector2F) -> Vector2F {\n Vector2F(self.0.min(other.0))\n }\n\n #[inline]\n pub fn max(self, other: Vector2F) -> Vector2F {\n Vector2F(self.0.max(other.0))\n }\n\n #[inline]\n pub fn clamp(self, min_val: Vector2F, max_val: Vector2F) -> Vector2F {\n self.max(min_val).min(max_val)\n }\n\n #[inline]\n pub fn det(self, other: Vector2F) -> f32 {\n self.x() * other.y() - self.y() * other.x()\n }\n\n #[inline]\n pub fn dot(self, other: Vector2F) -> f32 {\n let xy = self.0 * other.0;\n xy.x() + xy.y()\n }\n\n #[inline]\n pub fn floor(self) -> Vector2F {\n Vector2F(self.0.floor())\n }\n\n #[inline]\n pub fn ceil(self) -> Vector2F {\n Vector2F(self.0.ceil())\n }\n\n /// Rounds both coordinates to the nearest integer.\n #[inline]\n pub fn round(self) -> Vector2F {\n Vector2F(self.0.to_i32x2().to_f32x2())\n }\n\n /// Treats this point as a vector and calculates its squared length.\n #[inline]\n pub fn square_length(self) -> f32 {\n let squared = self.0 * self.0;\n squared[0] + squared[1]\n }\n\n /// Treats this point as a vector and calculates its length.\n #[inline]\n pub fn length(self) -> f32 {\n f32::sqrt(self.square_length())\n }\n\n /// Treats this point as a vector and normalizes it.\n #[inline]\n pub fn normalize(self) -> Vector2F {\n self * (1.0 / self.length())\n }\n\n /// Swaps y and x.\n #[inline]\n pub fn yx(self) -> Vector2F {\n Vector2F(self.0.yx())\n }\n\n /// Returns the coefficient when the given vector `a` is projected onto this one.\n ///\n /// That is, if this vector is `v` and this function returns `c`, then `proj_v a = cv`. In\n /// other words, this function computes `(a⋅v) / (v⋅v)`.\n #[inline]\n pub fn projection_coefficient(self, a: Vector2F) -> f32 {\n a.dot(self) / self.square_length()\n }\n\n #[inline]\n pub fn is_zero(self) -> bool {\n self == Vector2F::zero()\n }\n\n #[inline]\n pub fn lerp(self, other: Vector2F, t: f32) -> Vector2F {\n self + (other - self) * t\n }\n\n #[inline]\n pub fn to_i32(self) -> Vector2I {\n Vector2I(self.0.to_i32x2())\n }\n}\n\n/// A convenience alias for `Vector2F::new()`.\n#[inline]\npub fn vec2f(x: f32, y: f32) -> Vector2F {\n Vector2F::new(x, y)\n}\n\nimpl PartialEq for Vector2F {\n #[inline]\n fn eq(&self, other: &Vector2F) -> bool {\n self.0.packed_eq(other.0).all_true()\n }\n}\n\nimpl Add for Vector2F {\n type Output = Vector2F;\n #[inline]\n fn add(self, other: Vector2F) -> Vector2F {\n Vector2F(self.0 + other.0)\n }\n}\n\nimpl Add for Vector2F {\n type Output = Vector2F;\n #[inline]\n fn add(self, other: f32) -> Vector2F {\n self + Vector2F::splat(other)\n }\n}\n\nimpl AddAssign for Vector2F {\n #[inline]\n fn add_assign(&mut self, other: Vector2F) {\n *self = *self + other\n }\n}\n\nimpl Sub for Vector2F {\n type Output = Vector2F;\n #[inline]\n fn sub(self, other: Vector2F) -> Vector2F {\n Vector2F(self.0 - other.0)\n }\n}\n\nimpl Sub for Vector2F {\n type Output = Vector2F;\n #[inline]\n fn sub(self, other: f32) -> Vector2F {\n self - Vector2F::splat(other)\n }\n}\n\nimpl SubAssign for Vector2F {\n #[inline]\n fn sub_assign(&mut self, other: Vector2F) {\n *self = *self - other\n }\n}\n\nimpl Mul for Vector2F {\n type Output = Vector2F;\n #[inline]\n fn mul(self, other: Vector2F) -> Vector2F {\n Vector2F(self.0 * other.0)\n }\n}\n\nimpl Mul for Vector2F {\n type Output = Vector2F;\n #[inline]\n fn mul(self, other: f32) -> Vector2F {\n self * Vector2F::splat(other)\n }\n}\n\nimpl MulAssign for Vector2F {\n #[inline]\n fn mul_assign(&mut self, other: Vector2F) {\n *self = *self * other\n }\n}\n\nimpl MulAssign for Vector2F {\n #[inline]\n fn mul_assign(&mut self, other: f32) {\n *self = *self * other\n }\n}\n\nimpl Div for Vector2F {\n type Output = Vector2F;\n #[inline]\n fn div(self, other: Vector2F) -> Vector2F {\n Vector2F(self.0 / other.0)\n }\n}\n\nimpl Div for Vector2F {\n type Output = Vector2F;\n #[inline]\n fn div(self, other: f32) -> Vector2F {\n self / Vector2F::splat(other)\n }\n}\n\nimpl Neg for Vector2F {\n type Output = Vector2F;\n #[inline]\n fn neg(self) -> Vector2F {\n Vector2F(-self.0)\n }\n}\n\n/// Either a scalar or a `Vector2F`.\n/// \n/// Scalars will be automatically splatted (i.e. `x` becomes `vec2f(x, x)`).\n/// \n/// Be judicious with the use of this trait. Only use it if it will aid readability without the\n/// potential to introduce bugs.\npub trait IntoVector2F {\n fn into_vector_2f(self) -> Vector2F;\n}\n\nimpl IntoVector2F for Vector2F {\n #[inline]\n fn into_vector_2f(self) -> Vector2F {\n self\n }\n}\n\nimpl IntoVector2F for f32 {\n #[inline]\n fn into_vector_2f(self) -> Vector2F {\n Vector2F::splat(self)\n }\n}\n\n/// 2D points with 32-bit signed integer coordinates.\n#[derive(Clone, Copy, Debug, Default)]\npub struct Vector2I(pub I32x2);\n\nimpl Vector2I {\n #[inline]\n pub fn new(x: i32, y: i32) -> Vector2I {\n Vector2I(I32x2::new(x, y))\n }\n\n #[inline]\n pub fn splat(value: i32) -> Vector2I {\n Vector2I(I32x2::splat(value))\n }\n\n #[inline]\n pub fn zero() -> Vector2I {\n Vector2I::default()\n }\n\n #[inline]\n pub fn x(self) -> i32 {\n self.0[0]\n }\n\n #[inline]\n pub fn y(self) -> i32 {\n self.0[1]\n }\n\n #[inline]\n pub fn set_x(&mut self, x: i32) {\n self.0[0] = x;\n }\n\n #[inline]\n pub fn set_y(&mut self, y: i32) {\n self.0[1] = y;\n }\n\n #[inline]\n pub fn min(self, other: Vector2I) -> Vector2I {\n Vector2I(self.0.min(other.0))\n }\n\n #[inline]\n pub fn max(self, other: Vector2I) -> Vector2I {\n Vector2I(self.0.max(other.0))\n }\n\n #[inline]\n pub fn to_f32(self) -> Vector2F {\n Vector2F(self.0.to_f32x2())\n }\n}\n\n/// A convenience alias for `Vector2I::new()`.\n#[inline]\npub fn vec2i(x: i32, y: i32) -> Vector2I {\n Vector2I::new(x, y)\n}\n\nimpl Add for Vector2I {\n type Output = Vector2I;\n #[inline]\n fn add(self, other: Vector2I) -> Vector2I {\n Vector2I(self.0 + other.0)\n }\n}\n\nimpl Add for Vector2I {\n type Output = Vector2I;\n #[inline]\n fn add(self, other: i32) -> Vector2I {\n self + Vector2I::splat(other)\n }\n}\n\nimpl AddAssign for Vector2I {\n #[inline]\n fn add_assign(&mut self, other: Vector2I) {\n self.0 += other.0\n }\n}\n\nimpl Neg for Vector2I {\n type Output = Vector2I;\n #[inline]\n fn neg(self) -> Vector2I {\n Vector2I(-self.0)\n }\n}\n\nimpl Sub for Vector2I {\n type Output = Vector2I;\n #[inline]\n fn sub(self, other: Vector2I) -> Vector2I {\n Vector2I(self.0 - other.0)\n }\n}\n\nimpl Sub for Vector2I {\n type Output = Vector2I;\n #[inline]\n fn sub(self, other: i32) -> Vector2I {\n self - Vector2I::splat(other)\n }\n}\n\nimpl Mul for Vector2I {\n type Output = Vector2I;\n #[inline]\n fn mul(self, other: Vector2I) -> Vector2I {\n Vector2I(self.0 * other.0)\n }\n}\n\nimpl Mul for Vector2I {\n type Output = Vector2I;\n #[inline]\n fn mul(self, other: i32) -> Vector2I {\n self * Vector2I::splat(other)\n }\n}\n\nimpl PartialEq for Vector2I {\n #[inline]\n fn eq(&self, other: &Vector2I) -> bool {\n self.0.packed_eq(other.0).all_true()\n }\n}\n\nimpl Eq for Vector2I {}\n\nimpl Hash for Vector2I {\n #[inline]\n fn hash(&self, state: &mut H) where H: Hasher {\n self.x().hash(state);\n self.y().hash(state);\n }\n}\n\n/// 3D points.\n///\n/// The w value in the SIMD vector is always 0.0.\n#[derive(Clone, Copy, Debug, Default, PartialEq)]\npub struct Vector3F(pub F32x4);\n\nimpl Vector3F {\n #[inline]\n pub fn new(x: f32, y: f32, z: f32) -> Vector3F {\n Vector3F(F32x4::new(x, y, z, 0.0))\n }\n\n #[inline]\n pub fn splat(x: f32) -> Vector3F {\n let mut vector = F32x4::splat(x);\n vector.set_w(0.0);\n Vector3F(vector)\n }\n\n /// Truncates this vector to 2D.\n #[inline]\n pub fn to_2d(self) -> Vector2F {\n Vector2F(self.0.xy())\n }\n\n /// Converts this vector to an equivalent 3D homogeneous one with a w component of 1.0.\n #[inline]\n pub fn to_4d(self) -> Vector4F {\n let mut vector = self.0;\n vector.set_w(1.0);\n Vector4F(vector)\n }\n\n #[inline]\n pub fn cross(self, other: Vector3F) -> Vector3F {\n Vector3F(self.0.yzxw() * other.0.zxyw() - self.0.zxyw() * other.0.yzxw())\n }\n\n #[inline]\n pub fn square_length(self) -> f32 {\n let squared = self.0 * self.0;\n squared[0] + squared[1] + squared[2]\n }\n\n #[inline]\n pub fn length(self) -> f32 {\n f32::sqrt(self.square_length())\n }\n\n #[inline]\n pub fn normalize(self) -> Vector3F {\n Vector3F(self.0 * F32x4::splat(1.0 / self.length()))\n }\n\n #[inline]\n pub fn x(self) -> f32 {\n self.0[0]\n }\n\n #[inline]\n pub fn y(self) -> f32 {\n self.0[1]\n }\n\n #[inline]\n pub fn z(self) -> f32 {\n self.0[2]\n }\n\n #[inline]\n pub fn scale(self, factor: f32) -> Vector3F {\n Vector3F(self.0 * F32x4::splat(factor))\n }\n}\n\nimpl Add for Vector3F {\n type Output = Vector3F;\n #[inline]\n fn add(self, other: Vector3F) -> Vector3F {\n Vector3F(self.0 + other.0)\n }\n}\n\nimpl AddAssign for Vector3F {\n #[inline]\n fn add_assign(&mut self, other: Vector3F) {\n self.0 += other.0\n }\n}\n\nimpl Neg for Vector3F {\n type Output = Vector3F;\n #[inline]\n fn neg(self) -> Vector3F {\n Vector3F(self.0 * F32x4::new(-1.0, -1.0, -1.0, 0.0))\n }\n}\n\nimpl Sub for Vector3F {\n type Output = Vector3F;\n #[inline]\n fn sub(self, other: Vector3F) -> Vector3F {\n Vector3F(self.0 - other.0)\n }\n}\n\n/// 3D homogeneous points.\n#[derive(Clone, Copy, Debug, PartialEq)]\npub struct Vector4F(pub F32x4);\n\nimpl Vector4F {\n #[inline]\n pub fn new(x: f32, y: f32, z: f32, w: f32) -> Vector4F {\n Vector4F(F32x4::new(x, y, z, w))\n }\n\n #[inline]\n pub fn splat(value: f32) -> Vector4F {\n Vector4F(F32x4::splat(value))\n }\n\n #[inline]\n pub fn to_2d(self) -> Vector2F {\n self.to_3d().to_2d()\n }\n\n /// Performs perspective division to convert this vector to 3D.\n #[inline]\n pub fn to_3d(self) -> Vector3F {\n let mut vector = self.0 * F32x4::splat(1.0 / self.w());\n vector.set_w(0.0);\n Vector3F(vector)\n }\n\n #[inline]\n pub fn x(self) -> f32 {\n self.0[0]\n }\n\n #[inline]\n pub fn y(self) -> f32 {\n self.0[1]\n }\n\n #[inline]\n pub fn z(self) -> f32 {\n self.0[2]\n }\n\n #[inline]\n pub fn w(self) -> f32 {\n self.0[3]\n }\n\n #[inline]\n pub fn scale(self, x: f32) -> Vector4F {\n let mut factors = F32x4::splat(x);\n factors[3] = 1.0;\n Vector4F(self.0 * factors)\n }\n\n #[inline]\n pub fn set_x(&mut self, x: f32) {\n self.0[0] = x\n }\n\n #[inline]\n pub fn set_y(&mut self, y: f32) {\n self.0[1] = y\n }\n\n #[inline]\n pub fn set_z(&mut self, z: f32) {\n self.0[2] = z\n }\n\n #[inline]\n pub fn set_w(&mut self, w: f32) {\n self.0[3] = w\n }\n\n #[inline]\n pub fn approx_eq(self, other: Vector4F, epsilon: f32) -> bool {\n self.0.approx_eq(other.0, epsilon)\n }\n\n /// Checks to see whether this *homogeneous* coordinate equals zero.\n ///\n /// Note that since this treats the coordinate as a homogeneous coordinate, the `w` is ignored.\n // TODO(pcwalton): Optimize with SIMD.\n #[inline]\n pub fn is_zero(self) -> bool {\n self.x() == 0.0 && self.y() == 0.0 && self.z() == 0.0\n }\n\n #[inline]\n pub fn lerp(self, other: Vector4F, t: f32) -> Vector4F {\n Vector4F(self.0 + (other.0 - self.0) * F32x4::splat(t))\n }\n}\n\nimpl Add for Vector4F {\n type Output = Vector4F;\n #[inline]\n fn add(self, other: Vector4F) -> Vector4F {\n Vector4F(self.0 + other.0)\n }\n}\n\nimpl AddAssign for Vector4F {\n #[inline]\n fn add_assign(&mut self, other: Vector4F) {\n self.0 += other.0\n }\n}\n\nimpl Mul for Vector4F {\n type Output = Vector4F;\n #[inline]\n fn mul(self, other: Vector4F) -> Vector4F {\n Vector4F(self.0 * other.0)\n }\n}\n\nimpl Neg for Vector4F {\n type Output = Vector4F;\n /// NB: This does not negate w, because that is rarely what you what for homogeneous\n /// coordinates.\n #[inline]\n fn neg(self) -> Vector4F {\n Vector4F(self.0 * F32x4::new(-1.0, -1.0, -1.0, 1.0))\n }\n}\n\nimpl Sub for Vector4F {\n type Output = Vector4F;\n #[inline]\n fn sub(self, other: Vector4F) -> Vector4F {\n Vector4F(self.0 - other.0)\n }\n}\n\nimpl Default for Vector4F {\n #[inline]\n fn default() -> Vector4F {\n let mut point = F32x4::default();\n point.set_w(1.0);\n Vector4F(point)\n }\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "build.rs", "sha256": "b515ee342a21c17c65615c6001a90bf1d131e4a46046ae1b0ed92e33d258bb58", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/build.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nextern crate rustc_version;\n\nuse rustc_version::Channel;\n\nfn main() {\n // Assert we haven't travelled back in time\n assert!(rustc_version::version().unwrap().major >= 1);\n\n // Set cfg flags depending on release channel\n match rustc_version::version_meta().unwrap().channel {\n Channel::Nightly => {\n println!(\"cargo:rustc-cfg=pf_rustc_nightly\");\n }\n _ => {}\n }\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/arm/mod.rs", "sha256": "cb8566a07af8d8a95f6124bbbd1d94f69b3f084103e6452de3cef364909da849", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/arm.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse std::arch::aarch64::{self, float32x2_t, float32x4_t, int32x2_t, int32x4_t};\nuse std::arch::aarch64::{uint32x2_t, uint32x4_t};\nuse std::f32;\nuse std::fmt::{self, Debug, Formatter};\nuse std::intrinsics::simd::*;\nuse std::mem;\nuse std::ops::{Add, BitAnd, BitOr, Div, Index, IndexMut, Mul, Not, Shr, Sub};\n\nmod swizzle_f32x4;\nmod swizzle_i32x4;\n\n#[repr(simd)]\npub(crate) struct Simd([T; N]);\n\nmacro_rules! simd_shuffle2 {\n ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+> $idx:expr $(,)?) => {{\n struct ConstParam<$(const $imm: $ty),+>;\n impl<$(const $imm: $ty),+> ConstParam<$($imm),+> {\n const IDX: Simd = Simd($idx);\n }\n\n simd_shuffle($x, $y, ConstParam::<$($imm),+>::IDX)\n }};\n ($x:expr, $y:expr, $idx:expr $(,)?) => {{\n const IDX: Simd = Simd($idx);\n simd_shuffle($x, $y, IDX)\n }};\n}\n\nmacro_rules! simd_shuffle4 {\n ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+> $idx:expr $(,)?) => {{\n struct ConstParam<$(const $imm: $ty),+>;\n impl<$(const $imm: $ty),+> ConstParam<$($imm),+> {\n const IDX: Simd = Simd($idx);\n }\n\n simd_shuffle($x, $y, ConstParam::<$($imm),+>::IDX)\n }};\n ($x:expr, $y:expr, $idx:expr $(,)?) => {{\n const IDX: Simd = Simd($idx);\n simd_shuffle($x, $y, IDX)\n }};\n}\n\n// Two 32-bit floats\n\n#[derive(Clone, Copy)]\npub struct F32x2(pub float32x2_t);\n\nimpl F32x2 {\n // Constructors\n\n #[inline]\n pub fn new(a: f32, b: f32) -> F32x2 {\n unsafe { F32x2(mem::transmute([a, b])) }\n }\n\n #[inline]\n pub fn splat(x: f32) -> F32x2 {\n F32x2::new(x, x)\n }\n\n // Basic operations\n\n #[inline]\n pub fn approx_recip(self) -> F32x2 {\n unsafe { F32x2(vrecpe_v2f32(self.0)) }\n }\n\n #[inline]\n pub fn min(self, other: F32x2) -> F32x2 {\n unsafe { F32x2(simd_minimum_number_nsz(self.0, other.0)) }\n }\n\n #[inline]\n pub fn max(self, other: F32x2) -> F32x2 {\n unsafe { F32x2(simd_maximum_number_nsz(self.0, other.0)) }\n }\n\n #[inline]\n pub fn clamp(self, min: F32x2, max: F32x2) -> F32x2 {\n self.max(min).min(max)\n }\n\n #[inline]\n pub fn abs(self) -> F32x2 {\n unsafe { F32x2(fabs_v2f32(self.0)) }\n }\n\n #[inline]\n pub fn floor(self) -> F32x2 {\n unsafe { F32x2(floor_v2f32(self.0)) }\n }\n\n #[inline]\n pub fn ceil(self) -> F32x2 {\n unsafe { F32x2(ceil_v2f32(self.0)) }\n }\n\n #[inline]\n pub fn sqrt(self) -> F32x2 {\n unsafe { F32x2(sqrt_v2f32(self.0)) }\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: F32x2) -> U32x2 {\n unsafe { U32x2(simd_eq(self.0, other.0)) }\n }\n\n #[inline]\n pub fn packed_gt(self, other: F32x2) -> U32x2 {\n unsafe { U32x2(simd_gt(self.0, other.0)) }\n }\n\n #[inline]\n pub fn packed_lt(self, other: F32x2) -> U32x2 {\n unsafe { U32x2(simd_lt(self.0, other.0)) }\n }\n\n #[inline]\n pub fn packed_le(self, other: F32x2) -> U32x2 {\n unsafe { U32x2(simd_le(self.0, other.0)) }\n }\n\n // Conversions\n\n #[inline]\n pub fn to_f32x4(self) -> F32x4 {\n self.concat_xy_xy(F32x2::default())\n }\n\n /// Converts these packed floats to integers via rounding.\n #[inline]\n pub fn to_i32x2(self) -> I32x2 {\n unsafe { I32x2(simd_cast(round_v2f32(self.0))) }\n }\n\n #[inline]\n pub fn to_i32x4(self) -> I32x4 {\n self.to_i32x2().concat_xy_xy(I32x2::default())\n }\n\n // Swizzle\n\n #[inline]\n pub fn yx(self) -> F32x2 {\n unsafe { F32x2(simd_shuffle2!(self.0, self.0, [1, 0])) }\n }\n\n // Concatenations\n\n #[inline]\n pub fn concat_xy_xy(self, other: F32x2) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, other.0, [0, 1, 2, 3])) }\n }\n}\n\nimpl Default for F32x2 {\n #[inline]\n fn default() -> F32x2 {\n F32x2::new(0.0, 0.0)\n }\n}\n\nimpl Debug for F32x2 {\n #[inline]\n fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {\n write!(f, \"<{}, {}>\", self[0], self[1])\n }\n}\n\nimpl Index for F32x2 {\n type Output = f32;\n #[inline]\n fn index(&self, index: usize) -> &f32 {\n unsafe {\n assert!(index < 2);\n let ptr = &self.0 as *const float32x2_t as *const f32;\n mem::transmute::<*const f32, &f32>(ptr.offset(index as isize))\n }\n }\n}\n\nimpl IndexMut for F32x2 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut f32 {\n unsafe {\n assert!(index < 2);\n let ptr = &mut self.0 as *mut float32x2_t as *mut f32;\n mem::transmute::<*mut f32, &mut f32>(ptr.offset(index as isize))\n }\n }\n}\n\nimpl Add for F32x2 {\n type Output = F32x2;\n #[inline]\n fn add(self, other: F32x2) -> F32x2 {\n unsafe { F32x2(simd_add(self.0, other.0)) }\n }\n}\n\nimpl Div for F32x2 {\n type Output = F32x2;\n #[inline]\n fn div(self, other: F32x2) -> F32x2 {\n unsafe { F32x2(simd_div(self.0, other.0)) }\n }\n}\n\nimpl Mul for F32x2 {\n type Output = F32x2;\n #[inline]\n fn mul(self, other: F32x2) -> F32x2 {\n unsafe { F32x2(simd_mul(self.0, other.0)) }\n }\n}\n\nimpl Sub for F32x2 {\n type Output = F32x2;\n #[inline]\n fn sub(self, other: F32x2) -> F32x2 {\n unsafe { F32x2(simd_sub(self.0, other.0)) }\n }\n}\n\nimpl PartialEq for F32x2 {\n #[inline]\n fn eq(&self, other: &F32x2) -> bool {\n self.packed_eq(*other).all_true()\n }\n}\n\n// Four 32-bit floats\n\n#[derive(Clone, Copy)]\npub struct F32x4(pub float32x4_t);\n\nimpl F32x4 {\n #[inline]\n pub fn new(a: f32, b: f32, c: f32, d: f32) -> F32x4 {\n unsafe { F32x4(mem::transmute([a, b, c, d])) }\n }\n\n #[inline]\n pub fn splat(x: f32) -> F32x4 {\n F32x4::new(x, x, x, x)\n }\n\n // Basic operations\n\n #[inline]\n pub fn approx_recip(self) -> F32x4 {\n unsafe { F32x4(vrecpe_v4f32(self.0)) }\n }\n\n #[inline]\n pub fn min(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(simd_minimum_number_nsz(self.0, other.0)) }\n }\n\n #[inline]\n pub fn max(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(simd_maximum_number_nsz(self.0, other.0)) }\n }\n\n #[inline]\n pub fn clamp(self, min: F32x4, max: F32x4) -> F32x4 {\n self.max(min).min(max)\n }\n\n #[inline]\n pub fn abs(self) -> F32x4 {\n unsafe { F32x4(fabs_v4f32(self.0)) }\n }\n\n #[inline]\n pub fn floor(self) -> F32x4 {\n unsafe { F32x4(floor_v4f32(self.0)) }\n }\n\n #[inline]\n pub fn ceil(self) -> F32x4 {\n unsafe { F32x4(ceil_v4f32(self.0)) }\n }\n\n #[inline]\n pub fn sqrt(self) -> F32x4 {\n unsafe { F32x4(sqrt_v4f32(self.0)) }\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: F32x4) -> U32x4 {\n unsafe { U32x4(simd_eq(self.0, other.0)) }\n }\n\n #[inline]\n pub fn packed_gt(self, other: F32x4) -> U32x4 {\n unsafe { U32x4(simd_gt(self.0, other.0)) }\n }\n\n #[inline]\n pub fn packed_le(self, other: F32x4) -> U32x4 {\n unsafe { U32x4(simd_le(self.0, other.0)) }\n }\n\n #[inline]\n pub fn packed_lt(self, other: F32x4) -> U32x4 {\n unsafe { U32x4(simd_lt(self.0, other.0)) }\n }\n\n // Swizzle conversions\n\n #[inline]\n pub fn xy(self) -> F32x2 {\n unsafe { F32x2(simd_shuffle2!(self.0, self.0, [0, 1])) }\n }\n\n #[inline]\n pub fn yx(self) -> F32x2 {\n unsafe { F32x2(simd_shuffle2!(self.0, self.0, [1, 0])) }\n }\n\n #[inline]\n pub fn xw(self) -> F32x2 {\n unsafe { F32x2(simd_shuffle2!(self.0, self.0, [0, 3])) }\n }\n\n #[inline]\n pub fn zy(self) -> F32x2 {\n unsafe { F32x2(simd_shuffle2!(self.0, self.0, [2, 1])) }\n }\n\n #[inline]\n pub fn zw(self) -> F32x2 {\n unsafe { F32x2(simd_shuffle2!(self.0, self.0, [2, 3])) }\n }\n\n // Concatenations\n\n #[inline]\n pub fn concat_xy_xy(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, other.0, [0, 1, 4, 5])) }\n }\n\n #[inline]\n pub fn concat_xy_zw(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, other.0, [0, 1, 6, 7])) }\n }\n\n #[inline]\n pub fn concat_zw_zw(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, other.0, [2, 3, 6, 7])) }\n }\n\n #[inline]\n pub fn concat_wz_yx(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, other.0, [3, 2, 5, 4])) }\n }\n\n // Conversions\n\n /// Converts these packed floats to integers via rounding.\n #[inline]\n pub fn to_i32x4(self) -> I32x4 {\n unsafe { I32x4(simd_cast(round_v4f32(self.0))) }\n }\n}\n\nimpl Default for F32x4 {\n #[inline]\n fn default() -> F32x4 {\n F32x4::new(0.0, 0.0, 0.0, 0.0)\n }\n}\n\nimpl Index for F32x4 {\n type Output = f32;\n #[inline]\n fn index(&self, index: usize) -> &f32 {\n unsafe {\n assert!(index < 4);\n let ptr = &self.0 as *const float32x4_t as *const f32;\n mem::transmute::<*const f32, &f32>(ptr.offset(index as isize))\n }\n }\n}\n\nimpl IndexMut for F32x4 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut f32 {\n unsafe {\n assert!(index < 4);\n let ptr = &mut self.0 as *mut float32x4_t as *mut f32;\n mem::transmute::<*mut f32, &mut f32>(ptr.offset(index as isize))\n }\n }\n}\n\nimpl Debug for F32x4 {\n #[inline]\n fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {\n write!(f, \"<{}, {}, {}, {}>\", self[0], self[1], self[2], self[3])\n }\n}\n\nimpl PartialEq for F32x4 {\n #[inline]\n fn eq(&self, other: &F32x4) -> bool {\n self.packed_eq(*other).all_true()\n }\n}\n\nimpl Add for F32x4 {\n type Output = F32x4;\n #[inline]\n fn add(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(simd_add(self.0, other.0)) }\n }\n}\n\nimpl Div for F32x4 {\n type Output = F32x4;\n #[inline]\n fn div(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(simd_div(self.0, other.0)) }\n }\n}\n\nimpl Mul for F32x4 {\n type Output = F32x4;\n #[inline]\n fn mul(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(simd_mul(self.0, other.0)) }\n }\n}\n\nimpl Sub for F32x4 {\n type Output = F32x4;\n #[inline]\n fn sub(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(simd_sub(self.0, other.0)) }\n }\n}\n\n// Two 32-bit signed integers\n\n#[derive(Clone, Copy, Debug)]\npub struct I32x2(pub int32x2_t);\n\nimpl I32x2 {\n #[inline]\n pub fn new(x: i32, y: i32) -> I32x2 {\n unsafe { I32x2(mem::transmute([x, y])) }\n }\n\n #[inline]\n pub fn splat(x: i32) -> I32x2 {\n I32x2::new(x, x)\n }\n\n // Accessors\n\n #[inline]\n pub fn x(self) -> i32 {\n self[0]\n }\n\n #[inline]\n pub fn y(self) -> i32 {\n self[1]\n }\n\n #[inline]\n pub fn packed_eq(self, other: I32x2) -> U32x2 {\n unsafe { U32x2(simd_eq(self.0, other.0)) }\n }\n\n // Basic operations\n\n #[inline]\n pub fn max(self, other: I32x2) -> I32x2 {\n self.to_i32x4().max(other.to_i32x4()).xy()\n }\n\n #[inline]\n pub fn min(self, other: I32x2) -> I32x2 {\n self.to_i32x4().min(other.to_i32x4()).xy()\n }\n\n // Concatenations\n\n #[inline]\n pub fn concat_xy_xy(self, other: I32x2) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, other.0, [0, 1, 2, 3])) }\n }\n\n // Conversions\n\n /// Converts these packed integers to floats.\n #[inline]\n pub fn to_f32x2(self) -> F32x2 {\n unsafe { F32x2(simd_cast(self.0)) }\n }\n\n #[inline]\n pub fn to_i32x4(self) -> I32x4 {\n self.concat_xy_xy(I32x2::default())\n }\n}\n\nimpl Default for I32x2 {\n #[inline]\n fn default() -> I32x2 {\n I32x2::splat(0)\n }\n}\n\nimpl PartialEq for I32x2 {\n #[inline]\n fn eq(&self, other: &I32x2) -> bool {\n self.packed_eq(*other).all_true()\n }\n}\n\nimpl Index for I32x2 {\n type Output = i32;\n #[inline]\n fn index(&self, index: usize) -> &i32 {\n unsafe {\n assert!(index < 2);\n let ptr = &self.0 as *const int32x2_t as *const i32;\n mem::transmute::<*const i32, &i32>(ptr.offset(index as isize))\n }\n }\n}\n\nimpl IndexMut for I32x2 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut i32 {\n unsafe {\n assert!(index < 2);\n let ptr = &mut self.0 as *mut int32x2_t as *mut i32;\n mem::transmute::<*mut i32, &mut i32>(ptr.offset(index as isize))\n }\n }\n}\n\nimpl Add for I32x2 {\n type Output = I32x2;\n #[inline]\n fn add(self, other: I32x2) -> I32x2 {\n unsafe { I32x2(simd_add(self.0, other.0)) }\n }\n}\n\nimpl Sub for I32x2 {\n type Output = I32x2;\n #[inline]\n fn sub(self, other: I32x2) -> I32x2 {\n unsafe { I32x2(simd_sub(self.0, other.0)) }\n }\n}\n\nimpl Mul for I32x2 {\n type Output = I32x2;\n #[inline]\n fn mul(self, other: I32x2) -> I32x2 {\n unsafe { I32x2(simd_mul(self.0, other.0)) }\n }\n}\n\n// Four 32-bit signed integers\n\n#[derive(Clone, Copy, Debug)]\npub struct I32x4(pub int32x4_t);\n\nimpl I32x4 {\n #[inline]\n pub fn new(a: i32, b: i32, c: i32, d: i32) -> I32x4 {\n unsafe { I32x4(mem::transmute([a, b, c, d])) }\n }\n\n #[inline]\n pub fn splat(x: i32) -> I32x4 {\n I32x4::new(x, x, x, x)\n }\n\n // Basic operations\n\n #[inline]\n pub fn max(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(simd_cast(simd_maximum_number_nsz(self.to_f32x4().0, other.to_f32x4().0))) }\n }\n\n #[inline]\n pub fn min(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(simd_cast(simd_minimum_number_nsz(self.to_f32x4().0, other.to_f32x4().0))) }\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: I32x4) -> U32x4 {\n unsafe { U32x4(simd_eq(self.0, other.0)) }\n }\n\n #[inline]\n pub fn packed_le(self, other: I32x4) -> U32x4 {\n unsafe { U32x4(simd_le(self.0, other.0)) }\n }\n\n #[inline]\n pub fn packed_lt(self, other: I32x4) -> U32x4 {\n unsafe { U32x4(simd_lt(self.0, other.0)) }\n }\n\n // Concatenations\n\n #[inline]\n pub fn concat_xy_xy(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, other.0, [0, 1, 4, 5])) }\n }\n\n #[inline]\n pub fn concat_zw_zw(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, other.0, [2, 3, 6, 7])) }\n }\n\n // Swizzle conversions\n\n #[inline]\n pub fn xy(self) -> I32x2 {\n unsafe { I32x2(simd_shuffle2!(self.0, self.0, [0, 1])) }\n }\n\n #[inline]\n pub fn yx(self) -> I32x2 {\n unsafe { I32x2(simd_shuffle2!(self.0, self.0, [1, 0])) }\n }\n\n #[inline]\n pub fn xw(self) -> I32x2 {\n unsafe { I32x2(simd_shuffle2!(self.0, self.0, [0, 3])) }\n }\n\n #[inline]\n pub fn zy(self) -> I32x2 {\n unsafe { I32x2(simd_shuffle2!(self.0, self.0, [2, 1])) }\n }\n\n #[inline]\n pub fn zw(self) -> I32x2 {\n unsafe { I32x2(simd_shuffle2!(self.0, self.0, [2, 3])) }\n }\n\n // Conversions\n\n /// Converts these packed integers to floats.\n #[inline]\n pub fn to_f32x4(self) -> F32x4 {\n unsafe { F32x4(simd_cast(self.0)) }\n }\n}\n\nimpl Default for I32x4 {\n #[inline]\n fn default() -> I32x4 {\n I32x4::new(0, 0, 0, 0)\n }\n}\n\nimpl Index for I32x4 {\n type Output = i32;\n #[inline]\n fn index(&self, index: usize) -> &i32 {\n unsafe {\n assert!(index < 4);\n let ptr = &self.0 as *const int32x4_t as *const i32;\n mem::transmute::<*const i32, &i32>(ptr.offset(index as isize))\n }\n }\n}\n\nimpl IndexMut for I32x4 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut i32 {\n unsafe {\n assert!(index < 4);\n let ptr = &mut self.0 as *mut int32x4_t as *mut i32;\n mem::transmute::<*mut i32, &mut i32>(ptr.offset(index as isize))\n }\n }\n}\n\nimpl Add for I32x4 {\n type Output = I32x4;\n #[inline]\n fn add(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(simd_add(self.0, other.0)) }\n }\n}\n\nimpl Sub for I32x4 {\n type Output = I32x4;\n #[inline]\n fn sub(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(simd_sub(self.0, other.0)) }\n }\n}\n\nimpl Mul for I32x4 {\n type Output = I32x4;\n #[inline]\n fn mul(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(simd_mul(self.0, other.0)) }\n }\n}\n\nimpl PartialEq for I32x4 {\n #[inline]\n fn eq(&self, other: &I32x4) -> bool {\n self.packed_eq(*other).all_true()\n }\n}\n\nimpl BitAnd for I32x4 {\n type Output = I32x4;\n #[inline]\n fn bitand(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(simd_and(self.0, other.0)) }\n }\n}\n\nimpl BitOr for I32x4 {\n type Output = I32x4;\n #[inline]\n fn bitor(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(simd_or(self.0, other.0)) }\n }\n}\n\nimpl Shr for I32x4 {\n type Output = I32x4;\n #[inline]\n fn shr(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(simd_shr(self.0, other.0)) }\n }\n}\n\n// Two 32-bit unsigned integers\n\n#[derive(Clone, Copy)]\npub struct U32x2(pub uint32x2_t);\n\nimpl U32x2 {\n #[inline]\n pub fn new(x: u32, y: u32) -> U32x2 {\n unsafe { U32x2(mem::transmute([x, y])) }\n }\n\n #[inline]\n pub fn splat(x: u32) -> U32x2 {\n U32x2::new(x, x)\n }\n\n /// Returns true if both booleans in this vector are true.\n ///\n /// The result is *undefined* if both values in this vector are not booleans. A boolean is a\n /// value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_true(&self) -> bool {\n unsafe { aarch64::vminv_u32(self.0) == !0 }\n }\n\n /// Returns true if both booleans in this vector are false.\n ///\n /// The result is *undefined* if both values in this vector are not booleans. A boolean is a\n /// value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_false(&self) -> bool {\n unsafe { aarch64::vmaxv_u32(self.0) == 0 }\n }\n\n #[inline]\n pub fn to_i32x2(self) -> I32x2 {\n unsafe { I32x2(simd_cast(self.0)) }\n }\n}\n\nimpl Index for U32x2 {\n type Output = u32;\n #[inline]\n fn index(&self, index: usize) -> &u32 {\n unsafe {\n assert!(index < 2);\n let ptr = &self.0 as *const uint32x2_t as *const u32;\n mem::transmute::<*const u32, &u32>(ptr.offset(index as isize))\n }\n }\n}\n\nimpl Not for U32x2 {\n type Output = U32x2;\n #[inline]\n fn not(self) -> U32x2 {\n // FIXME(pcwalton): Is there a better way to do this?\n unsafe { U32x2(simd_xor(self.0, U32x2::splat(!0).0)) }\n }\n}\n\nimpl BitAnd for U32x2 {\n type Output = U32x2;\n #[inline]\n fn bitand(self, other: U32x2) -> U32x2 {\n unsafe { U32x2(simd_and(self.0, other.0)) }\n }\n}\n\nimpl BitOr for U32x2 {\n type Output = U32x2;\n #[inline]\n fn bitor(self, other: U32x2) -> U32x2 {\n unsafe { U32x2(simd_or(self.0, other.0)) }\n }\n}\n\n// Four 32-bit unsigned integers\n\n#[derive(Clone, Copy)]\npub struct U32x4(pub uint32x4_t);\n\nimpl U32x4 {\n #[inline]\n pub fn new(a: u32, b: u32, c: u32, d: u32) -> U32x4 {\n unsafe { U32x4(mem::transmute([a, b, c, d])) }\n }\n\n #[inline]\n pub fn splat(x: u32) -> U32x4 {\n U32x4::new(x, x, x, x)\n }\n\n /// Returns true if all four booleans in this vector are true.\n ///\n /// The result is *undefined* if all four values in this vector are not booleans. A boolean is\n /// a value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_true(&self) -> bool {\n unsafe { aarch64::vminvq_u32(self.0) == !0 }\n }\n\n /// Returns true if all four booleans in this vector are false.\n ///\n /// The result is *undefined* if all four values in this vector are not booleans. A boolean is\n /// a value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_false(&self) -> bool {\n unsafe { aarch64::vmaxvq_u32(self.0) == 0 }\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: U32x4) -> U32x4 {\n unsafe { U32x4(simd_eq(self.0, other.0)) }\n }\n}\n\nimpl Debug for U32x4 {\n #[inline]\n fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {\n write!(f, \"<{}, {}, {}, {}>\", self[0], self[1], self[2], self[3])\n }\n}\n\nimpl Index for U32x4 {\n type Output = u32;\n #[inline]\n fn index(&self, index: usize) -> &u32 {\n unsafe {\n assert!(index < 4);\n let ptr = &self.0 as *const uint32x4_t as *const u32;\n mem::transmute::<*const u32, &u32>(ptr.offset(index as isize))\n }\n }\n}\n\nimpl PartialEq for U32x4 {\n #[inline]\n fn eq(&self, other: &U32x4) -> bool {\n self.packed_eq(*other).all_true()\n }\n}\n\nextern \"C\" {\n #[link_name = \"llvm.fabs.v2f32\"]\n fn fabs_v2f32(a: float32x2_t) -> float32x2_t;\n #[link_name = \"llvm.floor.v2f32\"]\n fn floor_v2f32(a: float32x2_t) -> float32x2_t;\n #[link_name = \"llvm.ceil.v2f32\"]\n fn ceil_v2f32(a: float32x2_t) -> float32x2_t;\n #[link_name = \"llvm.round.v2f32\"]\n fn round_v2f32(a: float32x2_t) -> float32x2_t;\n #[link_name = \"llvm.sqrt.v2f32\"]\n fn sqrt_v2f32(a: float32x2_t) -> float32x2_t;\n\n #[link_name = \"llvm.fabs.v4f32\"]\n fn fabs_v4f32(a: float32x4_t) -> float32x4_t;\n #[link_name = \"llvm.floor.v4f32\"]\n fn floor_v4f32(a: float32x4_t) -> float32x4_t;\n #[link_name = \"llvm.ceil.v4f32\"]\n fn ceil_v4f32(a: float32x4_t) -> float32x4_t;\n #[link_name = \"llvm.round.v4f32\"]\n fn round_v4f32(a: float32x4_t) -> float32x4_t;\n #[link_name = \"llvm.sqrt.v4f32\"]\n fn sqrt_v4f32(a: float32x4_t) -> float32x4_t;\n\n #[link_name = \"llvm.aarch64.neon.frecpe.v2f32\"]\n fn vrecpe_v2f32(a: float32x2_t) -> float32x2_t;\n\n #[link_name = \"llvm.aarch64.neon.frecpe.v4f32\"]\n fn vrecpe_v4f32(a: float32x4_t) -> float32x4_t;\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/arm/swizzle_f32x4.rs", "sha256": "da7882326e637d636b6a2719285aee6e83fa16b326dbbc0a227564d54c2e2e96", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/arm/swizzle_f32x4.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::arm::{self, F32x4, Simd};\n\nmacro_rules! simd_shuffle4 {\n ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+> $idx:expr $(,)?) => {{\n struct ConstParam<$(const $imm: $ty),+>;\n impl<$(const $imm: $ty),+> ConstParam<$($imm),+> {\n const IDX: Simd = Simd($idx);\n }\n\n arm::simd_shuffle($x, $y, ConstParam::<$($imm),+>::IDX)\n }};\n ($x:expr, $y:expr, $idx:expr $(,)?) => {{\n const IDX: Simd = Simd($idx);\n arm::simd_shuffle($x, $y, IDX)\n }};\n}\n\nimpl F32x4 {\n /// Constructs a new vector from the first, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 0, 0])) }\n }\n\n /// Constructs a new vector from the second, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 0, 0])) }\n }\n\n /// Constructs a new vector from the third, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 0, 0])) }\n }\n\n /// Constructs a new vector from the fourth, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 0, 0])) }\n }\n\n /// Constructs a new vector from the first, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 0, 0])) }\n }\n\n /// Constructs a new vector from the second, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 0, 0])) }\n }\n\n /// Constructs a new vector from the third, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 0, 0])) }\n }\n\n /// Constructs a new vector from the fourth, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 0, 0])) }\n }\n\n /// Constructs a new vector from the first, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 0, 0])) }\n }\n\n /// Constructs a new vector from the second, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 0, 0])) }\n }\n\n /// Constructs a new vector from the third, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 0, 0])) }\n }\n\n /// Constructs a new vector from the fourth, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 0, 0])) }\n }\n\n /// Constructs a new vector from the first, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 0, 0])) }\n }\n\n /// Constructs a new vector from the second, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 0, 0])) }\n }\n\n /// Constructs a new vector from the third, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 0, 0])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 0, 0])) }\n }\n\n /// Constructs a new vector from the first, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 1, 0])) }\n }\n\n /// Constructs a new vector from the second, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 1, 0])) }\n }\n\n /// Constructs a new vector from the third, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 1, 0])) }\n }\n\n /// Constructs a new vector from the fourth, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 1, 0])) }\n }\n\n /// Constructs a new vector from the first, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 1, 0])) }\n }\n\n /// Constructs a new vector from the second, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 1, 0])) }\n }\n\n /// Constructs a new vector from the third, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 1, 0])) }\n }\n\n /// Constructs a new vector from the fourth, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 1, 0])) }\n }\n\n /// Constructs a new vector from the first, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 1, 0])) }\n }\n\n /// Constructs a new vector from the second, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 1, 0])) }\n }\n\n /// Constructs a new vector from the third, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 1, 0])) }\n }\n\n /// Constructs a new vector from the fourth, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 1, 0])) }\n }\n\n /// Constructs a new vector from the first, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 1, 0])) }\n }\n\n /// Constructs a new vector from the second, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 1, 0])) }\n }\n\n /// Constructs a new vector from the third, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 1, 0])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 1, 0])) }\n }\n\n /// Constructs a new vector from the first, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 2, 0])) }\n }\n\n /// Constructs a new vector from the second, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 2, 0])) }\n }\n\n /// Constructs a new vector from the third, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 2, 0])) }\n }\n\n /// Constructs a new vector from the fourth, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 2, 0])) }\n }\n\n /// Constructs a new vector from the first, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 2, 0])) }\n }\n\n /// Constructs a new vector from the second, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 2, 0])) }\n }\n\n /// Constructs a new vector from the third, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 2, 0])) }\n }\n\n /// Constructs a new vector from the fourth, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 2, 0])) }\n }\n\n /// Constructs a new vector from the first, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 2, 0])) }\n }\n\n /// Constructs a new vector from the second, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 2, 0])) }\n }\n\n /// Constructs a new vector from the third, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 2, 0])) }\n }\n\n /// Constructs a new vector from the fourth, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 2, 0])) }\n }\n\n /// Constructs a new vector from the first, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 2, 0])) }\n }\n\n /// Constructs a new vector from the second, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 2, 0])) }\n }\n\n /// Constructs a new vector from the third, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 2, 0])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 2, 0])) }\n }\n\n /// Constructs a new vector from the first, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 3, 0])) }\n }\n\n /// Constructs a new vector from the second, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 3, 0])) }\n }\n\n /// Constructs a new vector from the third, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 3, 0])) }\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 3, 0])) }\n }\n\n /// Constructs a new vector from the first, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 3, 0])) }\n }\n\n /// Constructs a new vector from the second, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 3, 0])) }\n }\n\n /// Constructs a new vector from the third, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 3, 0])) }\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 3, 0])) }\n }\n\n /// Constructs a new vector from the first, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 3, 0])) }\n }\n\n /// Constructs a new vector from the second, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 3, 0])) }\n }\n\n /// Constructs a new vector from the third, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 3, 0])) }\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 3, 0])) }\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 3, 0])) }\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 3, 0])) }\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 3, 0])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwx(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 3, 0])) }\n }\n\n /// Constructs a new vector from the first, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 0, 1])) }\n }\n\n /// Constructs a new vector from the second, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 0, 1])) }\n }\n\n /// Constructs a new vector from the third, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 0, 1])) }\n }\n\n /// Constructs a new vector from the fourth, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 0, 1])) }\n }\n\n /// Constructs a new vector from the first, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 0, 1])) }\n }\n\n /// Constructs a new vector from the second, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 0, 1])) }\n }\n\n /// Constructs a new vector from the third, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 0, 1])) }\n }\n\n /// Constructs a new vector from the fourth, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 0, 1])) }\n }\n\n /// Constructs a new vector from the first, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 0, 1])) }\n }\n\n /// Constructs a new vector from the second, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 0, 1])) }\n }\n\n /// Constructs a new vector from the third, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 0, 1])) }\n }\n\n /// Constructs a new vector from the fourth, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 0, 1])) }\n }\n\n /// Constructs a new vector from the first, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 0, 1])) }\n }\n\n /// Constructs a new vector from the second, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 0, 1])) }\n }\n\n /// Constructs a new vector from the third, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 0, 1])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 0, 1])) }\n }\n\n /// Constructs a new vector from the first, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 1, 1])) }\n }\n\n /// Constructs a new vector from the second, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 1, 1])) }\n }\n\n /// Constructs a new vector from the third, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 1, 1])) }\n }\n\n /// Constructs a new vector from the fourth, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 1, 1])) }\n }\n\n /// Constructs a new vector from the first, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 1, 1])) }\n }\n\n /// Constructs a new vector from the second, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 1, 1])) }\n }\n\n /// Constructs a new vector from the third, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 1, 1])) }\n }\n\n /// Constructs a new vector from the fourth, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 1, 1])) }\n }\n\n /// Constructs a new vector from the first, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 1, 1])) }\n }\n\n /// Constructs a new vector from the second, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 1, 1])) }\n }\n\n /// Constructs a new vector from the third, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 1, 1])) }\n }\n\n /// Constructs a new vector from the fourth, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 1, 1])) }\n }\n\n /// Constructs a new vector from the first, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 1, 1])) }\n }\n\n /// Constructs a new vector from the second, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 1, 1])) }\n }\n\n /// Constructs a new vector from the third, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 1, 1])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 1, 1])) }\n }\n\n /// Constructs a new vector from the first, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 2, 1])) }\n }\n\n /// Constructs a new vector from the second, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 2, 1])) }\n }\n\n /// Constructs a new vector from the third, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 2, 1])) }\n }\n\n /// Constructs a new vector from the fourth, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 2, 1])) }\n }\n\n /// Constructs a new vector from the first, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 2, 1])) }\n }\n\n /// Constructs a new vector from the second, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 2, 1])) }\n }\n\n /// Constructs a new vector from the third, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 2, 1])) }\n }\n\n /// Constructs a new vector from the fourth, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 2, 1])) }\n }\n\n /// Constructs a new vector from the first, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 2, 1])) }\n }\n\n /// Constructs a new vector from the second, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 2, 1])) }\n }\n\n /// Constructs a new vector from the third, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 2, 1])) }\n }\n\n /// Constructs a new vector from the fourth, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 2, 1])) }\n }\n\n /// Constructs a new vector from the first, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 2, 1])) }\n }\n\n /// Constructs a new vector from the second, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 2, 1])) }\n }\n\n /// Constructs a new vector from the third, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 2, 1])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 2, 1])) }\n }\n\n /// Constructs a new vector from the first, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 3, 1])) }\n }\n\n /// Constructs a new vector from the second, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 3, 1])) }\n }\n\n /// Constructs a new vector from the third, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 3, 1])) }\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 3, 1])) }\n }\n\n /// Constructs a new vector from the first, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 3, 1])) }\n }\n\n /// Constructs a new vector from the second, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 3, 1])) }\n }\n\n /// Constructs a new vector from the third, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 3, 1])) }\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 3, 1])) }\n }\n\n /// Constructs a new vector from the first, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 3, 1])) }\n }\n\n /// Constructs a new vector from the second, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 3, 1])) }\n }\n\n /// Constructs a new vector from the third, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 3, 1])) }\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 3, 1])) }\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 3, 1])) }\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 3, 1])) }\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 3, 1])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwy(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 3, 1])) }\n }\n\n /// Constructs a new vector from the first, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 0, 2])) }\n }\n\n /// Constructs a new vector from the second, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 0, 2])) }\n }\n\n /// Constructs a new vector from the third, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 0, 2])) }\n }\n\n /// Constructs a new vector from the fourth, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 0, 2])) }\n }\n\n /// Constructs a new vector from the first, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 0, 2])) }\n }\n\n /// Constructs a new vector from the second, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 0, 2])) }\n }\n\n /// Constructs a new vector from the third, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 0, 2])) }\n }\n\n /// Constructs a new vector from the fourth, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 0, 2])) }\n }\n\n /// Constructs a new vector from the first, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 0, 2])) }\n }\n\n /// Constructs a new vector from the second, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 0, 2])) }\n }\n\n /// Constructs a new vector from the third, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 0, 2])) }\n }\n\n /// Constructs a new vector from the fourth, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 0, 2])) }\n }\n\n /// Constructs a new vector from the first, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 0, 2])) }\n }\n\n /// Constructs a new vector from the second, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 0, 2])) }\n }\n\n /// Constructs a new vector from the third, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 0, 2])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 0, 2])) }\n }\n\n /// Constructs a new vector from the first, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 1, 2])) }\n }\n\n /// Constructs a new vector from the second, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 1, 2])) }\n }\n\n /// Constructs a new vector from the third, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 1, 2])) }\n }\n\n /// Constructs a new vector from the fourth, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 1, 2])) }\n }\n\n /// Constructs a new vector from the first, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 1, 2])) }\n }\n\n /// Constructs a new vector from the second, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 1, 2])) }\n }\n\n /// Constructs a new vector from the third, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 1, 2])) }\n }\n\n /// Constructs a new vector from the fourth, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 1, 2])) }\n }\n\n /// Constructs a new vector from the first, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 1, 2])) }\n }\n\n /// Constructs a new vector from the second, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 1, 2])) }\n }\n\n /// Constructs a new vector from the third, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 1, 2])) }\n }\n\n /// Constructs a new vector from the fourth, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 1, 2])) }\n }\n\n /// Constructs a new vector from the first, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 1, 2])) }\n }\n\n /// Constructs a new vector from the second, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 1, 2])) }\n }\n\n /// Constructs a new vector from the third, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 1, 2])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 1, 2])) }\n }\n\n /// Constructs a new vector from the first, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 2, 2])) }\n }\n\n /// Constructs a new vector from the second, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 2, 2])) }\n }\n\n /// Constructs a new vector from the third, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 2, 2])) }\n }\n\n /// Constructs a new vector from the fourth, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 2, 2])) }\n }\n\n /// Constructs a new vector from the first, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 2, 2])) }\n }\n\n /// Constructs a new vector from the second, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 2, 2])) }\n }\n\n /// Constructs a new vector from the third, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 2, 2])) }\n }\n\n /// Constructs a new vector from the fourth, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 2, 2])) }\n }\n\n /// Constructs a new vector from the first, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 2, 2])) }\n }\n\n /// Constructs a new vector from the second, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 2, 2])) }\n }\n\n /// Constructs a new vector from the third, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 2, 2])) }\n }\n\n /// Constructs a new vector from the fourth, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 2, 2])) }\n }\n\n /// Constructs a new vector from the first, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 2, 2])) }\n }\n\n /// Constructs a new vector from the second, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 2, 2])) }\n }\n\n /// Constructs a new vector from the third, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 2, 2])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 2, 2])) }\n }\n\n /// Constructs a new vector from the first, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 3, 2])) }\n }\n\n /// Constructs a new vector from the second, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 3, 2])) }\n }\n\n /// Constructs a new vector from the third, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 3, 2])) }\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 3, 2])) }\n }\n\n /// Constructs a new vector from the first, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 3, 2])) }\n }\n\n /// Constructs a new vector from the second, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 3, 2])) }\n }\n\n /// Constructs a new vector from the third, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 3, 2])) }\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 3, 2])) }\n }\n\n /// Constructs a new vector from the first, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 3, 2])) }\n }\n\n /// Constructs a new vector from the second, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 3, 2])) }\n }\n\n /// Constructs a new vector from the third, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 3, 2])) }\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 3, 2])) }\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 3, 2])) }\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 3, 2])) }\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 3, 2])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwz(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 3, 2])) }\n }\n\n /// Constructs a new vector from the first, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 0, 3])) }\n }\n\n /// Constructs a new vector from the second, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 0, 3])) }\n }\n\n /// Constructs a new vector from the third, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 0, 3])) }\n }\n\n /// Constructs a new vector from the fourth, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 0, 3])) }\n }\n\n /// Constructs a new vector from the first, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 0, 3])) }\n }\n\n /// Constructs a new vector from the second, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 0, 3])) }\n }\n\n /// Constructs a new vector from the third, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 0, 3])) }\n }\n\n /// Constructs a new vector from the fourth, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 0, 3])) }\n }\n\n /// Constructs a new vector from the first, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 0, 3])) }\n }\n\n /// Constructs a new vector from the second, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 0, 3])) }\n }\n\n /// Constructs a new vector from the third, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 0, 3])) }\n }\n\n /// Constructs a new vector from the fourth, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 0, 3])) }\n }\n\n /// Constructs a new vector from the first, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 0, 3])) }\n }\n\n /// Constructs a new vector from the second, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 0, 3])) }\n }\n\n /// Constructs a new vector from the third, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 0, 3])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 0, 3])) }\n }\n\n /// Constructs a new vector from the first, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 1, 3])) }\n }\n\n /// Constructs a new vector from the second, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 1, 3])) }\n }\n\n /// Constructs a new vector from the third, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 1, 3])) }\n }\n\n /// Constructs a new vector from the fourth, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 1, 3])) }\n }\n\n /// Constructs a new vector from the first, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 1, 3])) }\n }\n\n /// Constructs a new vector from the second, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 1, 3])) }\n }\n\n /// Constructs a new vector from the third, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 1, 3])) }\n }\n\n /// Constructs a new vector from the fourth, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 1, 3])) }\n }\n\n /// Constructs a new vector from the first, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 1, 3])) }\n }\n\n /// Constructs a new vector from the second, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 1, 3])) }\n }\n\n /// Constructs a new vector from the third, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 1, 3])) }\n }\n\n /// Constructs a new vector from the fourth, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 1, 3])) }\n }\n\n /// Constructs a new vector from the first, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 1, 3])) }\n }\n\n /// Constructs a new vector from the second, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 1, 3])) }\n }\n\n /// Constructs a new vector from the third, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 1, 3])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 1, 3])) }\n }\n\n /// Constructs a new vector from the first, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 2, 3])) }\n }\n\n /// Constructs a new vector from the second, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 2, 3])) }\n }\n\n /// Constructs a new vector from the third, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 2, 3])) }\n }\n\n /// Constructs a new vector from the fourth, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 2, 3])) }\n }\n\n /// Constructs a new vector from the first, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 2, 3])) }\n }\n\n /// Constructs a new vector from the second, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 2, 3])) }\n }\n\n /// Constructs a new vector from the third, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 2, 3])) }\n }\n\n /// Constructs a new vector from the fourth, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 2, 3])) }\n }\n\n /// Constructs a new vector from the first, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 2, 3])) }\n }\n\n /// Constructs a new vector from the second, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 2, 3])) }\n }\n\n /// Constructs a new vector from the third, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 2, 3])) }\n }\n\n /// Constructs a new vector from the fourth, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 2, 3])) }\n }\n\n /// Constructs a new vector from the first, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 2, 3])) }\n }\n\n /// Constructs a new vector from the second, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 2, 3])) }\n }\n\n /// Constructs a new vector from the third, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 2, 3])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzw(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 2, 3])) }\n }\n\n /// Constructs a new vector from the first, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 0, 3, 3])) }\n }\n\n /// Constructs a new vector from the second, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 0, 3, 3])) }\n }\n\n /// Constructs a new vector from the third, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 0, 3, 3])) }\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 0, 3, 3])) }\n }\n\n /// Constructs a new vector from the first, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 1, 3, 3])) }\n }\n\n /// Constructs a new vector from the second, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 1, 3, 3])) }\n }\n\n /// Constructs a new vector from the third, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 1, 3, 3])) }\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 1, 3, 3])) }\n }\n\n /// Constructs a new vector from the first, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 2, 3, 3])) }\n }\n\n /// Constructs a new vector from the second, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 2, 3, 3])) }\n }\n\n /// Constructs a new vector from the third, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 2, 3, 3])) }\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 2, 3, 3])) }\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [0, 3, 3, 3])) }\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [1, 3, 3, 3])) }\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [2, 3, 3, 3])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwww(self) -> F32x4 {\n unsafe { F32x4(simd_shuffle4!(self.0, self.0, [3, 3, 3, 3])) }\n }\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/arm/swizzle_i32x4.rs", "sha256": "b51fa2f04b97e3d81c40322e69883c757bdee0d3ef97bce25eb1ac8b181341d7", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/arm/swizzle_i32x4.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::arm::{self, I32x4, Simd};\n\nmacro_rules! simd_shuffle4 {\n ($x:expr, $y:expr, <$(const $imm:ident : $ty:ty),+> $idx:expr $(,)?) => {{\n struct ConstParam<$(const $imm: $ty),+>;\n impl<$(const $imm: $ty),+> ConstParam<$($imm),+> {\n const IDX: Simd = Simd($idx);\n }\n\n arm::simd_shuffle($x, $y, ConstParam::<$($imm),+>::IDX)\n }};\n ($x:expr, $y:expr, $idx:expr $(,)?) => {{\n const IDX: Simd = Simd($idx);\n arm::simd_shuffle($x, $y, IDX)\n }};\n}\n\nimpl I32x4 {\n /// Constructs a new vector from the first, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 0, 0])) }\n }\n\n /// Constructs a new vector from the second, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 0, 0])) }\n }\n\n /// Constructs a new vector from the third, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 0, 0])) }\n }\n\n /// Constructs a new vector from the fourth, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 0, 0])) }\n }\n\n /// Constructs a new vector from the first, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 0, 0])) }\n }\n\n /// Constructs a new vector from the second, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 0, 0])) }\n }\n\n /// Constructs a new vector from the third, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 0, 0])) }\n }\n\n /// Constructs a new vector from the fourth, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 0, 0])) }\n }\n\n /// Constructs a new vector from the first, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 0, 0])) }\n }\n\n /// Constructs a new vector from the second, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 0, 0])) }\n }\n\n /// Constructs a new vector from the third, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 0, 0])) }\n }\n\n /// Constructs a new vector from the fourth, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 0, 0])) }\n }\n\n /// Constructs a new vector from the first, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 0, 0])) }\n }\n\n /// Constructs a new vector from the second, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 0, 0])) }\n }\n\n /// Constructs a new vector from the third, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 0, 0])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 0, 0])) }\n }\n\n /// Constructs a new vector from the first, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 1, 0])) }\n }\n\n /// Constructs a new vector from the second, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 1, 0])) }\n }\n\n /// Constructs a new vector from the third, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 1, 0])) }\n }\n\n /// Constructs a new vector from the fourth, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 1, 0])) }\n }\n\n /// Constructs a new vector from the first, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 1, 0])) }\n }\n\n /// Constructs a new vector from the second, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 1, 0])) }\n }\n\n /// Constructs a new vector from the third, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 1, 0])) }\n }\n\n /// Constructs a new vector from the fourth, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 1, 0])) }\n }\n\n /// Constructs a new vector from the first, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 1, 0])) }\n }\n\n /// Constructs a new vector from the second, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 1, 0])) }\n }\n\n /// Constructs a new vector from the third, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 1, 0])) }\n }\n\n /// Constructs a new vector from the fourth, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 1, 0])) }\n }\n\n /// Constructs a new vector from the first, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 1, 0])) }\n }\n\n /// Constructs a new vector from the second, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 1, 0])) }\n }\n\n /// Constructs a new vector from the third, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 1, 0])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 1, 0])) }\n }\n\n /// Constructs a new vector from the first, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 2, 0])) }\n }\n\n /// Constructs a new vector from the second, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 2, 0])) }\n }\n\n /// Constructs a new vector from the third, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 2, 0])) }\n }\n\n /// Constructs a new vector from the fourth, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 2, 0])) }\n }\n\n /// Constructs a new vector from the first, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 2, 0])) }\n }\n\n /// Constructs a new vector from the second, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 2, 0])) }\n }\n\n /// Constructs a new vector from the third, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 2, 0])) }\n }\n\n /// Constructs a new vector from the fourth, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 2, 0])) }\n }\n\n /// Constructs a new vector from the first, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 2, 0])) }\n }\n\n /// Constructs a new vector from the second, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 2, 0])) }\n }\n\n /// Constructs a new vector from the third, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 2, 0])) }\n }\n\n /// Constructs a new vector from the fourth, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 2, 0])) }\n }\n\n /// Constructs a new vector from the first, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 2, 0])) }\n }\n\n /// Constructs a new vector from the second, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 2, 0])) }\n }\n\n /// Constructs a new vector from the third, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 2, 0])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 2, 0])) }\n }\n\n /// Constructs a new vector from the first, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 3, 0])) }\n }\n\n /// Constructs a new vector from the second, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 3, 0])) }\n }\n\n /// Constructs a new vector from the third, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 3, 0])) }\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 3, 0])) }\n }\n\n /// Constructs a new vector from the first, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 3, 0])) }\n }\n\n /// Constructs a new vector from the second, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 3, 0])) }\n }\n\n /// Constructs a new vector from the third, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 3, 0])) }\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 3, 0])) }\n }\n\n /// Constructs a new vector from the first, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 3, 0])) }\n }\n\n /// Constructs a new vector from the second, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 3, 0])) }\n }\n\n /// Constructs a new vector from the third, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 3, 0])) }\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 3, 0])) }\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 3, 0])) }\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 3, 0])) }\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 3, 0])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwx(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 3, 0])) }\n }\n\n /// Constructs a new vector from the first, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 0, 1])) }\n }\n\n /// Constructs a new vector from the second, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 0, 1])) }\n }\n\n /// Constructs a new vector from the third, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 0, 1])) }\n }\n\n /// Constructs a new vector from the fourth, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 0, 1])) }\n }\n\n /// Constructs a new vector from the first, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 0, 1])) }\n }\n\n /// Constructs a new vector from the second, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 0, 1])) }\n }\n\n /// Constructs a new vector from the third, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 0, 1])) }\n }\n\n /// Constructs a new vector from the fourth, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 0, 1])) }\n }\n\n /// Constructs a new vector from the first, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 0, 1])) }\n }\n\n /// Constructs a new vector from the second, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 0, 1])) }\n }\n\n /// Constructs a new vector from the third, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 0, 1])) }\n }\n\n /// Constructs a new vector from the fourth, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 0, 1])) }\n }\n\n /// Constructs a new vector from the first, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 0, 1])) }\n }\n\n /// Constructs a new vector from the second, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 0, 1])) }\n }\n\n /// Constructs a new vector from the third, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 0, 1])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 0, 1])) }\n }\n\n /// Constructs a new vector from the first, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 1, 1])) }\n }\n\n /// Constructs a new vector from the second, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 1, 1])) }\n }\n\n /// Constructs a new vector from the third, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 1, 1])) }\n }\n\n /// Constructs a new vector from the fourth, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 1, 1])) }\n }\n\n /// Constructs a new vector from the first, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 1, 1])) }\n }\n\n /// Constructs a new vector from the second, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 1, 1])) }\n }\n\n /// Constructs a new vector from the third, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 1, 1])) }\n }\n\n /// Constructs a new vector from the fourth, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 1, 1])) }\n }\n\n /// Constructs a new vector from the first, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 1, 1])) }\n }\n\n /// Constructs a new vector from the second, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 1, 1])) }\n }\n\n /// Constructs a new vector from the third, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 1, 1])) }\n }\n\n /// Constructs a new vector from the fourth, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 1, 1])) }\n }\n\n /// Constructs a new vector from the first, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 1, 1])) }\n }\n\n /// Constructs a new vector from the second, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 1, 1])) }\n }\n\n /// Constructs a new vector from the third, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 1, 1])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 1, 1])) }\n }\n\n /// Constructs a new vector from the first, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 2, 1])) }\n }\n\n /// Constructs a new vector from the second, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 2, 1])) }\n }\n\n /// Constructs a new vector from the third, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 2, 1])) }\n }\n\n /// Constructs a new vector from the fourth, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 2, 1])) }\n }\n\n /// Constructs a new vector from the first, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 2, 1])) }\n }\n\n /// Constructs a new vector from the second, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 2, 1])) }\n }\n\n /// Constructs a new vector from the third, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 2, 1])) }\n }\n\n /// Constructs a new vector from the fourth, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 2, 1])) }\n }\n\n /// Constructs a new vector from the first, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 2, 1])) }\n }\n\n /// Constructs a new vector from the second, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 2, 1])) }\n }\n\n /// Constructs a new vector from the third, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 2, 1])) }\n }\n\n /// Constructs a new vector from the fourth, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 2, 1])) }\n }\n\n /// Constructs a new vector from the first, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 2, 1])) }\n }\n\n /// Constructs a new vector from the second, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 2, 1])) }\n }\n\n /// Constructs a new vector from the third, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 2, 1])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 2, 1])) }\n }\n\n /// Constructs a new vector from the first, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 3, 1])) }\n }\n\n /// Constructs a new vector from the second, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 3, 1])) }\n }\n\n /// Constructs a new vector from the third, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 3, 1])) }\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 3, 1])) }\n }\n\n /// Constructs a new vector from the first, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 3, 1])) }\n }\n\n /// Constructs a new vector from the second, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 3, 1])) }\n }\n\n /// Constructs a new vector from the third, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 3, 1])) }\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 3, 1])) }\n }\n\n /// Constructs a new vector from the first, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 3, 1])) }\n }\n\n /// Constructs a new vector from the second, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 3, 1])) }\n }\n\n /// Constructs a new vector from the third, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 3, 1])) }\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 3, 1])) }\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 3, 1])) }\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 3, 1])) }\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 3, 1])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwy(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 3, 1])) }\n }\n\n /// Constructs a new vector from the first, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 0, 2])) }\n }\n\n /// Constructs a new vector from the second, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 0, 2])) }\n }\n\n /// Constructs a new vector from the third, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 0, 2])) }\n }\n\n /// Constructs a new vector from the fourth, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 0, 2])) }\n }\n\n /// Constructs a new vector from the first, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 0, 2])) }\n }\n\n /// Constructs a new vector from the second, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 0, 2])) }\n }\n\n /// Constructs a new vector from the third, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 0, 2])) }\n }\n\n /// Constructs a new vector from the fourth, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 0, 2])) }\n }\n\n /// Constructs a new vector from the first, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 0, 2])) }\n }\n\n /// Constructs a new vector from the second, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 0, 2])) }\n }\n\n /// Constructs a new vector from the third, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 0, 2])) }\n }\n\n /// Constructs a new vector from the fourth, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 0, 2])) }\n }\n\n /// Constructs a new vector from the first, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 0, 2])) }\n }\n\n /// Constructs a new vector from the second, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 0, 2])) }\n }\n\n /// Constructs a new vector from the third, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 0, 2])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 0, 2])) }\n }\n\n /// Constructs a new vector from the first, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 1, 2])) }\n }\n\n /// Constructs a new vector from the second, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 1, 2])) }\n }\n\n /// Constructs a new vector from the third, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 1, 2])) }\n }\n\n /// Constructs a new vector from the fourth, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 1, 2])) }\n }\n\n /// Constructs a new vector from the first, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 1, 2])) }\n }\n\n /// Constructs a new vector from the second, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 1, 2])) }\n }\n\n /// Constructs a new vector from the third, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 1, 2])) }\n }\n\n /// Constructs a new vector from the fourth, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 1, 2])) }\n }\n\n /// Constructs a new vector from the first, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 1, 2])) }\n }\n\n /// Constructs a new vector from the second, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 1, 2])) }\n }\n\n /// Constructs a new vector from the third, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 1, 2])) }\n }\n\n /// Constructs a new vector from the fourth, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 1, 2])) }\n }\n\n /// Constructs a new vector from the first, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 1, 2])) }\n }\n\n /// Constructs a new vector from the second, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 1, 2])) }\n }\n\n /// Constructs a new vector from the third, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 1, 2])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 1, 2])) }\n }\n\n /// Constructs a new vector from the first, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 2, 2])) }\n }\n\n /// Constructs a new vector from the second, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 2, 2])) }\n }\n\n /// Constructs a new vector from the third, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 2, 2])) }\n }\n\n /// Constructs a new vector from the fourth, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 2, 2])) }\n }\n\n /// Constructs a new vector from the first, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 2, 2])) }\n }\n\n /// Constructs a new vector from the second, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 2, 2])) }\n }\n\n /// Constructs a new vector from the third, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 2, 2])) }\n }\n\n /// Constructs a new vector from the fourth, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 2, 2])) }\n }\n\n /// Constructs a new vector from the first, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 2, 2])) }\n }\n\n /// Constructs a new vector from the second, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 2, 2])) }\n }\n\n /// Constructs a new vector from the third, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 2, 2])) }\n }\n\n /// Constructs a new vector from the fourth, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 2, 2])) }\n }\n\n /// Constructs a new vector from the first, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 2, 2])) }\n }\n\n /// Constructs a new vector from the second, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 2, 2])) }\n }\n\n /// Constructs a new vector from the third, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 2, 2])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 2, 2])) }\n }\n\n /// Constructs a new vector from the first, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 3, 2])) }\n }\n\n /// Constructs a new vector from the second, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 3, 2])) }\n }\n\n /// Constructs a new vector from the third, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 3, 2])) }\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 3, 2])) }\n }\n\n /// Constructs a new vector from the first, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 3, 2])) }\n }\n\n /// Constructs a new vector from the second, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 3, 2])) }\n }\n\n /// Constructs a new vector from the third, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 3, 2])) }\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 3, 2])) }\n }\n\n /// Constructs a new vector from the first, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 3, 2])) }\n }\n\n /// Constructs a new vector from the second, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 3, 2])) }\n }\n\n /// Constructs a new vector from the third, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 3, 2])) }\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 3, 2])) }\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 3, 2])) }\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 3, 2])) }\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 3, 2])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwz(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 3, 2])) }\n }\n\n /// Constructs a new vector from the first, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 0, 3])) }\n }\n\n /// Constructs a new vector from the second, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 0, 3])) }\n }\n\n /// Constructs a new vector from the third, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 0, 3])) }\n }\n\n /// Constructs a new vector from the fourth, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 0, 3])) }\n }\n\n /// Constructs a new vector from the first, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 0, 3])) }\n }\n\n /// Constructs a new vector from the second, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 0, 3])) }\n }\n\n /// Constructs a new vector from the third, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 0, 3])) }\n }\n\n /// Constructs a new vector from the fourth, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 0, 3])) }\n }\n\n /// Constructs a new vector from the first, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 0, 3])) }\n }\n\n /// Constructs a new vector from the second, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 0, 3])) }\n }\n\n /// Constructs a new vector from the third, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 0, 3])) }\n }\n\n /// Constructs a new vector from the fourth, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 0, 3])) }\n }\n\n /// Constructs a new vector from the first, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 0, 3])) }\n }\n\n /// Constructs a new vector from the second, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 0, 3])) }\n }\n\n /// Constructs a new vector from the third, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 0, 3])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 0, 3])) }\n }\n\n /// Constructs a new vector from the first, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 1, 3])) }\n }\n\n /// Constructs a new vector from the second, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 1, 3])) }\n }\n\n /// Constructs a new vector from the third, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 1, 3])) }\n }\n\n /// Constructs a new vector from the fourth, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 1, 3])) }\n }\n\n /// Constructs a new vector from the first, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 1, 3])) }\n }\n\n /// Constructs a new vector from the second, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 1, 3])) }\n }\n\n /// Constructs a new vector from the third, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 1, 3])) }\n }\n\n /// Constructs a new vector from the fourth, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 1, 3])) }\n }\n\n /// Constructs a new vector from the first, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 1, 3])) }\n }\n\n /// Constructs a new vector from the second, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 1, 3])) }\n }\n\n /// Constructs a new vector from the third, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 1, 3])) }\n }\n\n /// Constructs a new vector from the fourth, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 1, 3])) }\n }\n\n /// Constructs a new vector from the first, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 1, 3])) }\n }\n\n /// Constructs a new vector from the second, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 1, 3])) }\n }\n\n /// Constructs a new vector from the third, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 1, 3])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 1, 3])) }\n }\n\n /// Constructs a new vector from the first, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 2, 3])) }\n }\n\n /// Constructs a new vector from the second, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 2, 3])) }\n }\n\n /// Constructs a new vector from the third, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 2, 3])) }\n }\n\n /// Constructs a new vector from the fourth, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 2, 3])) }\n }\n\n /// Constructs a new vector from the first, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 2, 3])) }\n }\n\n /// Constructs a new vector from the second, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 2, 3])) }\n }\n\n /// Constructs a new vector from the third, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 2, 3])) }\n }\n\n /// Constructs a new vector from the fourth, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 2, 3])) }\n }\n\n /// Constructs a new vector from the first, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 2, 3])) }\n }\n\n /// Constructs a new vector from the second, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 2, 3])) }\n }\n\n /// Constructs a new vector from the third, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 2, 3])) }\n }\n\n /// Constructs a new vector from the fourth, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 2, 3])) }\n }\n\n /// Constructs a new vector from the first, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 2, 3])) }\n }\n\n /// Constructs a new vector from the second, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 2, 3])) }\n }\n\n /// Constructs a new vector from the third, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 2, 3])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzw(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 2, 3])) }\n }\n\n /// Constructs a new vector from the first, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 0, 3, 3])) }\n }\n\n /// Constructs a new vector from the second, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 0, 3, 3])) }\n }\n\n /// Constructs a new vector from the third, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 0, 3, 3])) }\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 0, 3, 3])) }\n }\n\n /// Constructs a new vector from the first, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 1, 3, 3])) }\n }\n\n /// Constructs a new vector from the second, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 1, 3, 3])) }\n }\n\n /// Constructs a new vector from the third, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 1, 3, 3])) }\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 1, 3, 3])) }\n }\n\n /// Constructs a new vector from the first, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 2, 3, 3])) }\n }\n\n /// Constructs a new vector from the second, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 2, 3, 3])) }\n }\n\n /// Constructs a new vector from the third, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 2, 3, 3])) }\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 2, 3, 3])) }\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [0, 3, 3, 3])) }\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [1, 3, 3, 3])) }\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [2, 3, 3, 3])) }\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwww(self) -> I32x4 {\n unsafe { I32x4(simd_shuffle4!(self.0, self.0, [3, 3, 3, 3])) }\n }\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/extras.rs", "sha256": "013cf183ca79dec2b5ac5f993843a8c123a10fdfdf3d9fa8024ef47ee60e57af", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/extras.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::default::{F32x2, F32x4, I32x2, I32x4};\nuse std::ops::{AddAssign, MulAssign, Neg, SubAssign};\n\n// Two 32-bit floats\n\nimpl F32x2 {\n // Constructors\n\n #[inline]\n pub fn from_slice(slice: &[f32]) -> F32x2 {\n F32x2::new(slice[0], slice[1])\n }\n\n // Accessors\n\n #[inline]\n pub fn x(self) -> f32 {\n self[0]\n }\n\n #[inline]\n pub fn y(self) -> f32 {\n self[1]\n }\n\n // Mutators\n\n #[inline]\n pub fn set_x(&mut self, x: f32) {\n self[0] = x\n }\n\n #[inline]\n pub fn set_y(&mut self, y: f32) {\n self[1] = y\n }\n\n // Comparisons\n\n #[inline]\n pub fn approx_eq(self, other: F32x2, epsilon: f32) -> bool {\n (self - other).abs().packed_gt(F32x2::splat(epsilon)).all_false()\n }\n}\n\nimpl AddAssign for F32x2 {\n #[inline]\n fn add_assign(&mut self, other: F32x2) {\n *self = *self + other\n }\n}\n\nimpl SubAssign for F32x2 {\n #[inline]\n fn sub_assign(&mut self, other: F32x2) {\n *self = *self - other\n }\n}\n\nimpl MulAssign for F32x2 {\n #[inline]\n fn mul_assign(&mut self, other: F32x2) {\n *self = *self * other\n }\n}\n\nimpl Neg for F32x2 {\n type Output = F32x2;\n #[inline]\n fn neg(self) -> F32x2 {\n F32x2::default() - self\n }\n}\n\n// Four 32-bit floats\n\nimpl F32x4 {\n // Constructors\n\n #[inline]\n pub fn from_slice(slice: &[f32]) -> F32x4 {\n F32x4::new(slice[0], slice[1], slice[2], slice[3])\n }\n\n // Accessors\n\n #[inline]\n pub fn x(self) -> f32 {\n self[0]\n }\n\n #[inline]\n pub fn y(self) -> f32 {\n self[1]\n }\n\n #[inline]\n pub fn z(self) -> f32 {\n self[2]\n }\n\n #[inline]\n pub fn w(self) -> f32 {\n self[3]\n }\n\n // Mutators\n\n #[inline]\n pub fn set_x(&mut self, x: f32) {\n self[0] = x\n }\n\n #[inline]\n pub fn set_y(&mut self, y: f32) {\n self[1] = y\n }\n\n #[inline]\n pub fn set_z(&mut self, z: f32) {\n self[2] = z\n }\n\n #[inline]\n pub fn set_w(&mut self, w: f32) {\n self[3] = w\n }\n\n // Comparisons\n\n #[inline]\n pub fn approx_eq(self, other: F32x4, epsilon: f32) -> bool {\n (self - other).abs().packed_gt(F32x4::splat(epsilon)).all_false()\n }\n}\n\nimpl AddAssign for F32x4 {\n #[inline]\n fn add_assign(&mut self, other: F32x4) {\n *self = *self + other\n }\n}\n\nimpl SubAssign for F32x4 {\n #[inline]\n fn sub_assign(&mut self, other: F32x4) {\n *self = *self - other\n }\n}\n\nimpl MulAssign for F32x4 {\n #[inline]\n fn mul_assign(&mut self, other: F32x4) {\n *self = *self * other\n }\n}\n\nimpl Neg for F32x4 {\n type Output = F32x4;\n #[inline]\n fn neg(self) -> F32x4 {\n F32x4::default() - self\n }\n}\n\n// Two 32-bit integers\n\nimpl AddAssign for I32x2 {\n #[inline]\n fn add_assign(&mut self, other: I32x2) {\n *self = *self + other\n }\n}\n\nimpl SubAssign for I32x2 {\n #[inline]\n fn sub_assign(&mut self, other: I32x2) {\n *self = *self - other\n }\n}\n\nimpl MulAssign for I32x2 {\n #[inline]\n fn mul_assign(&mut self, other: I32x2) {\n *self = *self * other\n }\n}\n\nimpl Neg for I32x2 {\n type Output = I32x2;\n #[inline]\n fn neg(self) -> I32x2 {\n I32x2::default() - self\n }\n}\n\n// Four 32-bit integers\n\nimpl I32x4 {\n // Accessors\n\n #[inline]\n pub fn x(self) -> i32 {\n self[0]\n }\n\n #[inline]\n pub fn y(self) -> i32 {\n self[1]\n }\n\n #[inline]\n pub fn z(self) -> i32 {\n self[2]\n }\n\n #[inline]\n pub fn w(self) -> i32 {\n self[3]\n }\n}\n\nimpl AddAssign for I32x4 {\n #[inline]\n fn add_assign(&mut self, other: I32x4) {\n *self = *self + other\n }\n}\n\nimpl SubAssign for I32x4 {\n #[inline]\n fn sub_assign(&mut self, other: I32x4) {\n *self = *self - other\n }\n}\n\nimpl MulAssign for I32x4 {\n #[inline]\n fn mul_assign(&mut self, other: I32x4) {\n *self = *self * other\n }\n}\n\nimpl Neg for I32x4 {\n type Output = I32x4;\n #[inline]\n fn neg(self) -> I32x4 {\n I32x4::default() - self\n }\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/lib.rs", "sha256": "7bc08757defaab4b17d7ceabb702cc0b8dfcd30c6ad364dec6e88a8e2c0ae646", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/lib.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n#![cfg_attr(pf_rustc_nightly, allow(internal_features))]\n#![cfg_attr(pf_rustc_nightly, feature(link_llvm_intrinsics, core_intrinsics))]\n#![cfg_attr(pf_rustc_nightly, feature(simd_ffi))]\n#![cfg_attr(pf_rustc_nightly, feature(repr_simd))]\n\n//! A minimal SIMD abstraction, usable outside of Pathfinder.\n\n#[cfg(all(not(feature = \"pf-no-simd\"), pf_rustc_nightly, target_arch = \"aarch64\"))]\npub use crate::arm as default;\n#[cfg(any(\n feature = \"pf-no-simd\",\n not(any(\n target_arch = \"x86\",\n target_arch = \"x86_64\",\n all(pf_rustc_nightly, target_arch = \"aarch64\")\n ))\n))]\npub use crate::scalar as default;\n#[cfg(all(\n not(feature = \"pf-no-simd\"),\n any(target_arch = \"x86\", target_arch = \"x86_64\")\n))]\npub use crate::x86 as default;\n\n#[cfg(all(pf_rustc_nightly, target_arch = \"aarch64\"))]\npub mod arm;\nmod extras;\npub mod scalar;\n#[cfg(any(target_arch = \"x86\", target_arch = \"x86_64\"))]\npub mod x86;\n\n#[cfg(test)]\nmod test;\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/scalar/mod.rs", "sha256": "e636abae729aa7c5f8d7c4be0f0ec0aae02fea4b8a679e0934cfc3b3f70c8109", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/scalar.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse std::f32;\nuse std::fmt::{self, Debug, Formatter};\nuse std::ops::{Add, BitAnd, BitOr, Div, Index, IndexMut, Mul, Shr, Sub, Not};\n\nmod swizzle_f32x4;\nmod swizzle_i32x4;\n\n// Two 32-bit floats\n\n#[derive(Clone, Copy, Debug, Default, PartialEq)]\npub struct F32x2(pub [f32; 2]);\n\nimpl F32x2 {\n // Constructors\n\n #[inline]\n pub fn new(a: f32, b: f32) -> F32x2 {\n F32x2([a, b])\n }\n\n #[inline]\n pub fn splat(x: f32) -> F32x2 {\n F32x2([x, x])\n }\n\n // Basic operations\n\n #[inline]\n pub fn approx_recip(self) -> F32x2 {\n F32x2([1.0 / self[0], 1.0 / self[1]])\n }\n\n #[inline]\n pub fn min(self, other: F32x2) -> F32x2 {\n F32x2([f32::min(self[0], other[0]), f32::min(self[1], other[1])])\n }\n\n #[inline]\n pub fn max(self, other: F32x2) -> F32x2 {\n F32x2([f32::max(self[0], other[0]), f32::max(self[1], other[1])])\n }\n\n #[inline]\n pub fn clamp(self, min: F32x2, max: F32x2) -> F32x2 {\n self.max(min).min(max)\n }\n\n #[inline]\n pub fn abs(self) -> F32x2 {\n F32x2([self[0].abs(), self[1].abs()])\n }\n\n #[inline]\n pub fn floor(self) -> F32x2 {\n F32x2([self[0].floor(), self[1].floor()])\n }\n\n #[inline]\n pub fn ceil(self) -> F32x2 {\n F32x2([self[0].ceil(), self[1].ceil()])\n }\n\n #[inline]\n pub fn sqrt(self) -> F32x2 {\n F32x2([self[0].sqrt(), self[1].sqrt()])\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: F32x2) -> U32x2 {\n U32x2([\n if self[0] == other[0] { !0 } else { 0 },\n if self[1] == other[1] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_gt(self, other: F32x2) -> U32x2 {\n U32x2([\n if self[0] > other[0] { !0 } else { 0 },\n if self[1] > other[1] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_lt(self, other: F32x2) -> U32x2 {\n U32x2([\n if self[0] < other[0] { !0 } else { 0 },\n if self[1] < other[1] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_le(self, other: F32x2) -> U32x2 {\n U32x2([\n if self[0] <= other[0] { !0 } else { 0 },\n if self[1] <= other[1] { !0 } else { 0 },\n ])\n }\n\n // Conversions\n\n #[inline]\n pub fn to_f32x4(self) -> F32x4 {\n F32x4([self[0] as f32, self[1] as f32, 0.0, 0.0])\n }\n\n /// Converts these packed floats to integers via rounding.\n #[inline]\n pub fn to_i32x2(self) -> I32x2 {\n I32x2([self[0].round_ties_even() as i32, self[1].round_ties_even() as i32])\n }\n\n /// Converts these packed floats to integers via rounding.\n #[inline]\n pub fn to_i32x4(self) -> I32x4 {\n I32x4([self[0].round_ties_even() as i32, self[1].round_ties_even() as i32, 0, 0])\n }\n\n // Swizzle\n\n #[inline]\n pub fn yx(self) -> F32x2 {\n F32x2([self[1], self[0]])\n }\n\n // Concatenations\n\n #[inline]\n pub fn concat_xy_xy(self, other: F32x2) -> F32x4 {\n F32x4([self[0], self[1], other[0], other[1]])\n }\n}\n\nimpl Index for F32x2 {\n type Output = f32;\n #[inline]\n fn index(&self, index: usize) -> &f32 {\n &self.0[index]\n }\n}\n\nimpl IndexMut for F32x2 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut f32 {\n &mut self.0[index]\n }\n}\n\nimpl Add for F32x2 {\n type Output = F32x2;\n #[inline]\n fn add(self, other: F32x2) -> F32x2 {\n F32x2([self[0] + other[0], self[1] + other[1]])\n }\n}\n\nimpl Div for F32x2 {\n type Output = F32x2;\n #[inline]\n fn div(self, other: F32x2) -> F32x2 {\n F32x2([self[0] / other[0], self[1] / other[1]])\n }\n}\n\nimpl Mul for F32x2 {\n type Output = F32x2;\n #[inline]\n fn mul(self, other: F32x2) -> F32x2 {\n F32x2([self[0] * other[0], self[1] * other[1]])\n }\n}\n\nimpl Sub for F32x2 {\n type Output = F32x2;\n #[inline]\n fn sub(self, other: F32x2) -> F32x2 {\n F32x2([self[0] - other[0], self[1] - other[1]])\n }\n}\n\n// Four 32-bit floats\n\n#[derive(Clone, Copy, Default, PartialEq)]\npub struct F32x4(pub [f32; 4]);\n\nimpl F32x4 {\n #[inline]\n pub fn new(a: f32, b: f32, c: f32, d: f32) -> F32x4 {\n F32x4([a, b, c, d])\n }\n\n #[inline]\n pub fn splat(x: f32) -> F32x4 {\n F32x4([x; 4])\n }\n\n // Basic operations\n\n #[inline]\n pub fn approx_recip(self) -> F32x4 {\n F32x4([1.0 / self[0], 1.0 / self[1], 1.0 / self[2], 1.0 / self[3]])\n }\n\n #[inline]\n pub fn min(self, other: F32x4) -> F32x4 {\n F32x4([\n self[0].min(other[0]),\n self[1].min(other[1]),\n self[2].min(other[2]),\n self[3].min(other[3]),\n ])\n }\n\n #[inline]\n pub fn max(self, other: F32x4) -> F32x4 {\n F32x4([\n self[0].max(other[0]),\n self[1].max(other[1]),\n self[2].max(other[2]),\n self[3].max(other[3]),\n ])\n }\n\n #[inline]\n pub fn clamp(self, min: F32x4, max: F32x4) -> F32x4 {\n self.max(min).min(max)\n }\n\n #[inline]\n pub fn abs(self) -> F32x4 {\n F32x4([self[0].abs(), self[1].abs(), self[2].abs(), self[3].abs()])\n }\n\n #[inline]\n pub fn floor(self) -> F32x4 {\n F32x4([\n self[0].floor(),\n self[1].floor(),\n self[2].floor(),\n self[3].floor(),\n ])\n }\n\n #[inline]\n pub fn ceil(self) -> F32x4 {\n F32x4([\n self[0].ceil(),\n self[1].ceil(),\n self[2].ceil(),\n self[3].ceil(),\n ])\n }\n\n #[inline]\n pub fn sqrt(self) -> F32x4 {\n F32x4([\n self[0].sqrt(),\n self[1].sqrt(),\n self[2].sqrt(),\n self[3].sqrt(),\n ])\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: F32x4) -> U32x4 {\n U32x4([\n if self[0] == other[0] { !0 } else { 0 },\n if self[1] == other[1] { !0 } else { 0 },\n if self[2] == other[2] { !0 } else { 0 },\n if self[3] == other[3] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_gt(self, other: F32x4) -> U32x4 {\n U32x4([\n if self[0] > other[0] { !0 } else { 0 },\n if self[1] > other[1] { !0 } else { 0 },\n if self[2] > other[2] { !0 } else { 0 },\n if self[3] > other[3] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_le(self, other: F32x4) -> U32x4 {\n U32x4([\n if self[0] <= other[0] { !0 } else { 0 },\n if self[1] <= other[1] { !0 } else { 0 },\n if self[2] <= other[2] { !0 } else { 0 },\n if self[3] <= other[3] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_lt(self, other: F32x4) -> U32x4 {\n U32x4([\n if self[0] < other[0] { !0 } else { 0 },\n if self[1] < other[1] { !0 } else { 0 },\n if self[2] < other[2] { !0 } else { 0 },\n if self[3] < other[3] { !0 } else { 0 },\n ])\n }\n\n /// Converts these packed floats to integers via rounding.\n #[inline]\n pub fn to_i32x4(self) -> I32x4 {\n I32x4([\n self[0].round_ties_even() as i32,\n self[1].round_ties_even() as i32,\n self[2].round_ties_even() as i32,\n self[3].round_ties_even() as i32,\n ])\n }\n\n // Swizzle conversions\n\n #[inline]\n pub fn xy(self) -> F32x2 {\n F32x2([self[0], self[1]])\n }\n\n #[inline]\n pub fn xw(self) -> F32x2 {\n F32x2([self[0], self[3]])\n }\n\n #[inline]\n pub fn yx(self) -> F32x2 {\n F32x2([self[1], self[0]])\n }\n\n #[inline]\n pub fn zy(self) -> F32x2 {\n F32x2([self[2], self[1]])\n }\n\n #[inline]\n pub fn zw(self) -> F32x2 {\n F32x2([self[2], self[3]])\n }\n\n // Concatenations\n\n #[inline]\n pub fn concat_xy_xy(self, other: F32x4) -> F32x4 {\n F32x4([self[0], self[1], other[0], other[1]])\n }\n\n #[inline]\n pub fn concat_xy_zw(self, other: F32x4) -> F32x4 {\n F32x4([self[0], self[1], other[2], other[3]])\n }\n\n #[inline]\n pub fn concat_zw_zw(self, other: F32x4) -> F32x4 {\n F32x4([self[2], self[3], other[2], other[3]])\n }\n\n #[inline]\n pub fn concat_wz_yx(self, other: F32x4) -> F32x4 {\n F32x4([self[3], self[2], other[1], other[0]])\n }\n}\n\nimpl Index for F32x4 {\n type Output = f32;\n #[inline]\n fn index(&self, index: usize) -> &f32 {\n &self.0[index]\n }\n}\n\nimpl IndexMut for F32x4 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut f32 {\n &mut self.0[index]\n }\n}\n\nimpl Debug for F32x4 {\n #[inline]\n fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {\n write!(f, \"<{}, {}, {}, {}>\", self[0], self[1], self[2], self[3])\n }\n}\n\nimpl Add for F32x4 {\n type Output = F32x4;\n #[inline]\n fn add(self, other: F32x4) -> F32x4 {\n F32x4([\n self[0] + other[0],\n self[1] + other[1],\n self[2] + other[2],\n self[3] + other[3],\n ])\n }\n}\n\nimpl Div for F32x4 {\n type Output = F32x4;\n #[inline]\n fn div(self, other: F32x4) -> F32x4 {\n F32x4([\n self[0] / other[0],\n self[1] / other[1],\n self[2] / other[2],\n self[3] / other[3],\n ])\n }\n}\n\nimpl Mul for F32x4 {\n type Output = F32x4;\n #[inline]\n fn mul(self, other: F32x4) -> F32x4 {\n F32x4([\n self[0] * other[0],\n self[1] * other[1],\n self[2] * other[2],\n self[3] * other[3],\n ])\n }\n}\n\nimpl Sub for F32x4 {\n type Output = F32x4;\n #[inline]\n fn sub(self, other: F32x4) -> F32x4 {\n F32x4([\n self[0] - other[0],\n self[1] - other[1],\n self[2] - other[2],\n self[3] - other[3],\n ])\n }\n}\n\n// Two 32-bit signed integers\n\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\npub struct I32x2([i32; 2]);\n\nimpl I32x2 {\n #[inline]\n pub fn new(x: i32, y: i32) -> I32x2 {\n I32x2([x, y])\n }\n\n #[inline]\n pub fn splat(x: i32) -> I32x2 {\n I32x2([x, x])\n }\n\n // Accessors\n\n #[inline]\n pub fn x(self) -> i32 {\n self[0]\n }\n\n #[inline]\n pub fn y(self) -> i32 {\n self[1]\n }\n\n #[inline]\n pub fn concat_xy_xy(self, other: I32x2) -> I32x4 {\n I32x4([self[0], self[1], other[0], other[1]])\n }\n\n #[inline]\n pub fn min(self, other: I32x2) -> I32x2 {\n I32x2([\n self[0].min(other[0]),\n self[1].min(other[1]),\n ])\n }\n\n #[inline]\n pub fn max(self, other: I32x2) -> I32x2 {\n I32x2([\n self[0].max(other[0]),\n self[1].max(other[1]),\n ])\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: I32x2) -> U32x2 {\n U32x2([\n if self[0] == other[0] { !0 } else { 0 },\n if self[1] == other[1] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_gt(self, other: I32x2) -> U32x2 {\n U32x2([\n if self[0] > other[0] { !0 } else { 0 },\n if self[1] > other[1] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_le(self, other: I32x2) -> U32x2 {\n U32x2([\n if self[0] <= other[0] { !0 } else { 0 },\n if self[1] <= other[1] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_lt(self, other: I32x2) -> U32x2 {\n U32x2([\n if self[0] < other[0] { !0 } else { 0 },\n if self[1] < other[1] { !0 } else { 0 },\n ])\n } \n\n // Conversions\n\n /// Converts these packed integers to floats.\n #[inline]\n pub fn to_f32x2(self) -> F32x2 {\n F32x2([self[0] as f32, self[1] as f32])\n }\n}\n\nimpl Index for I32x2 {\n type Output = i32;\n #[inline]\n fn index(&self, index: usize) -> &i32 {\n &self.0[index]\n }\n}\n\nimpl IndexMut for I32x2 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut i32 {\n &mut self.0[index]\n }\n}\n\nimpl Add for I32x2 {\n type Output = I32x2;\n #[inline]\n fn add(self, other: I32x2) -> I32x2 {\n I32x2([self[0] + other[0], self[1] + other[1]])\n }\n}\n\nimpl Sub for I32x2 {\n type Output = I32x2;\n #[inline]\n fn sub(self, other: I32x2) -> I32x2 {\n I32x2([self[0] - other[0], self[1] - other[1]])\n }\n}\n\nimpl Mul for I32x2 {\n type Output = I32x2;\n #[inline]\n fn mul(self, other: I32x2) -> I32x2 {\n I32x2([self[0] * other[0], self[1] * other[1]])\n }\n}\n\n// Four 32-bit signed integers\n\n#[derive(Clone, Copy, Default, Debug, PartialEq)]\npub struct I32x4([i32; 4]);\n\nimpl I32x4 {\n #[inline]\n pub fn new(a: i32, b: i32, c: i32, d: i32) -> I32x4 {\n I32x4([a, b, c, d])\n }\n\n #[inline]\n pub fn splat(x: i32) -> I32x4 {\n I32x4([x; 4])\n }\n\n // Basic operations\n\n #[inline]\n pub fn min(self, other: I32x4) -> I32x4 {\n I32x4([\n self[0].min(other[0]),\n self[1].min(other[1]),\n self[2].min(other[2]),\n self[3].min(other[3]),\n ])\n }\n\n #[inline]\n pub fn max(self, other: I32x4) -> I32x4 {\n I32x4([\n self[0].max(other[0]),\n self[1].max(other[1]),\n self[2].max(other[2]),\n self[3].max(other[3]),\n ])\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: I32x4) -> U32x4 {\n U32x4([\n if self[0] == other[0] { !0 } else { 0 },\n if self[1] == other[1] { !0 } else { 0 },\n if self[2] == other[2] { !0 } else { 0 },\n if self[3] == other[3] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_gt(self, other: I32x4) -> U32x4 {\n U32x4([\n if self[0] > other[0] { !0 } else { 0 },\n if self[1] > other[1] { !0 } else { 0 },\n if self[2] > other[2] { !0 } else { 0 },\n if self[3] > other[3] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_le(self, other: I32x4) -> U32x4 {\n U32x4([\n if self[0] <= other[0] { !0 } else { 0 },\n if self[1] <= other[1] { !0 } else { 0 },\n if self[2] <= other[2] { !0 } else { 0 },\n if self[3] <= other[3] { !0 } else { 0 },\n ])\n }\n\n #[inline]\n pub fn packed_lt(self, other: I32x4) -> U32x4 {\n U32x4([\n if self[0] < other[0] { !0 } else { 0 },\n if self[1] < other[1] { !0 } else { 0 },\n if self[2] < other[2] { !0 } else { 0 },\n if self[3] < other[3] { !0 } else { 0 },\n ])\n }\n\n // Concatenations\n\n #[inline]\n pub fn concat_xy_xy(self, other: I32x4) -> I32x4 {\n I32x4([self[0], self[1], other[0], other[1]])\n }\n\n #[inline]\n pub fn concat_zw_zw(self, other: I32x4) -> I32x4 {\n I32x4([self[2], self[3], other[2], other[3]])\n }\n\n // Swizzle conversions\n\n #[inline]\n pub fn xy(self) -> I32x2 {\n I32x2([self[0], self[1]])\n }\n\n #[inline]\n pub fn xw(self) -> I32x2 {\n I32x2([self[0], self[3]])\n }\n\n #[inline]\n pub fn zy(self) -> I32x2 {\n I32x2([self[2], self[1]])\n }\n\n #[inline]\n pub fn zw(self) -> I32x2 {\n I32x2([self[2], self[3]])\n }\n\n // Conversions\n\n /// Converts these packed integers to floats.\n #[inline]\n pub fn to_f32x4(self) -> F32x4 {\n F32x4([\n self[0] as f32,\n self[1] as f32,\n self[2] as f32,\n self[3] as f32,\n ])\n }\n\n /// Converts these packed signed integers to unsigned integers.\n ///\n /// Overflowing values will wrap around.\n ///\n /// FIXME(pcwalton): Should they? This will assert on overflow in debug.\n #[inline]\n pub fn to_u32x4(self) -> U32x4 {\n U32x4([self[0] as u32, self[1] as u32, self[2] as u32, self[3] as u32])\n }\n}\n\nimpl Index for I32x4 {\n type Output = i32;\n #[inline]\n fn index(&self, index: usize) -> &i32 {\n &self.0[index]\n }\n}\n\nimpl IndexMut for I32x4 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut i32 {\n &mut self.0[index]\n }\n}\n\nimpl Add for I32x4 {\n type Output = I32x4;\n #[inline]\n fn add(self, other: I32x4) -> I32x4 {\n I32x4([\n self[0] + other[0],\n self[1] + other[1],\n self[2] + other[2],\n self[3] + other[3],\n ])\n }\n}\n\nimpl Sub for I32x4 {\n type Output = I32x4;\n #[inline]\n fn sub(self, other: I32x4) -> I32x4 {\n I32x4([\n self[0] - other[0],\n self[1] - other[1],\n self[2] - other[2],\n self[3] - other[3],\n ])\n }\n}\n\nimpl Mul for I32x4 {\n type Output = I32x4;\n #[inline]\n fn mul(self, other: I32x4) -> I32x4 {\n I32x4([\n self[0] * other[0],\n self[1] * other[1],\n self[2] * other[2],\n self[3] * other[3],\n ])\n }\n}\n\nimpl BitAnd for I32x4 {\n type Output = I32x4;\n #[inline]\n fn bitand(self, other: I32x4) -> I32x4 {\n I32x4([self[0] & other[0], self[1] & other[1], self[2] & other[2], self[3] & other[3]])\n }\n}\n\nimpl BitOr for I32x4 {\n type Output = I32x4;\n #[inline]\n fn bitor(self, other: I32x4) -> I32x4 {\n I32x4([self[0] | other[0], self[1] | other[1], self[2] | other[2], self[3] | other[3]])\n }\n}\n\nimpl Shr for I32x4 {\n type Output = I32x4;\n #[inline]\n fn shr(self, other: I32x4) -> I32x4 {\n I32x4([\n self[0] >> other[0],\n self[1] >> other[1],\n self[2] >> other[2],\n self[3] >> other[3],\n ])\n }\n}\n\n// Two 32-bit unsigned integers\n\n#[derive(Clone, Copy)]\npub struct U32x2(pub [u32; 2]);\n\nimpl U32x2 {\n #[inline]\n pub fn new(x: u32, y: u32) -> U32x2 {\n U32x2([x, y])\n }\n\n #[inline]\n pub fn splat(x: u32) -> U32x2 {\n U32x2::new(x, x)\n }\n\n /// Returns true if both booleans in this vector are true.\n ///\n /// The result is *undefined* if both values in this vector are not booleans. A boolean is a\n /// value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_true(&self) -> bool {\n self[0] == !0 && self[1] == !0\n }\n\n /// Returns true if both booleans in this vector are false.\n ///\n /// The result is *undefined* if both values in this vector are not booleans. A boolean is a\n /// value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_false(&self) -> bool {\n self[0] == 0 && self[1] == 0\n }\n\n #[inline]\n pub fn to_i32x2(self) -> I32x2 {\n I32x2::new(self[0] as i32, self[1] as i32)\n }\n\n}\n\nimpl BitAnd for U32x2 {\n type Output = U32x2;\n #[inline]\n fn bitand(self, other: U32x2) -> U32x2 {\n U32x2([self[0] & other[0], self[1] & other[1]])\n }\n}\n\nimpl BitOr for U32x2 {\n type Output = U32x2;\n #[inline]\n fn bitor(self, other: U32x2) -> U32x2 {\n U32x2([self[0] | other[0], self[1] | other[1]])\n }\n}\n\nimpl Not for U32x2 {\n type Output = U32x2;\n #[inline]\n fn not(self) -> U32x2 {\n U32x2([!self[0], !self[1]])\n }\n}\n\nimpl Index for U32x2 {\n type Output = u32;\n #[inline]\n fn index(&self, index: usize) -> &u32 {\n &self.0[index]\n }\n}\n\n// Four 32-bit unsigned integers\n\n#[derive(Clone, Copy, PartialEq, Eq, Debug)]\npub struct U32x4(pub [u32; 4]);\n\nimpl U32x4 {\n pub fn new(a: u32, b: u32, c: u32, d: u32) -> U32x4 {\n U32x4([a, b, c, d])\n }\n\n // Conversions\n\n /// Converts these packed unsigned integers to signed integers.\n ///\n /// Overflowing values will wrap around.\n ///\n /// FIXME(pcwalton): Should they? This will assert on overflow in debug.\n #[inline]\n pub fn to_i32x4(self) -> I32x4 {\n I32x4([self[0] as i32, self[1] as i32, self[2] as i32, self[3] as i32])\n }\n\n // Basic operations\n\n /// Returns true if all four booleans in this vector are true.\n ///\n /// The result is *undefined* if all four values in this vector are not booleans. A boolean is\n /// a value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_true(&self) -> bool {\n self[0] == !0 && self[1] == !0 && self[2] == !0 && self[3] == !0\n }\n\n /// Returns true if all four booleans in this vector are false.\n ///\n /// The result is *undefined* if all four values in this vector are not booleans. A boolean is\n /// a value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_false(&self) -> bool {\n self[0] == 0 && self[1] == 0 && self[2] == 0 && self[3] == 0\n }\n}\n\nimpl Index for U32x4 {\n type Output = u32;\n #[inline]\n fn index(&self, index: usize) -> &u32 {\n &self.0[index]\n }\n}\n\nimpl Shr for U32x4 {\n type Output = U32x4;\n #[inline]\n fn shr(self, amount: u32) -> U32x4 {\n U32x4([self[0] >> amount, self[1] >> amount, self[2] >> amount, self[3] >> amount])\n }\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/scalar/swizzle_f32x4.rs", "sha256": "0b243334740f3004ca870c2eb57b69bec32fa90a23c35f96b119529eb8e11557", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/scalar/swizzle_f32x4.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::scalar::F32x4;\n\nimpl F32x4 {\n /// Constructs a new vector from the first, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxx(self) -> F32x4 {\n F32x4([self[0], self[0], self[0], self[0]])\n }\n\n /// Constructs a new vector from the second, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxx(self) -> F32x4 {\n F32x4([self[1], self[0], self[0], self[0]])\n }\n\n /// Constructs a new vector from the third, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxx(self) -> F32x4 {\n F32x4([self[2], self[0], self[0], self[0]])\n }\n\n /// Constructs a new vector from the fourth, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxx(self) -> F32x4 {\n F32x4([self[3], self[0], self[0], self[0]])\n }\n\n /// Constructs a new vector from the first, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxx(self) -> F32x4 {\n F32x4([self[0], self[1], self[0], self[0]])\n }\n\n /// Constructs a new vector from the second, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxx(self) -> F32x4 {\n F32x4([self[1], self[1], self[0], self[0]])\n }\n\n /// Constructs a new vector from the third, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxx(self) -> F32x4 {\n F32x4([self[2], self[1], self[0], self[0]])\n }\n\n /// Constructs a new vector from the fourth, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxx(self) -> F32x4 {\n F32x4([self[3], self[1], self[0], self[0]])\n }\n\n /// Constructs a new vector from the first, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxx(self) -> F32x4 {\n F32x4([self[0], self[2], self[0], self[0]])\n }\n\n /// Constructs a new vector from the second, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxx(self) -> F32x4 {\n F32x4([self[1], self[2], self[0], self[0]])\n }\n\n /// Constructs a new vector from the third, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxx(self) -> F32x4 {\n F32x4([self[2], self[2], self[0], self[0]])\n }\n\n /// Constructs a new vector from the fourth, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxx(self) -> F32x4 {\n F32x4([self[3], self[2], self[0], self[0]])\n }\n\n /// Constructs a new vector from the first, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxx(self) -> F32x4 {\n F32x4([self[0], self[3], self[0], self[0]])\n }\n\n /// Constructs a new vector from the second, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxx(self) -> F32x4 {\n F32x4([self[1], self[3], self[0], self[0]])\n }\n\n /// Constructs a new vector from the third, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxx(self) -> F32x4 {\n F32x4([self[2], self[3], self[0], self[0]])\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxx(self) -> F32x4 {\n F32x4([self[3], self[3], self[0], self[0]])\n }\n\n /// Constructs a new vector from the first, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyx(self) -> F32x4 {\n F32x4([self[0], self[0], self[1], self[0]])\n }\n\n /// Constructs a new vector from the second, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyx(self) -> F32x4 {\n F32x4([self[1], self[0], self[1], self[0]])\n }\n\n /// Constructs a new vector from the third, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyx(self) -> F32x4 {\n F32x4([self[2], self[0], self[1], self[0]])\n }\n\n /// Constructs a new vector from the fourth, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyx(self) -> F32x4 {\n F32x4([self[3], self[0], self[1], self[0]])\n }\n\n /// Constructs a new vector from the first, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyx(self) -> F32x4 {\n F32x4([self[0], self[1], self[1], self[0]])\n }\n\n /// Constructs a new vector from the second, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyx(self) -> F32x4 {\n F32x4([self[1], self[1], self[1], self[0]])\n }\n\n /// Constructs a new vector from the third, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyx(self) -> F32x4 {\n F32x4([self[2], self[1], self[1], self[0]])\n }\n\n /// Constructs a new vector from the fourth, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyx(self) -> F32x4 {\n F32x4([self[3], self[1], self[1], self[0]])\n }\n\n /// Constructs a new vector from the first, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyx(self) -> F32x4 {\n F32x4([self[0], self[2], self[1], self[0]])\n }\n\n /// Constructs a new vector from the second, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyx(self) -> F32x4 {\n F32x4([self[1], self[2], self[1], self[0]])\n }\n\n /// Constructs a new vector from the third, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyx(self) -> F32x4 {\n F32x4([self[2], self[2], self[1], self[0]])\n }\n\n /// Constructs a new vector from the fourth, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyx(self) -> F32x4 {\n F32x4([self[3], self[2], self[1], self[0]])\n }\n\n /// Constructs a new vector from the first, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyx(self) -> F32x4 {\n F32x4([self[0], self[3], self[1], self[0]])\n }\n\n /// Constructs a new vector from the second, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyx(self) -> F32x4 {\n F32x4([self[1], self[3], self[1], self[0]])\n }\n\n /// Constructs a new vector from the third, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyx(self) -> F32x4 {\n F32x4([self[2], self[3], self[1], self[0]])\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyx(self) -> F32x4 {\n F32x4([self[3], self[3], self[1], self[0]])\n }\n\n /// Constructs a new vector from the first, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzx(self) -> F32x4 {\n F32x4([self[0], self[0], self[2], self[0]])\n }\n\n /// Constructs a new vector from the second, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzx(self) -> F32x4 {\n F32x4([self[1], self[0], self[2], self[0]])\n }\n\n /// Constructs a new vector from the third, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzx(self) -> F32x4 {\n F32x4([self[2], self[0], self[2], self[0]])\n }\n\n /// Constructs a new vector from the fourth, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzx(self) -> F32x4 {\n F32x4([self[3], self[0], self[2], self[0]])\n }\n\n /// Constructs a new vector from the first, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzx(self) -> F32x4 {\n F32x4([self[0], self[1], self[2], self[0]])\n }\n\n /// Constructs a new vector from the second, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzx(self) -> F32x4 {\n F32x4([self[1], self[1], self[2], self[0]])\n }\n\n /// Constructs a new vector from the third, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzx(self) -> F32x4 {\n F32x4([self[2], self[1], self[2], self[0]])\n }\n\n /// Constructs a new vector from the fourth, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzx(self) -> F32x4 {\n F32x4([self[3], self[1], self[2], self[0]])\n }\n\n /// Constructs a new vector from the first, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzx(self) -> F32x4 {\n F32x4([self[0], self[2], self[2], self[0]])\n }\n\n /// Constructs a new vector from the second, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzx(self) -> F32x4 {\n F32x4([self[1], self[2], self[2], self[0]])\n }\n\n /// Constructs a new vector from the third, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzx(self) -> F32x4 {\n F32x4([self[2], self[2], self[2], self[0]])\n }\n\n /// Constructs a new vector from the fourth, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzx(self) -> F32x4 {\n F32x4([self[3], self[2], self[2], self[0]])\n }\n\n /// Constructs a new vector from the first, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzx(self) -> F32x4 {\n F32x4([self[0], self[3], self[2], self[0]])\n }\n\n /// Constructs a new vector from the second, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzx(self) -> F32x4 {\n F32x4([self[1], self[3], self[2], self[0]])\n }\n\n /// Constructs a new vector from the third, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzx(self) -> F32x4 {\n F32x4([self[2], self[3], self[2], self[0]])\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzx(self) -> F32x4 {\n F32x4([self[3], self[3], self[2], self[0]])\n }\n\n /// Constructs a new vector from the first, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwx(self) -> F32x4 {\n F32x4([self[0], self[0], self[3], self[0]])\n }\n\n /// Constructs a new vector from the second, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwx(self) -> F32x4 {\n F32x4([self[1], self[0], self[3], self[0]])\n }\n\n /// Constructs a new vector from the third, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwx(self) -> F32x4 {\n F32x4([self[2], self[0], self[3], self[0]])\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwx(self) -> F32x4 {\n F32x4([self[3], self[0], self[3], self[0]])\n }\n\n /// Constructs a new vector from the first, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywx(self) -> F32x4 {\n F32x4([self[0], self[1], self[3], self[0]])\n }\n\n /// Constructs a new vector from the second, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywx(self) -> F32x4 {\n F32x4([self[1], self[1], self[3], self[0]])\n }\n\n /// Constructs a new vector from the third, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywx(self) -> F32x4 {\n F32x4([self[2], self[1], self[3], self[0]])\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywx(self) -> F32x4 {\n F32x4([self[3], self[1], self[3], self[0]])\n }\n\n /// Constructs a new vector from the first, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwx(self) -> F32x4 {\n F32x4([self[0], self[2], self[3], self[0]])\n }\n\n /// Constructs a new vector from the second, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwx(self) -> F32x4 {\n F32x4([self[1], self[2], self[3], self[0]])\n }\n\n /// Constructs a new vector from the third, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwx(self) -> F32x4 {\n F32x4([self[2], self[2], self[3], self[0]])\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwx(self) -> F32x4 {\n F32x4([self[3], self[2], self[3], self[0]])\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwx(self) -> F32x4 {\n F32x4([self[0], self[3], self[3], self[0]])\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwx(self) -> F32x4 {\n F32x4([self[1], self[3], self[3], self[0]])\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwx(self) -> F32x4 {\n F32x4([self[2], self[3], self[3], self[0]])\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwx(self) -> F32x4 {\n F32x4([self[3], self[3], self[3], self[0]])\n }\n\n /// Constructs a new vector from the first, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxy(self) -> F32x4 {\n F32x4([self[0], self[0], self[0], self[1]])\n }\n\n /// Constructs a new vector from the second, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxy(self) -> F32x4 {\n F32x4([self[1], self[0], self[0], self[1]])\n }\n\n /// Constructs a new vector from the third, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxy(self) -> F32x4 {\n F32x4([self[2], self[0], self[0], self[1]])\n }\n\n /// Constructs a new vector from the fourth, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxy(self) -> F32x4 {\n F32x4([self[3], self[0], self[0], self[1]])\n }\n\n /// Constructs a new vector from the first, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxy(self) -> F32x4 {\n F32x4([self[0], self[1], self[0], self[1]])\n }\n\n /// Constructs a new vector from the second, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxy(self) -> F32x4 {\n F32x4([self[1], self[1], self[0], self[1]])\n }\n\n /// Constructs a new vector from the third, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxy(self) -> F32x4 {\n F32x4([self[2], self[1], self[0], self[1]])\n }\n\n /// Constructs a new vector from the fourth, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxy(self) -> F32x4 {\n F32x4([self[3], self[1], self[0], self[1]])\n }\n\n /// Constructs a new vector from the first, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxy(self) -> F32x4 {\n F32x4([self[0], self[2], self[0], self[1]])\n }\n\n /// Constructs a new vector from the second, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxy(self) -> F32x4 {\n F32x4([self[1], self[2], self[0], self[1]])\n }\n\n /// Constructs a new vector from the third, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxy(self) -> F32x4 {\n F32x4([self[2], self[2], self[0], self[1]])\n }\n\n /// Constructs a new vector from the fourth, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxy(self) -> F32x4 {\n F32x4([self[3], self[2], self[0], self[1]])\n }\n\n /// Constructs a new vector from the first, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxy(self) -> F32x4 {\n F32x4([self[0], self[3], self[0], self[1]])\n }\n\n /// Constructs a new vector from the second, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxy(self) -> F32x4 {\n F32x4([self[1], self[3], self[0], self[1]])\n }\n\n /// Constructs a new vector from the third, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxy(self) -> F32x4 {\n F32x4([self[2], self[3], self[0], self[1]])\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxy(self) -> F32x4 {\n F32x4([self[3], self[3], self[0], self[1]])\n }\n\n /// Constructs a new vector from the first, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyy(self) -> F32x4 {\n F32x4([self[0], self[0], self[1], self[1]])\n }\n\n /// Constructs a new vector from the second, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyy(self) -> F32x4 {\n F32x4([self[1], self[0], self[1], self[1]])\n }\n\n /// Constructs a new vector from the third, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyy(self) -> F32x4 {\n F32x4([self[2], self[0], self[1], self[1]])\n }\n\n /// Constructs a new vector from the fourth, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyy(self) -> F32x4 {\n F32x4([self[3], self[0], self[1], self[1]])\n }\n\n /// Constructs a new vector from the first, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyy(self) -> F32x4 {\n F32x4([self[0], self[1], self[1], self[1]])\n }\n\n /// Constructs a new vector from the second, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyy(self) -> F32x4 {\n F32x4([self[1], self[1], self[1], self[1]])\n }\n\n /// Constructs a new vector from the third, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyy(self) -> F32x4 {\n F32x4([self[2], self[1], self[1], self[1]])\n }\n\n /// Constructs a new vector from the fourth, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyy(self) -> F32x4 {\n F32x4([self[3], self[1], self[1], self[1]])\n }\n\n /// Constructs a new vector from the first, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyy(self) -> F32x4 {\n F32x4([self[0], self[2], self[1], self[1]])\n }\n\n /// Constructs a new vector from the second, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyy(self) -> F32x4 {\n F32x4([self[1], self[2], self[1], self[1]])\n }\n\n /// Constructs a new vector from the third, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyy(self) -> F32x4 {\n F32x4([self[2], self[2], self[1], self[1]])\n }\n\n /// Constructs a new vector from the fourth, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyy(self) -> F32x4 {\n F32x4([self[3], self[2], self[1], self[1]])\n }\n\n /// Constructs a new vector from the first, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyy(self) -> F32x4 {\n F32x4([self[0], self[3], self[1], self[1]])\n }\n\n /// Constructs a new vector from the second, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyy(self) -> F32x4 {\n F32x4([self[1], self[3], self[1], self[1]])\n }\n\n /// Constructs a new vector from the third, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyy(self) -> F32x4 {\n F32x4([self[2], self[3], self[1], self[1]])\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyy(self) -> F32x4 {\n F32x4([self[3], self[3], self[1], self[1]])\n }\n\n /// Constructs a new vector from the first, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzy(self) -> F32x4 {\n F32x4([self[0], self[0], self[2], self[1]])\n }\n\n /// Constructs a new vector from the second, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzy(self) -> F32x4 {\n F32x4([self[1], self[0], self[2], self[1]])\n }\n\n /// Constructs a new vector from the third, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzy(self) -> F32x4 {\n F32x4([self[2], self[0], self[2], self[1]])\n }\n\n /// Constructs a new vector from the fourth, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzy(self) -> F32x4 {\n F32x4([self[3], self[0], self[2], self[1]])\n }\n\n /// Constructs a new vector from the first, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzy(self) -> F32x4 {\n F32x4([self[0], self[1], self[2], self[1]])\n }\n\n /// Constructs a new vector from the second, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzy(self) -> F32x4 {\n F32x4([self[1], self[1], self[2], self[1]])\n }\n\n /// Constructs a new vector from the third, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzy(self) -> F32x4 {\n F32x4([self[2], self[1], self[2], self[1]])\n }\n\n /// Constructs a new vector from the fourth, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzy(self) -> F32x4 {\n F32x4([self[3], self[1], self[2], self[1]])\n }\n\n /// Constructs a new vector from the first, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzy(self) -> F32x4 {\n F32x4([self[0], self[2], self[2], self[1]])\n }\n\n /// Constructs a new vector from the second, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzy(self) -> F32x4 {\n F32x4([self[1], self[2], self[2], self[1]])\n }\n\n /// Constructs a new vector from the third, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzy(self) -> F32x4 {\n F32x4([self[2], self[2], self[2], self[1]])\n }\n\n /// Constructs a new vector from the fourth, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzy(self) -> F32x4 {\n F32x4([self[3], self[2], self[2], self[1]])\n }\n\n /// Constructs a new vector from the first, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzy(self) -> F32x4 {\n F32x4([self[0], self[3], self[2], self[1]])\n }\n\n /// Constructs a new vector from the second, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzy(self) -> F32x4 {\n F32x4([self[1], self[3], self[2], self[1]])\n }\n\n /// Constructs a new vector from the third, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzy(self) -> F32x4 {\n F32x4([self[2], self[3], self[2], self[1]])\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzy(self) -> F32x4 {\n F32x4([self[3], self[3], self[2], self[1]])\n }\n\n /// Constructs a new vector from the first, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwy(self) -> F32x4 {\n F32x4([self[0], self[0], self[3], self[1]])\n }\n\n /// Constructs a new vector from the second, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwy(self) -> F32x4 {\n F32x4([self[1], self[0], self[3], self[1]])\n }\n\n /// Constructs a new vector from the third, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwy(self) -> F32x4 {\n F32x4([self[2], self[0], self[3], self[1]])\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwy(self) -> F32x4 {\n F32x4([self[3], self[0], self[3], self[1]])\n }\n\n /// Constructs a new vector from the first, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywy(self) -> F32x4 {\n F32x4([self[0], self[1], self[3], self[1]])\n }\n\n /// Constructs a new vector from the second, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywy(self) -> F32x4 {\n F32x4([self[1], self[1], self[3], self[1]])\n }\n\n /// Constructs a new vector from the third, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywy(self) -> F32x4 {\n F32x4([self[2], self[1], self[3], self[1]])\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywy(self) -> F32x4 {\n F32x4([self[3], self[1], self[3], self[1]])\n }\n\n /// Constructs a new vector from the first, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwy(self) -> F32x4 {\n F32x4([self[0], self[2], self[3], self[1]])\n }\n\n /// Constructs a new vector from the second, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwy(self) -> F32x4 {\n F32x4([self[1], self[2], self[3], self[1]])\n }\n\n /// Constructs a new vector from the third, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwy(self) -> F32x4 {\n F32x4([self[2], self[2], self[3], self[1]])\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwy(self) -> F32x4 {\n F32x4([self[3], self[2], self[3], self[1]])\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwy(self) -> F32x4 {\n F32x4([self[0], self[3], self[3], self[1]])\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwy(self) -> F32x4 {\n F32x4([self[1], self[3], self[3], self[1]])\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwy(self) -> F32x4 {\n F32x4([self[2], self[3], self[3], self[1]])\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwy(self) -> F32x4 {\n F32x4([self[3], self[3], self[3], self[1]])\n }\n\n /// Constructs a new vector from the first, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxz(self) -> F32x4 {\n F32x4([self[0], self[0], self[0], self[2]])\n }\n\n /// Constructs a new vector from the second, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxz(self) -> F32x4 {\n F32x4([self[1], self[0], self[0], self[2]])\n }\n\n /// Constructs a new vector from the third, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxz(self) -> F32x4 {\n F32x4([self[2], self[0], self[0], self[2]])\n }\n\n /// Constructs a new vector from the fourth, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxz(self) -> F32x4 {\n F32x4([self[3], self[0], self[0], self[2]])\n }\n\n /// Constructs a new vector from the first, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxz(self) -> F32x4 {\n F32x4([self[0], self[1], self[0], self[2]])\n }\n\n /// Constructs a new vector from the second, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxz(self) -> F32x4 {\n F32x4([self[1], self[1], self[0], self[2]])\n }\n\n /// Constructs a new vector from the third, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxz(self) -> F32x4 {\n F32x4([self[2], self[1], self[0], self[2]])\n }\n\n /// Constructs a new vector from the fourth, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxz(self) -> F32x4 {\n F32x4([self[3], self[1], self[0], self[2]])\n }\n\n /// Constructs a new vector from the first, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxz(self) -> F32x4 {\n F32x4([self[0], self[2], self[0], self[2]])\n }\n\n /// Constructs a new vector from the second, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxz(self) -> F32x4 {\n F32x4([self[1], self[2], self[0], self[2]])\n }\n\n /// Constructs a new vector from the third, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxz(self) -> F32x4 {\n F32x4([self[2], self[2], self[0], self[2]])\n }\n\n /// Constructs a new vector from the fourth, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxz(self) -> F32x4 {\n F32x4([self[3], self[2], self[0], self[2]])\n }\n\n /// Constructs a new vector from the first, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxz(self) -> F32x4 {\n F32x4([self[0], self[3], self[0], self[2]])\n }\n\n /// Constructs a new vector from the second, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxz(self) -> F32x4 {\n F32x4([self[1], self[3], self[0], self[2]])\n }\n\n /// Constructs a new vector from the third, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxz(self) -> F32x4 {\n F32x4([self[2], self[3], self[0], self[2]])\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxz(self) -> F32x4 {\n F32x4([self[3], self[3], self[0], self[2]])\n }\n\n /// Constructs a new vector from the first, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyz(self) -> F32x4 {\n F32x4([self[0], self[0], self[1], self[2]])\n }\n\n /// Constructs a new vector from the second, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyz(self) -> F32x4 {\n F32x4([self[1], self[0], self[1], self[2]])\n }\n\n /// Constructs a new vector from the third, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyz(self) -> F32x4 {\n F32x4([self[2], self[0], self[1], self[2]])\n }\n\n /// Constructs a new vector from the fourth, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyz(self) -> F32x4 {\n F32x4([self[3], self[0], self[1], self[2]])\n }\n\n /// Constructs a new vector from the first, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyz(self) -> F32x4 {\n F32x4([self[0], self[1], self[1], self[2]])\n }\n\n /// Constructs a new vector from the second, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyz(self) -> F32x4 {\n F32x4([self[1], self[1], self[1], self[2]])\n }\n\n /// Constructs a new vector from the third, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyz(self) -> F32x4 {\n F32x4([self[2], self[1], self[1], self[2]])\n }\n\n /// Constructs a new vector from the fourth, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyz(self) -> F32x4 {\n F32x4([self[3], self[1], self[1], self[2]])\n }\n\n /// Constructs a new vector from the first, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyz(self) -> F32x4 {\n F32x4([self[0], self[2], self[1], self[2]])\n }\n\n /// Constructs a new vector from the second, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyz(self) -> F32x4 {\n F32x4([self[1], self[2], self[1], self[2]])\n }\n\n /// Constructs a new vector from the third, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyz(self) -> F32x4 {\n F32x4([self[2], self[2], self[1], self[2]])\n }\n\n /// Constructs a new vector from the fourth, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyz(self) -> F32x4 {\n F32x4([self[3], self[2], self[1], self[2]])\n }\n\n /// Constructs a new vector from the first, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyz(self) -> F32x4 {\n F32x4([self[0], self[3], self[1], self[2]])\n }\n\n /// Constructs a new vector from the second, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyz(self) -> F32x4 {\n F32x4([self[1], self[3], self[1], self[2]])\n }\n\n /// Constructs a new vector from the third, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyz(self) -> F32x4 {\n F32x4([self[2], self[3], self[1], self[2]])\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyz(self) -> F32x4 {\n F32x4([self[3], self[3], self[1], self[2]])\n }\n\n /// Constructs a new vector from the first, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzz(self) -> F32x4 {\n F32x4([self[0], self[0], self[2], self[2]])\n }\n\n /// Constructs a new vector from the second, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzz(self) -> F32x4 {\n F32x4([self[1], self[0], self[2], self[2]])\n }\n\n /// Constructs a new vector from the third, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzz(self) -> F32x4 {\n F32x4([self[2], self[0], self[2], self[2]])\n }\n\n /// Constructs a new vector from the fourth, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzz(self) -> F32x4 {\n F32x4([self[3], self[0], self[2], self[2]])\n }\n\n /// Constructs a new vector from the first, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzz(self) -> F32x4 {\n F32x4([self[0], self[1], self[2], self[2]])\n }\n\n /// Constructs a new vector from the second, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzz(self) -> F32x4 {\n F32x4([self[1], self[1], self[2], self[2]])\n }\n\n /// Constructs a new vector from the third, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzz(self) -> F32x4 {\n F32x4([self[2], self[1], self[2], self[2]])\n }\n\n /// Constructs a new vector from the fourth, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzz(self) -> F32x4 {\n F32x4([self[3], self[1], self[2], self[2]])\n }\n\n /// Constructs a new vector from the first, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzz(self) -> F32x4 {\n F32x4([self[0], self[2], self[2], self[2]])\n }\n\n /// Constructs a new vector from the second, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzz(self) -> F32x4 {\n F32x4([self[1], self[2], self[2], self[2]])\n }\n\n /// Constructs a new vector from the third, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzz(self) -> F32x4 {\n F32x4([self[2], self[2], self[2], self[2]])\n }\n\n /// Constructs a new vector from the fourth, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzz(self) -> F32x4 {\n F32x4([self[3], self[2], self[2], self[2]])\n }\n\n /// Constructs a new vector from the first, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzz(self) -> F32x4 {\n F32x4([self[0], self[3], self[2], self[2]])\n }\n\n /// Constructs a new vector from the second, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzz(self) -> F32x4 {\n F32x4([self[1], self[3], self[2], self[2]])\n }\n\n /// Constructs a new vector from the third, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzz(self) -> F32x4 {\n F32x4([self[2], self[3], self[2], self[2]])\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzz(self) -> F32x4 {\n F32x4([self[3], self[3], self[2], self[2]])\n }\n\n /// Constructs a new vector from the first, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwz(self) -> F32x4 {\n F32x4([self[0], self[0], self[3], self[2]])\n }\n\n /// Constructs a new vector from the second, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwz(self) -> F32x4 {\n F32x4([self[1], self[0], self[3], self[2]])\n }\n\n /// Constructs a new vector from the third, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwz(self) -> F32x4 {\n F32x4([self[2], self[0], self[3], self[2]])\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwz(self) -> F32x4 {\n F32x4([self[3], self[0], self[3], self[2]])\n }\n\n /// Constructs a new vector from the first, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywz(self) -> F32x4 {\n F32x4([self[0], self[1], self[3], self[2]])\n }\n\n /// Constructs a new vector from the second, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywz(self) -> F32x4 {\n F32x4([self[1], self[1], self[3], self[2]])\n }\n\n /// Constructs a new vector from the third, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywz(self) -> F32x4 {\n F32x4([self[2], self[1], self[3], self[2]])\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywz(self) -> F32x4 {\n F32x4([self[3], self[1], self[3], self[2]])\n }\n\n /// Constructs a new vector from the first, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwz(self) -> F32x4 {\n F32x4([self[0], self[2], self[3], self[2]])\n }\n\n /// Constructs a new vector from the second, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwz(self) -> F32x4 {\n F32x4([self[1], self[2], self[3], self[2]])\n }\n\n /// Constructs a new vector from the third, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwz(self) -> F32x4 {\n F32x4([self[2], self[2], self[3], self[2]])\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwz(self) -> F32x4 {\n F32x4([self[3], self[2], self[3], self[2]])\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwz(self) -> F32x4 {\n F32x4([self[0], self[3], self[3], self[2]])\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwz(self) -> F32x4 {\n F32x4([self[1], self[3], self[3], self[2]])\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwz(self) -> F32x4 {\n F32x4([self[2], self[3], self[3], self[2]])\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwz(self) -> F32x4 {\n F32x4([self[3], self[3], self[3], self[2]])\n }\n\n /// Constructs a new vector from the first, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxw(self) -> F32x4 {\n F32x4([self[0], self[0], self[0], self[3]])\n }\n\n /// Constructs a new vector from the second, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxw(self) -> F32x4 {\n F32x4([self[1], self[0], self[0], self[3]])\n }\n\n /// Constructs a new vector from the third, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxw(self) -> F32x4 {\n F32x4([self[2], self[0], self[0], self[3]])\n }\n\n /// Constructs a new vector from the fourth, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxw(self) -> F32x4 {\n F32x4([self[3], self[0], self[0], self[3]])\n }\n\n /// Constructs a new vector from the first, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxw(self) -> F32x4 {\n F32x4([self[0], self[1], self[0], self[3]])\n }\n\n /// Constructs a new vector from the second, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxw(self) -> F32x4 {\n F32x4([self[1], self[1], self[0], self[3]])\n }\n\n /// Constructs a new vector from the third, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxw(self) -> F32x4 {\n F32x4([self[2], self[1], self[0], self[3]])\n }\n\n /// Constructs a new vector from the fourth, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxw(self) -> F32x4 {\n F32x4([self[3], self[1], self[0], self[3]])\n }\n\n /// Constructs a new vector from the first, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxw(self) -> F32x4 {\n F32x4([self[0], self[2], self[0], self[3]])\n }\n\n /// Constructs a new vector from the second, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxw(self) -> F32x4 {\n F32x4([self[1], self[2], self[0], self[3]])\n }\n\n /// Constructs a new vector from the third, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxw(self) -> F32x4 {\n F32x4([self[2], self[2], self[0], self[3]])\n }\n\n /// Constructs a new vector from the fourth, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxw(self) -> F32x4 {\n F32x4([self[3], self[2], self[0], self[3]])\n }\n\n /// Constructs a new vector from the first, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxw(self) -> F32x4 {\n F32x4([self[0], self[3], self[0], self[3]])\n }\n\n /// Constructs a new vector from the second, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxw(self) -> F32x4 {\n F32x4([self[1], self[3], self[0], self[3]])\n }\n\n /// Constructs a new vector from the third, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxw(self) -> F32x4 {\n F32x4([self[2], self[3], self[0], self[3]])\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxw(self) -> F32x4 {\n F32x4([self[3], self[3], self[0], self[3]])\n }\n\n /// Constructs a new vector from the first, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyw(self) -> F32x4 {\n F32x4([self[0], self[0], self[1], self[3]])\n }\n\n /// Constructs a new vector from the second, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyw(self) -> F32x4 {\n F32x4([self[1], self[0], self[1], self[3]])\n }\n\n /// Constructs a new vector from the third, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyw(self) -> F32x4 {\n F32x4([self[2], self[0], self[1], self[3]])\n }\n\n /// Constructs a new vector from the fourth, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyw(self) -> F32x4 {\n F32x4([self[3], self[0], self[1], self[3]])\n }\n\n /// Constructs a new vector from the first, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyw(self) -> F32x4 {\n F32x4([self[0], self[1], self[1], self[3]])\n }\n\n /// Constructs a new vector from the second, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyw(self) -> F32x4 {\n F32x4([self[1], self[1], self[1], self[3]])\n }\n\n /// Constructs a new vector from the third, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyw(self) -> F32x4 {\n F32x4([self[2], self[1], self[1], self[3]])\n }\n\n /// Constructs a new vector from the fourth, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyw(self) -> F32x4 {\n F32x4([self[3], self[1], self[1], self[3]])\n }\n\n /// Constructs a new vector from the first, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyw(self) -> F32x4 {\n F32x4([self[0], self[2], self[1], self[3]])\n }\n\n /// Constructs a new vector from the second, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyw(self) -> F32x4 {\n F32x4([self[1], self[2], self[1], self[3]])\n }\n\n /// Constructs a new vector from the third, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyw(self) -> F32x4 {\n F32x4([self[2], self[2], self[1], self[3]])\n }\n\n /// Constructs a new vector from the fourth, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyw(self) -> F32x4 {\n F32x4([self[3], self[2], self[1], self[3]])\n }\n\n /// Constructs a new vector from the first, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyw(self) -> F32x4 {\n F32x4([self[0], self[3], self[1], self[3]])\n }\n\n /// Constructs a new vector from the second, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyw(self) -> F32x4 {\n F32x4([self[1], self[3], self[1], self[3]])\n }\n\n /// Constructs a new vector from the third, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyw(self) -> F32x4 {\n F32x4([self[2], self[3], self[1], self[3]])\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyw(self) -> F32x4 {\n F32x4([self[3], self[3], self[1], self[3]])\n }\n\n /// Constructs a new vector from the first, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzw(self) -> F32x4 {\n F32x4([self[0], self[0], self[2], self[3]])\n }\n\n /// Constructs a new vector from the second, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzw(self) -> F32x4 {\n F32x4([self[1], self[0], self[2], self[3]])\n }\n\n /// Constructs a new vector from the third, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzw(self) -> F32x4 {\n F32x4([self[2], self[0], self[2], self[3]])\n }\n\n /// Constructs a new vector from the fourth, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzw(self) -> F32x4 {\n F32x4([self[3], self[0], self[2], self[3]])\n }\n\n /// Constructs a new vector from the first, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzw(self) -> F32x4 {\n F32x4([self[0], self[1], self[2], self[3]])\n }\n\n /// Constructs a new vector from the second, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzw(self) -> F32x4 {\n F32x4([self[1], self[1], self[2], self[3]])\n }\n\n /// Constructs a new vector from the third, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzw(self) -> F32x4 {\n F32x4([self[2], self[1], self[2], self[3]])\n }\n\n /// Constructs a new vector from the fourth, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzw(self) -> F32x4 {\n F32x4([self[3], self[1], self[2], self[3]])\n }\n\n /// Constructs a new vector from the first, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzw(self) -> F32x4 {\n F32x4([self[0], self[2], self[2], self[3]])\n }\n\n /// Constructs a new vector from the second, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzw(self) -> F32x4 {\n F32x4([self[1], self[2], self[2], self[3]])\n }\n\n /// Constructs a new vector from the third, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzw(self) -> F32x4 {\n F32x4([self[2], self[2], self[2], self[3]])\n }\n\n /// Constructs a new vector from the fourth, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzw(self) -> F32x4 {\n F32x4([self[3], self[2], self[2], self[3]])\n }\n\n /// Constructs a new vector from the first, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzw(self) -> F32x4 {\n F32x4([self[0], self[3], self[2], self[3]])\n }\n\n /// Constructs a new vector from the second, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzw(self) -> F32x4 {\n F32x4([self[1], self[3], self[2], self[3]])\n }\n\n /// Constructs a new vector from the third, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzw(self) -> F32x4 {\n F32x4([self[2], self[3], self[2], self[3]])\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzw(self) -> F32x4 {\n F32x4([self[3], self[3], self[2], self[3]])\n }\n\n /// Constructs a new vector from the first, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxww(self) -> F32x4 {\n F32x4([self[0], self[0], self[3], self[3]])\n }\n\n /// Constructs a new vector from the second, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxww(self) -> F32x4 {\n F32x4([self[1], self[0], self[3], self[3]])\n }\n\n /// Constructs a new vector from the third, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxww(self) -> F32x4 {\n F32x4([self[2], self[0], self[3], self[3]])\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxww(self) -> F32x4 {\n F32x4([self[3], self[0], self[3], self[3]])\n }\n\n /// Constructs a new vector from the first, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyww(self) -> F32x4 {\n F32x4([self[0], self[1], self[3], self[3]])\n }\n\n /// Constructs a new vector from the second, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyww(self) -> F32x4 {\n F32x4([self[1], self[1], self[3], self[3]])\n }\n\n /// Constructs a new vector from the third, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyww(self) -> F32x4 {\n F32x4([self[2], self[1], self[3], self[3]])\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyww(self) -> F32x4 {\n F32x4([self[3], self[1], self[3], self[3]])\n }\n\n /// Constructs a new vector from the first, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzww(self) -> F32x4 {\n F32x4([self[0], self[2], self[3], self[3]])\n }\n\n /// Constructs a new vector from the second, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzww(self) -> F32x4 {\n F32x4([self[1], self[2], self[3], self[3]])\n }\n\n /// Constructs a new vector from the third, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzww(self) -> F32x4 {\n F32x4([self[2], self[2], self[3], self[3]])\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzww(self) -> F32x4 {\n F32x4([self[3], self[2], self[3], self[3]])\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwww(self) -> F32x4 {\n F32x4([self[0], self[3], self[3], self[3]])\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywww(self) -> F32x4 {\n F32x4([self[1], self[3], self[3], self[3]])\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwww(self) -> F32x4 {\n F32x4([self[2], self[3], self[3], self[3]])\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwww(self) -> F32x4 {\n F32x4([self[3], self[3], self[3], self[3]])\n }\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/scalar/swizzle_i32x4.rs", "sha256": "1612776f2bc21fbbb5b0770a90b9b248abae28ad6e3a5585abad5b8bbfa499bd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/scalar/swizzle_i32x4.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::scalar::I32x4;\n\nimpl I32x4 {\n /// Constructs a new vector from the first, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxx(self) -> I32x4 {\n I32x4([self[0], self[0], self[0], self[0]])\n }\n\n /// Constructs a new vector from the second, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxx(self) -> I32x4 {\n I32x4([self[1], self[0], self[0], self[0]])\n }\n\n /// Constructs a new vector from the third, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxx(self) -> I32x4 {\n I32x4([self[2], self[0], self[0], self[0]])\n }\n\n /// Constructs a new vector from the fourth, first, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxx(self) -> I32x4 {\n I32x4([self[3], self[0], self[0], self[0]])\n }\n\n /// Constructs a new vector from the first, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxx(self) -> I32x4 {\n I32x4([self[0], self[1], self[0], self[0]])\n }\n\n /// Constructs a new vector from the second, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxx(self) -> I32x4 {\n I32x4([self[1], self[1], self[0], self[0]])\n }\n\n /// Constructs a new vector from the third, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxx(self) -> I32x4 {\n I32x4([self[2], self[1], self[0], self[0]])\n }\n\n /// Constructs a new vector from the fourth, second, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxx(self) -> I32x4 {\n I32x4([self[3], self[1], self[0], self[0]])\n }\n\n /// Constructs a new vector from the first, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxx(self) -> I32x4 {\n I32x4([self[0], self[2], self[0], self[0]])\n }\n\n /// Constructs a new vector from the second, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxx(self) -> I32x4 {\n I32x4([self[1], self[2], self[0], self[0]])\n }\n\n /// Constructs a new vector from the third, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxx(self) -> I32x4 {\n I32x4([self[2], self[2], self[0], self[0]])\n }\n\n /// Constructs a new vector from the fourth, third, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxx(self) -> I32x4 {\n I32x4([self[3], self[2], self[0], self[0]])\n }\n\n /// Constructs a new vector from the first, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxx(self) -> I32x4 {\n I32x4([self[0], self[3], self[0], self[0]])\n }\n\n /// Constructs a new vector from the second, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxx(self) -> I32x4 {\n I32x4([self[1], self[3], self[0], self[0]])\n }\n\n /// Constructs a new vector from the third, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxx(self) -> I32x4 {\n I32x4([self[2], self[3], self[0], self[0]])\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxx(self) -> I32x4 {\n I32x4([self[3], self[3], self[0], self[0]])\n }\n\n /// Constructs a new vector from the first, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyx(self) -> I32x4 {\n I32x4([self[0], self[0], self[1], self[0]])\n }\n\n /// Constructs a new vector from the second, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyx(self) -> I32x4 {\n I32x4([self[1], self[0], self[1], self[0]])\n }\n\n /// Constructs a new vector from the third, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyx(self) -> I32x4 {\n I32x4([self[2], self[0], self[1], self[0]])\n }\n\n /// Constructs a new vector from the fourth, first, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyx(self) -> I32x4 {\n I32x4([self[3], self[0], self[1], self[0]])\n }\n\n /// Constructs a new vector from the first, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyx(self) -> I32x4 {\n I32x4([self[0], self[1], self[1], self[0]])\n }\n\n /// Constructs a new vector from the second, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyx(self) -> I32x4 {\n I32x4([self[1], self[1], self[1], self[0]])\n }\n\n /// Constructs a new vector from the third, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyx(self) -> I32x4 {\n I32x4([self[2], self[1], self[1], self[0]])\n }\n\n /// Constructs a new vector from the fourth, second, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyx(self) -> I32x4 {\n I32x4([self[3], self[1], self[1], self[0]])\n }\n\n /// Constructs a new vector from the first, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyx(self) -> I32x4 {\n I32x4([self[0], self[2], self[1], self[0]])\n }\n\n /// Constructs a new vector from the second, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyx(self) -> I32x4 {\n I32x4([self[1], self[2], self[1], self[0]])\n }\n\n /// Constructs a new vector from the third, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyx(self) -> I32x4 {\n I32x4([self[2], self[2], self[1], self[0]])\n }\n\n /// Constructs a new vector from the fourth, third, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyx(self) -> I32x4 {\n I32x4([self[3], self[2], self[1], self[0]])\n }\n\n /// Constructs a new vector from the first, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyx(self) -> I32x4 {\n I32x4([self[0], self[3], self[1], self[0]])\n }\n\n /// Constructs a new vector from the second, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyx(self) -> I32x4 {\n I32x4([self[1], self[3], self[1], self[0]])\n }\n\n /// Constructs a new vector from the third, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyx(self) -> I32x4 {\n I32x4([self[2], self[3], self[1], self[0]])\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyx(self) -> I32x4 {\n I32x4([self[3], self[3], self[1], self[0]])\n }\n\n /// Constructs a new vector from the first, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzx(self) -> I32x4 {\n I32x4([self[0], self[0], self[2], self[0]])\n }\n\n /// Constructs a new vector from the second, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzx(self) -> I32x4 {\n I32x4([self[1], self[0], self[2], self[0]])\n }\n\n /// Constructs a new vector from the third, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzx(self) -> I32x4 {\n I32x4([self[2], self[0], self[2], self[0]])\n }\n\n /// Constructs a new vector from the fourth, first, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzx(self) -> I32x4 {\n I32x4([self[3], self[0], self[2], self[0]])\n }\n\n /// Constructs a new vector from the first, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzx(self) -> I32x4 {\n I32x4([self[0], self[1], self[2], self[0]])\n }\n\n /// Constructs a new vector from the second, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzx(self) -> I32x4 {\n I32x4([self[1], self[1], self[2], self[0]])\n }\n\n /// Constructs a new vector from the third, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzx(self) -> I32x4 {\n I32x4([self[2], self[1], self[2], self[0]])\n }\n\n /// Constructs a new vector from the fourth, second, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzx(self) -> I32x4 {\n I32x4([self[3], self[1], self[2], self[0]])\n }\n\n /// Constructs a new vector from the first, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzx(self) -> I32x4 {\n I32x4([self[0], self[2], self[2], self[0]])\n }\n\n /// Constructs a new vector from the second, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzx(self) -> I32x4 {\n I32x4([self[1], self[2], self[2], self[0]])\n }\n\n /// Constructs a new vector from the third, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzx(self) -> I32x4 {\n I32x4([self[2], self[2], self[2], self[0]])\n }\n\n /// Constructs a new vector from the fourth, third, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzx(self) -> I32x4 {\n I32x4([self[3], self[2], self[2], self[0]])\n }\n\n /// Constructs a new vector from the first, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzx(self) -> I32x4 {\n I32x4([self[0], self[3], self[2], self[0]])\n }\n\n /// Constructs a new vector from the second, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzx(self) -> I32x4 {\n I32x4([self[1], self[3], self[2], self[0]])\n }\n\n /// Constructs a new vector from the third, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzx(self) -> I32x4 {\n I32x4([self[2], self[3], self[2], self[0]])\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzx(self) -> I32x4 {\n I32x4([self[3], self[3], self[2], self[0]])\n }\n\n /// Constructs a new vector from the first, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwx(self) -> I32x4 {\n I32x4([self[0], self[0], self[3], self[0]])\n }\n\n /// Constructs a new vector from the second, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwx(self) -> I32x4 {\n I32x4([self[1], self[0], self[3], self[0]])\n }\n\n /// Constructs a new vector from the third, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwx(self) -> I32x4 {\n I32x4([self[2], self[0], self[3], self[0]])\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwx(self) -> I32x4 {\n I32x4([self[3], self[0], self[3], self[0]])\n }\n\n /// Constructs a new vector from the first, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywx(self) -> I32x4 {\n I32x4([self[0], self[1], self[3], self[0]])\n }\n\n /// Constructs a new vector from the second, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywx(self) -> I32x4 {\n I32x4([self[1], self[1], self[3], self[0]])\n }\n\n /// Constructs a new vector from the third, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywx(self) -> I32x4 {\n I32x4([self[2], self[1], self[3], self[0]])\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywx(self) -> I32x4 {\n I32x4([self[3], self[1], self[3], self[0]])\n }\n\n /// Constructs a new vector from the first, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwx(self) -> I32x4 {\n I32x4([self[0], self[2], self[3], self[0]])\n }\n\n /// Constructs a new vector from the second, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwx(self) -> I32x4 {\n I32x4([self[1], self[2], self[3], self[0]])\n }\n\n /// Constructs a new vector from the third, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwx(self) -> I32x4 {\n I32x4([self[2], self[2], self[3], self[0]])\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwx(self) -> I32x4 {\n I32x4([self[3], self[2], self[3], self[0]])\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwx(self) -> I32x4 {\n I32x4([self[0], self[3], self[3], self[0]])\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwx(self) -> I32x4 {\n I32x4([self[1], self[3], self[3], self[0]])\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwx(self) -> I32x4 {\n I32x4([self[2], self[3], self[3], self[0]])\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and first\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwx(self) -> I32x4 {\n I32x4([self[3], self[3], self[3], self[0]])\n }\n\n /// Constructs a new vector from the first, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxy(self) -> I32x4 {\n I32x4([self[0], self[0], self[0], self[1]])\n }\n\n /// Constructs a new vector from the second, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxy(self) -> I32x4 {\n I32x4([self[1], self[0], self[0], self[1]])\n }\n\n /// Constructs a new vector from the third, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxy(self) -> I32x4 {\n I32x4([self[2], self[0], self[0], self[1]])\n }\n\n /// Constructs a new vector from the fourth, first, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxy(self) -> I32x4 {\n I32x4([self[3], self[0], self[0], self[1]])\n }\n\n /// Constructs a new vector from the first, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxy(self) -> I32x4 {\n I32x4([self[0], self[1], self[0], self[1]])\n }\n\n /// Constructs a new vector from the second, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxy(self) -> I32x4 {\n I32x4([self[1], self[1], self[0], self[1]])\n }\n\n /// Constructs a new vector from the third, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxy(self) -> I32x4 {\n I32x4([self[2], self[1], self[0], self[1]])\n }\n\n /// Constructs a new vector from the fourth, second, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxy(self) -> I32x4 {\n I32x4([self[3], self[1], self[0], self[1]])\n }\n\n /// Constructs a new vector from the first, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxy(self) -> I32x4 {\n I32x4([self[0], self[2], self[0], self[1]])\n }\n\n /// Constructs a new vector from the second, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxy(self) -> I32x4 {\n I32x4([self[1], self[2], self[0], self[1]])\n }\n\n /// Constructs a new vector from the third, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxy(self) -> I32x4 {\n I32x4([self[2], self[2], self[0], self[1]])\n }\n\n /// Constructs a new vector from the fourth, third, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxy(self) -> I32x4 {\n I32x4([self[3], self[2], self[0], self[1]])\n }\n\n /// Constructs a new vector from the first, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxy(self) -> I32x4 {\n I32x4([self[0], self[3], self[0], self[1]])\n }\n\n /// Constructs a new vector from the second, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxy(self) -> I32x4 {\n I32x4([self[1], self[3], self[0], self[1]])\n }\n\n /// Constructs a new vector from the third, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxy(self) -> I32x4 {\n I32x4([self[2], self[3], self[0], self[1]])\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxy(self) -> I32x4 {\n I32x4([self[3], self[3], self[0], self[1]])\n }\n\n /// Constructs a new vector from the first, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyy(self) -> I32x4 {\n I32x4([self[0], self[0], self[1], self[1]])\n }\n\n /// Constructs a new vector from the second, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyy(self) -> I32x4 {\n I32x4([self[1], self[0], self[1], self[1]])\n }\n\n /// Constructs a new vector from the third, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyy(self) -> I32x4 {\n I32x4([self[2], self[0], self[1], self[1]])\n }\n\n /// Constructs a new vector from the fourth, first, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyy(self) -> I32x4 {\n I32x4([self[3], self[0], self[1], self[1]])\n }\n\n /// Constructs a new vector from the first, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyy(self) -> I32x4 {\n I32x4([self[0], self[1], self[1], self[1]])\n }\n\n /// Constructs a new vector from the second, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyy(self) -> I32x4 {\n I32x4([self[1], self[1], self[1], self[1]])\n }\n\n /// Constructs a new vector from the third, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyy(self) -> I32x4 {\n I32x4([self[2], self[1], self[1], self[1]])\n }\n\n /// Constructs a new vector from the fourth, second, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyy(self) -> I32x4 {\n I32x4([self[3], self[1], self[1], self[1]])\n }\n\n /// Constructs a new vector from the first, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyy(self) -> I32x4 {\n I32x4([self[0], self[2], self[1], self[1]])\n }\n\n /// Constructs a new vector from the second, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyy(self) -> I32x4 {\n I32x4([self[1], self[2], self[1], self[1]])\n }\n\n /// Constructs a new vector from the third, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyy(self) -> I32x4 {\n I32x4([self[2], self[2], self[1], self[1]])\n }\n\n /// Constructs a new vector from the fourth, third, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyy(self) -> I32x4 {\n I32x4([self[3], self[2], self[1], self[1]])\n }\n\n /// Constructs a new vector from the first, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyy(self) -> I32x4 {\n I32x4([self[0], self[3], self[1], self[1]])\n }\n\n /// Constructs a new vector from the second, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyy(self) -> I32x4 {\n I32x4([self[1], self[3], self[1], self[1]])\n }\n\n /// Constructs a new vector from the third, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyy(self) -> I32x4 {\n I32x4([self[2], self[3], self[1], self[1]])\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyy(self) -> I32x4 {\n I32x4([self[3], self[3], self[1], self[1]])\n }\n\n /// Constructs a new vector from the first, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzy(self) -> I32x4 {\n I32x4([self[0], self[0], self[2], self[1]])\n }\n\n /// Constructs a new vector from the second, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzy(self) -> I32x4 {\n I32x4([self[1], self[0], self[2], self[1]])\n }\n\n /// Constructs a new vector from the third, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzy(self) -> I32x4 {\n I32x4([self[2], self[0], self[2], self[1]])\n }\n\n /// Constructs a new vector from the fourth, first, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzy(self) -> I32x4 {\n I32x4([self[3], self[0], self[2], self[1]])\n }\n\n /// Constructs a new vector from the first, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzy(self) -> I32x4 {\n I32x4([self[0], self[1], self[2], self[1]])\n }\n\n /// Constructs a new vector from the second, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzy(self) -> I32x4 {\n I32x4([self[1], self[1], self[2], self[1]])\n }\n\n /// Constructs a new vector from the third, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzy(self) -> I32x4 {\n I32x4([self[2], self[1], self[2], self[1]])\n }\n\n /// Constructs a new vector from the fourth, second, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzy(self) -> I32x4 {\n I32x4([self[3], self[1], self[2], self[1]])\n }\n\n /// Constructs a new vector from the first, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzy(self) -> I32x4 {\n I32x4([self[0], self[2], self[2], self[1]])\n }\n\n /// Constructs a new vector from the second, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzy(self) -> I32x4 {\n I32x4([self[1], self[2], self[2], self[1]])\n }\n\n /// Constructs a new vector from the third, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzy(self) -> I32x4 {\n I32x4([self[2], self[2], self[2], self[1]])\n }\n\n /// Constructs a new vector from the fourth, third, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzy(self) -> I32x4 {\n I32x4([self[3], self[2], self[2], self[1]])\n }\n\n /// Constructs a new vector from the first, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzy(self) -> I32x4 {\n I32x4([self[0], self[3], self[2], self[1]])\n }\n\n /// Constructs a new vector from the second, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzy(self) -> I32x4 {\n I32x4([self[1], self[3], self[2], self[1]])\n }\n\n /// Constructs a new vector from the third, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzy(self) -> I32x4 {\n I32x4([self[2], self[3], self[2], self[1]])\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzy(self) -> I32x4 {\n I32x4([self[3], self[3], self[2], self[1]])\n }\n\n /// Constructs a new vector from the first, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwy(self) -> I32x4 {\n I32x4([self[0], self[0], self[3], self[1]])\n }\n\n /// Constructs a new vector from the second, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwy(self) -> I32x4 {\n I32x4([self[1], self[0], self[3], self[1]])\n }\n\n /// Constructs a new vector from the third, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwy(self) -> I32x4 {\n I32x4([self[2], self[0], self[3], self[1]])\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwy(self) -> I32x4 {\n I32x4([self[3], self[0], self[3], self[1]])\n }\n\n /// Constructs a new vector from the first, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywy(self) -> I32x4 {\n I32x4([self[0], self[1], self[3], self[1]])\n }\n\n /// Constructs a new vector from the second, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywy(self) -> I32x4 {\n I32x4([self[1], self[1], self[3], self[1]])\n }\n\n /// Constructs a new vector from the third, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywy(self) -> I32x4 {\n I32x4([self[2], self[1], self[3], self[1]])\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywy(self) -> I32x4 {\n I32x4([self[3], self[1], self[3], self[1]])\n }\n\n /// Constructs a new vector from the first, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwy(self) -> I32x4 {\n I32x4([self[0], self[2], self[3], self[1]])\n }\n\n /// Constructs a new vector from the second, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwy(self) -> I32x4 {\n I32x4([self[1], self[2], self[3], self[1]])\n }\n\n /// Constructs a new vector from the third, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwy(self) -> I32x4 {\n I32x4([self[2], self[2], self[3], self[1]])\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwy(self) -> I32x4 {\n I32x4([self[3], self[2], self[3], self[1]])\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwy(self) -> I32x4 {\n I32x4([self[0], self[3], self[3], self[1]])\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwy(self) -> I32x4 {\n I32x4([self[1], self[3], self[3], self[1]])\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwy(self) -> I32x4 {\n I32x4([self[2], self[3], self[3], self[1]])\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and second\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwy(self) -> I32x4 {\n I32x4([self[3], self[3], self[3], self[1]])\n }\n\n /// Constructs a new vector from the first, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxz(self) -> I32x4 {\n I32x4([self[0], self[0], self[0], self[2]])\n }\n\n /// Constructs a new vector from the second, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxz(self) -> I32x4 {\n I32x4([self[1], self[0], self[0], self[2]])\n }\n\n /// Constructs a new vector from the third, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxz(self) -> I32x4 {\n I32x4([self[2], self[0], self[0], self[2]])\n }\n\n /// Constructs a new vector from the fourth, first, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxz(self) -> I32x4 {\n I32x4([self[3], self[0], self[0], self[2]])\n }\n\n /// Constructs a new vector from the first, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxz(self) -> I32x4 {\n I32x4([self[0], self[1], self[0], self[2]])\n }\n\n /// Constructs a new vector from the second, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxz(self) -> I32x4 {\n I32x4([self[1], self[1], self[0], self[2]])\n }\n\n /// Constructs a new vector from the third, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxz(self) -> I32x4 {\n I32x4([self[2], self[1], self[0], self[2]])\n }\n\n /// Constructs a new vector from the fourth, second, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxz(self) -> I32x4 {\n I32x4([self[3], self[1], self[0], self[2]])\n }\n\n /// Constructs a new vector from the first, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxz(self) -> I32x4 {\n I32x4([self[0], self[2], self[0], self[2]])\n }\n\n /// Constructs a new vector from the second, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxz(self) -> I32x4 {\n I32x4([self[1], self[2], self[0], self[2]])\n }\n\n /// Constructs a new vector from the third, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxz(self) -> I32x4 {\n I32x4([self[2], self[2], self[0], self[2]])\n }\n\n /// Constructs a new vector from the fourth, third, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxz(self) -> I32x4 {\n I32x4([self[3], self[2], self[0], self[2]])\n }\n\n /// Constructs a new vector from the first, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxz(self) -> I32x4 {\n I32x4([self[0], self[3], self[0], self[2]])\n }\n\n /// Constructs a new vector from the second, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxz(self) -> I32x4 {\n I32x4([self[1], self[3], self[0], self[2]])\n }\n\n /// Constructs a new vector from the third, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxz(self) -> I32x4 {\n I32x4([self[2], self[3], self[0], self[2]])\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxz(self) -> I32x4 {\n I32x4([self[3], self[3], self[0], self[2]])\n }\n\n /// Constructs a new vector from the first, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyz(self) -> I32x4 {\n I32x4([self[0], self[0], self[1], self[2]])\n }\n\n /// Constructs a new vector from the second, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyz(self) -> I32x4 {\n I32x4([self[1], self[0], self[1], self[2]])\n }\n\n /// Constructs a new vector from the third, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyz(self) -> I32x4 {\n I32x4([self[2], self[0], self[1], self[2]])\n }\n\n /// Constructs a new vector from the fourth, first, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyz(self) -> I32x4 {\n I32x4([self[3], self[0], self[1], self[2]])\n }\n\n /// Constructs a new vector from the first, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyz(self) -> I32x4 {\n I32x4([self[0], self[1], self[1], self[2]])\n }\n\n /// Constructs a new vector from the second, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyz(self) -> I32x4 {\n I32x4([self[1], self[1], self[1], self[2]])\n }\n\n /// Constructs a new vector from the third, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyz(self) -> I32x4 {\n I32x4([self[2], self[1], self[1], self[2]])\n }\n\n /// Constructs a new vector from the fourth, second, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyz(self) -> I32x4 {\n I32x4([self[3], self[1], self[1], self[2]])\n }\n\n /// Constructs a new vector from the first, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyz(self) -> I32x4 {\n I32x4([self[0], self[2], self[1], self[2]])\n }\n\n /// Constructs a new vector from the second, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyz(self) -> I32x4 {\n I32x4([self[1], self[2], self[1], self[2]])\n }\n\n /// Constructs a new vector from the third, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyz(self) -> I32x4 {\n I32x4([self[2], self[2], self[1], self[2]])\n }\n\n /// Constructs a new vector from the fourth, third, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyz(self) -> I32x4 {\n I32x4([self[3], self[2], self[1], self[2]])\n }\n\n /// Constructs a new vector from the first, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyz(self) -> I32x4 {\n I32x4([self[0], self[3], self[1], self[2]])\n }\n\n /// Constructs a new vector from the second, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyz(self) -> I32x4 {\n I32x4([self[1], self[3], self[1], self[2]])\n }\n\n /// Constructs a new vector from the third, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyz(self) -> I32x4 {\n I32x4([self[2], self[3], self[1], self[2]])\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyz(self) -> I32x4 {\n I32x4([self[3], self[3], self[1], self[2]])\n }\n\n /// Constructs a new vector from the first, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzz(self) -> I32x4 {\n I32x4([self[0], self[0], self[2], self[2]])\n }\n\n /// Constructs a new vector from the second, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzz(self) -> I32x4 {\n I32x4([self[1], self[0], self[2], self[2]])\n }\n\n /// Constructs a new vector from the third, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzz(self) -> I32x4 {\n I32x4([self[2], self[0], self[2], self[2]])\n }\n\n /// Constructs a new vector from the fourth, first, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzz(self) -> I32x4 {\n I32x4([self[3], self[0], self[2], self[2]])\n }\n\n /// Constructs a new vector from the first, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzz(self) -> I32x4 {\n I32x4([self[0], self[1], self[2], self[2]])\n }\n\n /// Constructs a new vector from the second, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzz(self) -> I32x4 {\n I32x4([self[1], self[1], self[2], self[2]])\n }\n\n /// Constructs a new vector from the third, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzz(self) -> I32x4 {\n I32x4([self[2], self[1], self[2], self[2]])\n }\n\n /// Constructs a new vector from the fourth, second, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzz(self) -> I32x4 {\n I32x4([self[3], self[1], self[2], self[2]])\n }\n\n /// Constructs a new vector from the first, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzz(self) -> I32x4 {\n I32x4([self[0], self[2], self[2], self[2]])\n }\n\n /// Constructs a new vector from the second, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzz(self) -> I32x4 {\n I32x4([self[1], self[2], self[2], self[2]])\n }\n\n /// Constructs a new vector from the third, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzz(self) -> I32x4 {\n I32x4([self[2], self[2], self[2], self[2]])\n }\n\n /// Constructs a new vector from the fourth, third, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzz(self) -> I32x4 {\n I32x4([self[3], self[2], self[2], self[2]])\n }\n\n /// Constructs a new vector from the first, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzz(self) -> I32x4 {\n I32x4([self[0], self[3], self[2], self[2]])\n }\n\n /// Constructs a new vector from the second, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzz(self) -> I32x4 {\n I32x4([self[1], self[3], self[2], self[2]])\n }\n\n /// Constructs a new vector from the third, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzz(self) -> I32x4 {\n I32x4([self[2], self[3], self[2], self[2]])\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzz(self) -> I32x4 {\n I32x4([self[3], self[3], self[2], self[2]])\n }\n\n /// Constructs a new vector from the first, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxwz(self) -> I32x4 {\n I32x4([self[0], self[0], self[3], self[2]])\n }\n\n /// Constructs a new vector from the second, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxwz(self) -> I32x4 {\n I32x4([self[1], self[0], self[3], self[2]])\n }\n\n /// Constructs a new vector from the third, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxwz(self) -> I32x4 {\n I32x4([self[2], self[0], self[3], self[2]])\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxwz(self) -> I32x4 {\n I32x4([self[3], self[0], self[3], self[2]])\n }\n\n /// Constructs a new vector from the first, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xywz(self) -> I32x4 {\n I32x4([self[0], self[1], self[3], self[2]])\n }\n\n /// Constructs a new vector from the second, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yywz(self) -> I32x4 {\n I32x4([self[1], self[1], self[3], self[2]])\n }\n\n /// Constructs a new vector from the third, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zywz(self) -> I32x4 {\n I32x4([self[2], self[1], self[3], self[2]])\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wywz(self) -> I32x4 {\n I32x4([self[3], self[1], self[3], self[2]])\n }\n\n /// Constructs a new vector from the first, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzwz(self) -> I32x4 {\n I32x4([self[0], self[2], self[3], self[2]])\n }\n\n /// Constructs a new vector from the second, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzwz(self) -> I32x4 {\n I32x4([self[1], self[2], self[3], self[2]])\n }\n\n /// Constructs a new vector from the third, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzwz(self) -> I32x4 {\n I32x4([self[2], self[2], self[3], self[2]])\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzwz(self) -> I32x4 {\n I32x4([self[3], self[2], self[3], self[2]])\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwwz(self) -> I32x4 {\n I32x4([self[0], self[3], self[3], self[2]])\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywwz(self) -> I32x4 {\n I32x4([self[1], self[3], self[3], self[2]])\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwwz(self) -> I32x4 {\n I32x4([self[2], self[3], self[3], self[2]])\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and third\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwwz(self) -> I32x4 {\n I32x4([self[3], self[3], self[3], self[2]])\n }\n\n /// Constructs a new vector from the first, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxxw(self) -> I32x4 {\n I32x4([self[0], self[0], self[0], self[3]])\n }\n\n /// Constructs a new vector from the second, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxxw(self) -> I32x4 {\n I32x4([self[1], self[0], self[0], self[3]])\n }\n\n /// Constructs a new vector from the third, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxxw(self) -> I32x4 {\n I32x4([self[2], self[0], self[0], self[3]])\n }\n\n /// Constructs a new vector from the fourth, first, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxxw(self) -> I32x4 {\n I32x4([self[3], self[0], self[0], self[3]])\n }\n\n /// Constructs a new vector from the first, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyxw(self) -> I32x4 {\n I32x4([self[0], self[1], self[0], self[3]])\n }\n\n /// Constructs a new vector from the second, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyxw(self) -> I32x4 {\n I32x4([self[1], self[1], self[0], self[3]])\n }\n\n /// Constructs a new vector from the third, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyxw(self) -> I32x4 {\n I32x4([self[2], self[1], self[0], self[3]])\n }\n\n /// Constructs a new vector from the fourth, second, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyxw(self) -> I32x4 {\n I32x4([self[3], self[1], self[0], self[3]])\n }\n\n /// Constructs a new vector from the first, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzxw(self) -> I32x4 {\n I32x4([self[0], self[2], self[0], self[3]])\n }\n\n /// Constructs a new vector from the second, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzxw(self) -> I32x4 {\n I32x4([self[1], self[2], self[0], self[3]])\n }\n\n /// Constructs a new vector from the third, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzxw(self) -> I32x4 {\n I32x4([self[2], self[2], self[0], self[3]])\n }\n\n /// Constructs a new vector from the fourth, third, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzxw(self) -> I32x4 {\n I32x4([self[3], self[2], self[0], self[3]])\n }\n\n /// Constructs a new vector from the first, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwxw(self) -> I32x4 {\n I32x4([self[0], self[3], self[0], self[3]])\n }\n\n /// Constructs a new vector from the second, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywxw(self) -> I32x4 {\n I32x4([self[1], self[3], self[0], self[3]])\n }\n\n /// Constructs a new vector from the third, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwxw(self) -> I32x4 {\n I32x4([self[2], self[3], self[0], self[3]])\n }\n\n /// Constructs a new vector from the fourth, fourth, first, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwxw(self) -> I32x4 {\n I32x4([self[3], self[3], self[0], self[3]])\n }\n\n /// Constructs a new vector from the first, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxyw(self) -> I32x4 {\n I32x4([self[0], self[0], self[1], self[3]])\n }\n\n /// Constructs a new vector from the second, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxyw(self) -> I32x4 {\n I32x4([self[1], self[0], self[1], self[3]])\n }\n\n /// Constructs a new vector from the third, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxyw(self) -> I32x4 {\n I32x4([self[2], self[0], self[1], self[3]])\n }\n\n /// Constructs a new vector from the fourth, first, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxyw(self) -> I32x4 {\n I32x4([self[3], self[0], self[1], self[3]])\n }\n\n /// Constructs a new vector from the first, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyyw(self) -> I32x4 {\n I32x4([self[0], self[1], self[1], self[3]])\n }\n\n /// Constructs a new vector from the second, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyyw(self) -> I32x4 {\n I32x4([self[1], self[1], self[1], self[3]])\n }\n\n /// Constructs a new vector from the third, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyyw(self) -> I32x4 {\n I32x4([self[2], self[1], self[1], self[3]])\n }\n\n /// Constructs a new vector from the fourth, second, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyyw(self) -> I32x4 {\n I32x4([self[3], self[1], self[1], self[3]])\n }\n\n /// Constructs a new vector from the first, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzyw(self) -> I32x4 {\n I32x4([self[0], self[2], self[1], self[3]])\n }\n\n /// Constructs a new vector from the second, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzyw(self) -> I32x4 {\n I32x4([self[1], self[2], self[1], self[3]])\n }\n\n /// Constructs a new vector from the third, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzyw(self) -> I32x4 {\n I32x4([self[2], self[2], self[1], self[3]])\n }\n\n /// Constructs a new vector from the fourth, third, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzyw(self) -> I32x4 {\n I32x4([self[3], self[2], self[1], self[3]])\n }\n\n /// Constructs a new vector from the first, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwyw(self) -> I32x4 {\n I32x4([self[0], self[3], self[1], self[3]])\n }\n\n /// Constructs a new vector from the second, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywyw(self) -> I32x4 {\n I32x4([self[1], self[3], self[1], self[3]])\n }\n\n /// Constructs a new vector from the third, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwyw(self) -> I32x4 {\n I32x4([self[2], self[3], self[1], self[3]])\n }\n\n /// Constructs a new vector from the fourth, fourth, second, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwyw(self) -> I32x4 {\n I32x4([self[3], self[3], self[1], self[3]])\n }\n\n /// Constructs a new vector from the first, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxzw(self) -> I32x4 {\n I32x4([self[0], self[0], self[2], self[3]])\n }\n\n /// Constructs a new vector from the second, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxzw(self) -> I32x4 {\n I32x4([self[1], self[0], self[2], self[3]])\n }\n\n /// Constructs a new vector from the third, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxzw(self) -> I32x4 {\n I32x4([self[2], self[0], self[2], self[3]])\n }\n\n /// Constructs a new vector from the fourth, first, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxzw(self) -> I32x4 {\n I32x4([self[3], self[0], self[2], self[3]])\n }\n\n /// Constructs a new vector from the first, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyzw(self) -> I32x4 {\n I32x4([self[0], self[1], self[2], self[3]])\n }\n\n /// Constructs a new vector from the second, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyzw(self) -> I32x4 {\n I32x4([self[1], self[1], self[2], self[3]])\n }\n\n /// Constructs a new vector from the third, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyzw(self) -> I32x4 {\n I32x4([self[2], self[1], self[2], self[3]])\n }\n\n /// Constructs a new vector from the fourth, second, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyzw(self) -> I32x4 {\n I32x4([self[3], self[1], self[2], self[3]])\n }\n\n /// Constructs a new vector from the first, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzzw(self) -> I32x4 {\n I32x4([self[0], self[2], self[2], self[3]])\n }\n\n /// Constructs a new vector from the second, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzzw(self) -> I32x4 {\n I32x4([self[1], self[2], self[2], self[3]])\n }\n\n /// Constructs a new vector from the third, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzzw(self) -> I32x4 {\n I32x4([self[2], self[2], self[2], self[3]])\n }\n\n /// Constructs a new vector from the fourth, third, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzzw(self) -> I32x4 {\n I32x4([self[3], self[2], self[2], self[3]])\n }\n\n /// Constructs a new vector from the first, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwzw(self) -> I32x4 {\n I32x4([self[0], self[3], self[2], self[3]])\n }\n\n /// Constructs a new vector from the second, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywzw(self) -> I32x4 {\n I32x4([self[1], self[3], self[2], self[3]])\n }\n\n /// Constructs a new vector from the third, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwzw(self) -> I32x4 {\n I32x4([self[2], self[3], self[2], self[3]])\n }\n\n /// Constructs a new vector from the fourth, fourth, third, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwzw(self) -> I32x4 {\n I32x4([self[3], self[3], self[2], self[3]])\n }\n\n /// Constructs a new vector from the first, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xxww(self) -> I32x4 {\n I32x4([self[0], self[0], self[3], self[3]])\n }\n\n /// Constructs a new vector from the second, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yxww(self) -> I32x4 {\n I32x4([self[1], self[0], self[3], self[3]])\n }\n\n /// Constructs a new vector from the third, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zxww(self) -> I32x4 {\n I32x4([self[2], self[0], self[3], self[3]])\n }\n\n /// Constructs a new vector from the fourth, first, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wxww(self) -> I32x4 {\n I32x4([self[3], self[0], self[3], self[3]])\n }\n\n /// Constructs a new vector from the first, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xyww(self) -> I32x4 {\n I32x4([self[0], self[1], self[3], self[3]])\n }\n\n /// Constructs a new vector from the second, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yyww(self) -> I32x4 {\n I32x4([self[1], self[1], self[3], self[3]])\n }\n\n /// Constructs a new vector from the third, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zyww(self) -> I32x4 {\n I32x4([self[2], self[1], self[3], self[3]])\n }\n\n /// Constructs a new vector from the fourth, second, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wyww(self) -> I32x4 {\n I32x4([self[3], self[1], self[3], self[3]])\n }\n\n /// Constructs a new vector from the first, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xzww(self) -> I32x4 {\n I32x4([self[0], self[2], self[3], self[3]])\n }\n\n /// Constructs a new vector from the second, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn yzww(self) -> I32x4 {\n I32x4([self[1], self[2], self[3], self[3]])\n }\n\n /// Constructs a new vector from the third, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zzww(self) -> I32x4 {\n I32x4([self[2], self[2], self[3], self[3]])\n }\n\n /// Constructs a new vector from the fourth, third, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wzww(self) -> I32x4 {\n I32x4([self[3], self[2], self[3], self[3]])\n }\n\n /// Constructs a new vector from the first, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn xwww(self) -> I32x4 {\n I32x4([self[0], self[3], self[3], self[3]])\n }\n\n /// Constructs a new vector from the second, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn ywww(self) -> I32x4 {\n I32x4([self[1], self[3], self[3], self[3]])\n }\n\n /// Constructs a new vector from the third, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn zwww(self) -> I32x4 {\n I32x4([self[2], self[3], self[3], self[3]])\n }\n\n /// Constructs a new vector from the fourth, fourth, fourth, and fourth\n /// lanes in this vector, respectively.\n #[inline]\n pub fn wwww(self) -> I32x4 {\n I32x4([self[3], self[3], self[3], self[3]])\n }\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/test.rs", "sha256": "62e1159ff2977b0e3ddcc302b1c210105a6f1c087a089fd1fffeaef1814e50f6", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/test.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::default::{F32x4, I32x4, U32x4};\nuse crate::scalar::F32x4 as F32x4S;\n\n// F32x4\n\n#[test]\nfn test_f32x4_constructors() {\n let a = F32x4::new(1.0, 2.0, 3.0, 4.0);\n assert_eq!((a[0], a[1], a[2], a[3]), (1.0, 2.0, 3.0, 4.0));\n let b = F32x4::splat(10.0);\n assert_eq!(b, F32x4::new(10.0, 10.0, 10.0, 10.0));\n}\n\n#[test]\nfn test_f32x4_accessors_and_mutators() {\n let a = F32x4::new(5.0, 6.0, 7.0, 8.0);\n assert_eq!((a.x(), a.y(), a.z(), a.w()), (5.0, 6.0, 7.0, 8.0));\n let mut b = F32x4::new(10.0, 11.0, 12.0, 13.0);\n b.set_x(20.0);\n b.set_y(30.0);\n b.set_z(40.0);\n b.set_w(50.0);\n assert_eq!(b, F32x4::new(20.0, 30.0, 40.0, 50.0));\n}\n\n#[test]\nfn test_f32x4_basic_ops() {\n let a = F32x4::new(1.0, 3.0, 5.0, 7.0);\n let b = F32x4::new(2.0, 2.0, 6.0, 6.0);\n let approx_recip = a.approx_recip();\n\n // The greatest diff we observe between the correct answer to the reciprocal and the answer that\n // llvm gives us is 1.0 vs 0.9980469, so we need to have the allowable diff be 0.002 so that it\n // passes. It's approximate, so it's fine to be a bit off.\n const ALLOWABLE_RECIP_DIFF: f32 = 0.002;\n assert!((approx_recip[0] - (1. / 1.)).abs() < ALLOWABLE_RECIP_DIFF);\n assert!((approx_recip[1] - (1. / 3.)).abs() < ALLOWABLE_RECIP_DIFF);\n assert!((approx_recip[2] - (1. / 5.)).abs() < ALLOWABLE_RECIP_DIFF);\n assert!((approx_recip[3] - (1. / 7.)).abs() < ALLOWABLE_RECIP_DIFF);\n\n assert_eq!(a.min(b), F32x4::new(1.0, 2.0, 5.0, 6.0));\n assert_eq!(a.max(b), F32x4::new(2.0, 3.0, 6.0, 7.0));\n let c = F32x4::new(-1.0, 1.3, -20.0, 3.6);\n assert_eq!(c.clamp(a, b), F32x4::new(1.0, 2.0, 5.0, 6.0));\n assert_eq!(c.abs(), F32x4::new(1.0, 1.3, 20.0, 3.6));\n assert_eq!(c.floor(), F32x4::new(-1.0, 1.0, -20.0, 3.0));\n assert_eq!(c.ceil(), F32x4::new(-1.0, 2.0, -20.0, 4.0));\n assert_eq!(c.to_i32x4().to_f32x4(), F32x4::new(-1.0, 1.0, -20.0, 4.0));\n let d = F32x4::new(1.0, 2.0, 3.0, 4.0);\n assert_eq!(d.sqrt(), F32x4::new(1.0, 1.4142135, 1.7320508, 2.0));\n}\n\n#[test]\nfn test_f32x4_packed_comparisons() {\n let a = F32x4::new(7.0, 3.0, 6.0, -2.0);\n let b = F32x4::new(10.0, 3.0, 5.0, -2.0);\n assert_eq!(a.packed_eq(b), U32x4::new(0, !0, 0, !0));\n assert_eq!(a.packed_gt(b), U32x4::new(0, 0, !0, 0));\n assert_eq!(a.packed_le(b), U32x4::new(!0, !0, 0, !0));\n}\n\n#[test]\nfn test_f32x4_swizzles() {\n let a = F32x4::new(1.0, 2.0, 3.0, 4.0);\n assert_eq!(a.xxxx(), F32x4::splat(1.0));\n assert_eq!(a.yyyy(), F32x4::splat(2.0));\n assert_eq!(a.zzzz(), F32x4::splat(3.0));\n assert_eq!(a.wwww(), F32x4::splat(4.0));\n\n assert_eq!(a.yxxx(), F32x4::new(2.0, 1.0, 1.0, 1.0));\n assert_eq!(a.zxxx(), F32x4::new(3.0, 1.0, 1.0, 1.0));\n assert_eq!(a.wxxx(), F32x4::new(4.0, 1.0, 1.0, 1.0));\n assert_eq!(a.xyxx(), F32x4::new(1.0, 2.0, 1.0, 1.0));\n assert_eq!(a.yyxx(), F32x4::new(2.0, 2.0, 1.0, 1.0));\n assert_eq!(a.zyxx(), F32x4::new(3.0, 2.0, 1.0, 1.0));\n assert_eq!(a.wyxx(), F32x4::new(4.0, 2.0, 1.0, 1.0));\n assert_eq!(a.xzxx(), F32x4::new(1.0, 3.0, 1.0, 1.0));\n assert_eq!(a.yzxx(), F32x4::new(2.0, 3.0, 1.0, 1.0));\n assert_eq!(a.zzxx(), F32x4::new(3.0, 3.0, 1.0, 1.0));\n assert_eq!(a.wzxx(), F32x4::new(4.0, 3.0, 1.0, 1.0));\n assert_eq!(a.xwxx(), F32x4::new(1.0, 4.0, 1.0, 1.0));\n assert_eq!(a.ywxx(), F32x4::new(2.0, 4.0, 1.0, 1.0));\n assert_eq!(a.zwxx(), F32x4::new(3.0, 4.0, 1.0, 1.0));\n assert_eq!(a.wwxx(), F32x4::new(4.0, 4.0, 1.0, 1.0));\n assert_eq!(a.xxyx(), F32x4::new(1.0, 1.0, 2.0, 1.0));\n assert_eq!(a.yxyx(), F32x4::new(2.0, 1.0, 2.0, 1.0));\n assert_eq!(a.zxyx(), F32x4::new(3.0, 1.0, 2.0, 1.0));\n assert_eq!(a.wxyx(), F32x4::new(4.0, 1.0, 2.0, 1.0));\n assert_eq!(a.xyyx(), F32x4::new(1.0, 2.0, 2.0, 1.0));\n assert_eq!(a.yyyx(), F32x4::new(2.0, 2.0, 2.0, 1.0));\n assert_eq!(a.zyyx(), F32x4::new(3.0, 2.0, 2.0, 1.0));\n assert_eq!(a.wyyx(), F32x4::new(4.0, 2.0, 2.0, 1.0));\n assert_eq!(a.xzyx(), F32x4::new(1.0, 3.0, 2.0, 1.0));\n assert_eq!(a.yzyx(), F32x4::new(2.0, 3.0, 2.0, 1.0));\n assert_eq!(a.zzyx(), F32x4::new(3.0, 3.0, 2.0, 1.0));\n assert_eq!(a.wzyx(), F32x4::new(4.0, 3.0, 2.0, 1.0));\n assert_eq!(a.xwyx(), F32x4::new(1.0, 4.0, 2.0, 1.0));\n assert_eq!(a.ywyx(), F32x4::new(2.0, 4.0, 2.0, 1.0));\n assert_eq!(a.zwyx(), F32x4::new(3.0, 4.0, 2.0, 1.0));\n assert_eq!(a.wwyx(), F32x4::new(4.0, 4.0, 2.0, 1.0));\n assert_eq!(a.xxzx(), F32x4::new(1.0, 1.0, 3.0, 1.0));\n assert_eq!(a.yxzx(), F32x4::new(2.0, 1.0, 3.0, 1.0));\n assert_eq!(a.zxzx(), F32x4::new(3.0, 1.0, 3.0, 1.0));\n assert_eq!(a.wxzx(), F32x4::new(4.0, 1.0, 3.0, 1.0));\n assert_eq!(a.xyzx(), F32x4::new(1.0, 2.0, 3.0, 1.0));\n assert_eq!(a.yyzx(), F32x4::new(2.0, 2.0, 3.0, 1.0));\n assert_eq!(a.zyzx(), F32x4::new(3.0, 2.0, 3.0, 1.0));\n assert_eq!(a.wyzx(), F32x4::new(4.0, 2.0, 3.0, 1.0));\n assert_eq!(a.xzzx(), F32x4::new(1.0, 3.0, 3.0, 1.0));\n assert_eq!(a.yzzx(), F32x4::new(2.0, 3.0, 3.0, 1.0));\n assert_eq!(a.zzzx(), F32x4::new(3.0, 3.0, 3.0, 1.0));\n assert_eq!(a.wzzx(), F32x4::new(4.0, 3.0, 3.0, 1.0));\n assert_eq!(a.xwzx(), F32x4::new(1.0, 4.0, 3.0, 1.0));\n assert_eq!(a.ywzx(), F32x4::new(2.0, 4.0, 3.0, 1.0));\n assert_eq!(a.zwzx(), F32x4::new(3.0, 4.0, 3.0, 1.0));\n assert_eq!(a.wwzx(), F32x4::new(4.0, 4.0, 3.0, 1.0));\n assert_eq!(a.xxwx(), F32x4::new(1.0, 1.0, 4.0, 1.0));\n assert_eq!(a.yxwx(), F32x4::new(2.0, 1.0, 4.0, 1.0));\n assert_eq!(a.zxwx(), F32x4::new(3.0, 1.0, 4.0, 1.0));\n assert_eq!(a.wxwx(), F32x4::new(4.0, 1.0, 4.0, 1.0));\n assert_eq!(a.xywx(), F32x4::new(1.0, 2.0, 4.0, 1.0));\n assert_eq!(a.yywx(), F32x4::new(2.0, 2.0, 4.0, 1.0));\n assert_eq!(a.zywx(), F32x4::new(3.0, 2.0, 4.0, 1.0));\n assert_eq!(a.wywx(), F32x4::new(4.0, 2.0, 4.0, 1.0));\n assert_eq!(a.xzwx(), F32x4::new(1.0, 3.0, 4.0, 1.0));\n assert_eq!(a.yzwx(), F32x4::new(2.0, 3.0, 4.0, 1.0));\n assert_eq!(a.zzwx(), F32x4::new(3.0, 3.0, 4.0, 1.0));\n assert_eq!(a.wzwx(), F32x4::new(4.0, 3.0, 4.0, 1.0));\n assert_eq!(a.xwwx(), F32x4::new(1.0, 4.0, 4.0, 1.0));\n assert_eq!(a.ywwx(), F32x4::new(2.0, 4.0, 4.0, 1.0));\n assert_eq!(a.zwwx(), F32x4::new(3.0, 4.0, 4.0, 1.0));\n assert_eq!(a.wwwx(), F32x4::new(4.0, 4.0, 4.0, 1.0));\n\n assert_eq!(a.xxxy(), F32x4::new(1.0, 1.0, 1.0, 2.0));\n assert_eq!(a.yxxy(), F32x4::new(2.0, 1.0, 1.0, 2.0));\n assert_eq!(a.zxxy(), F32x4::new(3.0, 1.0, 1.0, 2.0));\n assert_eq!(a.wxxy(), F32x4::new(4.0, 1.0, 1.0, 2.0));\n assert_eq!(a.xyxy(), F32x4::new(1.0, 2.0, 1.0, 2.0));\n assert_eq!(a.yyxy(), F32x4::new(2.0, 2.0, 1.0, 2.0));\n assert_eq!(a.zyxy(), F32x4::new(3.0, 2.0, 1.0, 2.0));\n assert_eq!(a.wyxy(), F32x4::new(4.0, 2.0, 1.0, 2.0));\n assert_eq!(a.xzxy(), F32x4::new(1.0, 3.0, 1.0, 2.0));\n assert_eq!(a.yzxy(), F32x4::new(2.0, 3.0, 1.0, 2.0));\n assert_eq!(a.zzxy(), F32x4::new(3.0, 3.0, 1.0, 2.0));\n assert_eq!(a.wzxy(), F32x4::new(4.0, 3.0, 1.0, 2.0));\n assert_eq!(a.xwxy(), F32x4::new(1.0, 4.0, 1.0, 2.0));\n assert_eq!(a.ywxy(), F32x4::new(2.0, 4.0, 1.0, 2.0));\n assert_eq!(a.zwxy(), F32x4::new(3.0, 4.0, 1.0, 2.0));\n assert_eq!(a.wwxy(), F32x4::new(4.0, 4.0, 1.0, 2.0));\n assert_eq!(a.xxyy(), F32x4::new(1.0, 1.0, 2.0, 2.0));\n assert_eq!(a.yxyy(), F32x4::new(2.0, 1.0, 2.0, 2.0));\n assert_eq!(a.zxyy(), F32x4::new(3.0, 1.0, 2.0, 2.0));\n assert_eq!(a.wxyy(), F32x4::new(4.0, 1.0, 2.0, 2.0));\n assert_eq!(a.xyyy(), F32x4::new(1.0, 2.0, 2.0, 2.0));\n assert_eq!(a.zyyy(), F32x4::new(3.0, 2.0, 2.0, 2.0));\n assert_eq!(a.wyyy(), F32x4::new(4.0, 2.0, 2.0, 2.0));\n assert_eq!(a.xzyy(), F32x4::new(1.0, 3.0, 2.0, 2.0));\n assert_eq!(a.yzyy(), F32x4::new(2.0, 3.0, 2.0, 2.0));\n assert_eq!(a.zzyy(), F32x4::new(3.0, 3.0, 2.0, 2.0));\n assert_eq!(a.wzyy(), F32x4::new(4.0, 3.0, 2.0, 2.0));\n assert_eq!(a.xwyy(), F32x4::new(1.0, 4.0, 2.0, 2.0));\n assert_eq!(a.ywyy(), F32x4::new(2.0, 4.0, 2.0, 2.0));\n assert_eq!(a.zwyy(), F32x4::new(3.0, 4.0, 2.0, 2.0));\n assert_eq!(a.wwyy(), F32x4::new(4.0, 4.0, 2.0, 2.0));\n assert_eq!(a.xxzy(), F32x4::new(1.0, 1.0, 3.0, 2.0));\n assert_eq!(a.yxzy(), F32x4::new(2.0, 1.0, 3.0, 2.0));\n assert_eq!(a.zxzy(), F32x4::new(3.0, 1.0, 3.0, 2.0));\n assert_eq!(a.wxzy(), F32x4::new(4.0, 1.0, 3.0, 2.0));\n assert_eq!(a.xyzy(), F32x4::new(1.0, 2.0, 3.0, 2.0));\n assert_eq!(a.yyzy(), F32x4::new(2.0, 2.0, 3.0, 2.0));\n assert_eq!(a.zyzy(), F32x4::new(3.0, 2.0, 3.0, 2.0));\n assert_eq!(a.wyzy(), F32x4::new(4.0, 2.0, 3.0, 2.0));\n assert_eq!(a.xzzy(), F32x4::new(1.0, 3.0, 3.0, 2.0));\n assert_eq!(a.yzzy(), F32x4::new(2.0, 3.0, 3.0, 2.0));\n assert_eq!(a.zzzy(), F32x4::new(3.0, 3.0, 3.0, 2.0));\n assert_eq!(a.wzzy(), F32x4::new(4.0, 3.0, 3.0, 2.0));\n assert_eq!(a.xwzy(), F32x4::new(1.0, 4.0, 3.0, 2.0));\n assert_eq!(a.ywzy(), F32x4::new(2.0, 4.0, 3.0, 2.0));\n assert_eq!(a.zwzy(), F32x4::new(3.0, 4.0, 3.0, 2.0));\n assert_eq!(a.wwzy(), F32x4::new(4.0, 4.0, 3.0, 2.0));\n assert_eq!(a.xxwy(), F32x4::new(1.0, 1.0, 4.0, 2.0));\n assert_eq!(a.yxwy(), F32x4::new(2.0, 1.0, 4.0, 2.0));\n assert_eq!(a.zxwy(), F32x4::new(3.0, 1.0, 4.0, 2.0));\n assert_eq!(a.wxwy(), F32x4::new(4.0, 1.0, 4.0, 2.0));\n assert_eq!(a.xywy(), F32x4::new(1.0, 2.0, 4.0, 2.0));\n assert_eq!(a.yywy(), F32x4::new(2.0, 2.0, 4.0, 2.0));\n assert_eq!(a.zywy(), F32x4::new(3.0, 2.0, 4.0, 2.0));\n assert_eq!(a.wywy(), F32x4::new(4.0, 2.0, 4.0, 2.0));\n assert_eq!(a.xzwy(), F32x4::new(1.0, 3.0, 4.0, 2.0));\n assert_eq!(a.yzwy(), F32x4::new(2.0, 3.0, 4.0, 2.0));\n assert_eq!(a.zzwy(), F32x4::new(3.0, 3.0, 4.0, 2.0));\n assert_eq!(a.wzwy(), F32x4::new(4.0, 3.0, 4.0, 2.0));\n assert_eq!(a.xwwy(), F32x4::new(1.0, 4.0, 4.0, 2.0));\n assert_eq!(a.ywwy(), F32x4::new(2.0, 4.0, 4.0, 2.0));\n assert_eq!(a.zwwy(), F32x4::new(3.0, 4.0, 4.0, 2.0));\n assert_eq!(a.wwwy(), F32x4::new(4.0, 4.0, 4.0, 2.0));\n\n assert_eq!(a.xxxw(), F32x4::new(1.0, 1.0, 1.0, 4.0));\n assert_eq!(a.yxxw(), F32x4::new(2.0, 1.0, 1.0, 4.0));\n assert_eq!(a.zxxw(), F32x4::new(3.0, 1.0, 1.0, 4.0));\n assert_eq!(a.wxxw(), F32x4::new(4.0, 1.0, 1.0, 4.0));\n assert_eq!(a.xyxw(), F32x4::new(1.0, 2.0, 1.0, 4.0));\n assert_eq!(a.yyxw(), F32x4::new(2.0, 2.0, 1.0, 4.0));\n assert_eq!(a.zyxw(), F32x4::new(3.0, 2.0, 1.0, 4.0));\n assert_eq!(a.wyxw(), F32x4::new(4.0, 2.0, 1.0, 4.0));\n assert_eq!(a.xzxw(), F32x4::new(1.0, 3.0, 1.0, 4.0));\n assert_eq!(a.yzxw(), F32x4::new(2.0, 3.0, 1.0, 4.0));\n assert_eq!(a.zzxw(), F32x4::new(3.0, 3.0, 1.0, 4.0));\n assert_eq!(a.wzxw(), F32x4::new(4.0, 3.0, 1.0, 4.0));\n assert_eq!(a.xwxw(), F32x4::new(1.0, 4.0, 1.0, 4.0));\n assert_eq!(a.ywxw(), F32x4::new(2.0, 4.0, 1.0, 4.0));\n assert_eq!(a.zwxw(), F32x4::new(3.0, 4.0, 1.0, 4.0));\n assert_eq!(a.wwxw(), F32x4::new(4.0, 4.0, 1.0, 4.0));\n assert_eq!(a.xxyw(), F32x4::new(1.0, 1.0, 2.0, 4.0));\n assert_eq!(a.yxyw(), F32x4::new(2.0, 1.0, 2.0, 4.0));\n assert_eq!(a.zxyw(), F32x4::new(3.0, 1.0, 2.0, 4.0));\n assert_eq!(a.wxyw(), F32x4::new(4.0, 1.0, 2.0, 4.0));\n assert_eq!(a.xyyw(), F32x4::new(1.0, 2.0, 2.0, 4.0));\n assert_eq!(a.yyyw(), F32x4::new(2.0, 2.0, 2.0, 4.0));\n assert_eq!(a.zyyw(), F32x4::new(3.0, 2.0, 2.0, 4.0));\n assert_eq!(a.wyyw(), F32x4::new(4.0, 2.0, 2.0, 4.0));\n assert_eq!(a.xzyw(), F32x4::new(1.0, 3.0, 2.0, 4.0));\n assert_eq!(a.yzyw(), F32x4::new(2.0, 3.0, 2.0, 4.0));\n assert_eq!(a.zzyw(), F32x4::new(3.0, 3.0, 2.0, 4.0));\n assert_eq!(a.wzyw(), F32x4::new(4.0, 3.0, 2.0, 4.0));\n assert_eq!(a.xwyw(), F32x4::new(1.0, 4.0, 2.0, 4.0));\n assert_eq!(a.ywyw(), F32x4::new(2.0, 4.0, 2.0, 4.0));\n assert_eq!(a.zwyw(), F32x4::new(3.0, 4.0, 2.0, 4.0));\n assert_eq!(a.wwyw(), F32x4::new(4.0, 4.0, 2.0, 4.0));\n assert_eq!(a.xxzw(), F32x4::new(1.0, 1.0, 3.0, 4.0));\n assert_eq!(a.yxzw(), F32x4::new(2.0, 1.0, 3.0, 4.0));\n assert_eq!(a.zxzw(), F32x4::new(3.0, 1.0, 3.0, 4.0));\n assert_eq!(a.wxzw(), F32x4::new(4.0, 1.0, 3.0, 4.0));\n assert_eq!(a.xyzw(), F32x4::new(1.0, 2.0, 3.0, 4.0));\n assert_eq!(a.yyzw(), F32x4::new(2.0, 2.0, 3.0, 4.0));\n assert_eq!(a.zyzw(), F32x4::new(3.0, 2.0, 3.0, 4.0));\n assert_eq!(a.wyzw(), F32x4::new(4.0, 2.0, 3.0, 4.0));\n assert_eq!(a.xzzw(), F32x4::new(1.0, 3.0, 3.0, 4.0));\n assert_eq!(a.yzzw(), F32x4::new(2.0, 3.0, 3.0, 4.0));\n assert_eq!(a.zzzw(), F32x4::new(3.0, 3.0, 3.0, 4.0));\n assert_eq!(a.wzzw(), F32x4::new(4.0, 3.0, 3.0, 4.0));\n assert_eq!(a.xwzw(), F32x4::new(1.0, 4.0, 3.0, 4.0));\n assert_eq!(a.ywzw(), F32x4::new(2.0, 4.0, 3.0, 4.0));\n assert_eq!(a.zwzw(), F32x4::new(3.0, 4.0, 3.0, 4.0));\n assert_eq!(a.wwzw(), F32x4::new(4.0, 4.0, 3.0, 4.0));\n assert_eq!(a.xxww(), F32x4::new(1.0, 1.0, 4.0, 4.0));\n assert_eq!(a.yxww(), F32x4::new(2.0, 1.0, 4.0, 4.0));\n assert_eq!(a.zxww(), F32x4::new(3.0, 1.0, 4.0, 4.0));\n assert_eq!(a.wxww(), F32x4::new(4.0, 1.0, 4.0, 4.0));\n assert_eq!(a.xyww(), F32x4::new(1.0, 2.0, 4.0, 4.0));\n assert_eq!(a.yyww(), F32x4::new(2.0, 2.0, 4.0, 4.0));\n assert_eq!(a.zyww(), F32x4::new(3.0, 2.0, 4.0, 4.0));\n assert_eq!(a.wzww(), F32x4::new(4.0, 3.0, 4.0, 4.0));\n assert_eq!(a.xzww(), F32x4::new(1.0, 3.0, 4.0, 4.0));\n assert_eq!(a.yzww(), F32x4::new(2.0, 3.0, 4.0, 4.0));\n assert_eq!(a.zzww(), F32x4::new(3.0, 3.0, 4.0, 4.0));\n assert_eq!(a.wzww(), F32x4::new(4.0, 3.0, 4.0, 4.0));\n assert_eq!(a.xwww(), F32x4::new(1.0, 4.0, 4.0, 4.0));\n assert_eq!(a.ywww(), F32x4::new(2.0, 4.0, 4.0, 4.0));\n assert_eq!(a.zwww(), F32x4::new(3.0, 4.0, 4.0, 4.0));\n\n assert_eq!(a.xxxz(), F32x4::new(1.0, 1.0, 1.0, 3.0));\n assert_eq!(a.yxxz(), F32x4::new(2.0, 1.0, 1.0, 3.0));\n assert_eq!(a.zxxz(), F32x4::new(3.0, 1.0, 1.0, 3.0));\n assert_eq!(a.wxxz(), F32x4::new(4.0, 1.0, 1.0, 3.0));\n assert_eq!(a.xyxz(), F32x4::new(1.0, 2.0, 1.0, 3.0));\n assert_eq!(a.yyxz(), F32x4::new(2.0, 2.0, 1.0, 3.0));\n assert_eq!(a.zyxz(), F32x4::new(3.0, 2.0, 1.0, 3.0));\n assert_eq!(a.wyxz(), F32x4::new(4.0, 2.0, 1.0, 3.0));\n assert_eq!(a.xzxz(), F32x4::new(1.0, 3.0, 1.0, 3.0));\n assert_eq!(a.yzxz(), F32x4::new(2.0, 3.0, 1.0, 3.0));\n assert_eq!(a.zzxz(), F32x4::new(3.0, 3.0, 1.0, 3.0));\n assert_eq!(a.wzxz(), F32x4::new(4.0, 3.0, 1.0, 3.0));\n assert_eq!(a.xwxz(), F32x4::new(1.0, 4.0, 1.0, 3.0));\n assert_eq!(a.ywxz(), F32x4::new(2.0, 4.0, 1.0, 3.0));\n assert_eq!(a.zwxz(), F32x4::new(3.0, 4.0, 1.0, 3.0));\n assert_eq!(a.wwxz(), F32x4::new(4.0, 4.0, 1.0, 3.0));\n assert_eq!(a.xxyz(), F32x4::new(1.0, 1.0, 2.0, 3.0));\n assert_eq!(a.yxyz(), F32x4::new(2.0, 1.0, 2.0, 3.0));\n assert_eq!(a.zxyz(), F32x4::new(3.0, 1.0, 2.0, 3.0));\n assert_eq!(a.wxyz(), F32x4::new(4.0, 1.0, 2.0, 3.0));\n assert_eq!(a.xyyz(), F32x4::new(1.0, 2.0, 2.0, 3.0));\n assert_eq!(a.yyyz(), F32x4::new(2.0, 2.0, 2.0, 3.0));\n assert_eq!(a.zyyz(), F32x4::new(3.0, 2.0, 2.0, 3.0));\n assert_eq!(a.wyyz(), F32x4::new(4.0, 2.0, 2.0, 3.0));\n assert_eq!(a.xzyz(), F32x4::new(1.0, 3.0, 2.0, 3.0));\n assert_eq!(a.yzyz(), F32x4::new(2.0, 3.0, 2.0, 3.0));\n assert_eq!(a.zzyz(), F32x4::new(3.0, 3.0, 2.0, 3.0));\n assert_eq!(a.wzyz(), F32x4::new(4.0, 3.0, 2.0, 3.0));\n assert_eq!(a.xwyz(), F32x4::new(1.0, 4.0, 2.0, 3.0));\n assert_eq!(a.ywyz(), F32x4::new(2.0, 4.0, 2.0, 3.0));\n assert_eq!(a.zwyz(), F32x4::new(3.0, 4.0, 2.0, 3.0));\n assert_eq!(a.wwyz(), F32x4::new(4.0, 4.0, 2.0, 3.0));\n assert_eq!(a.xxzz(), F32x4::new(1.0, 1.0, 3.0, 3.0));\n assert_eq!(a.yxzz(), F32x4::new(2.0, 1.0, 3.0, 3.0));\n assert_eq!(a.zxzz(), F32x4::new(3.0, 1.0, 3.0, 3.0));\n assert_eq!(a.wxzz(), F32x4::new(4.0, 1.0, 3.0, 3.0));\n assert_eq!(a.xyzz(), F32x4::new(1.0, 2.0, 3.0, 3.0));\n assert_eq!(a.yyzz(), F32x4::new(2.0, 2.0, 3.0, 3.0));\n assert_eq!(a.zyzz(), F32x4::new(3.0, 2.0, 3.0, 3.0));\n assert_eq!(a.wyzz(), F32x4::new(4.0, 2.0, 3.0, 3.0));\n assert_eq!(a.xzzz(), F32x4::new(1.0, 3.0, 3.0, 3.0));\n assert_eq!(a.yzzz(), F32x4::new(2.0, 3.0, 3.0, 3.0));\n assert_eq!(a.wzzz(), F32x4::new(4.0, 3.0, 3.0, 3.0));\n assert_eq!(a.xwzz(), F32x4::new(1.0, 4.0, 3.0, 3.0));\n assert_eq!(a.ywzz(), F32x4::new(2.0, 4.0, 3.0, 3.0));\n assert_eq!(a.zwzz(), F32x4::new(3.0, 4.0, 3.0, 3.0));\n assert_eq!(a.wwzz(), F32x4::new(4.0, 4.0, 3.0, 3.0));\n assert_eq!(a.xxwz(), F32x4::new(1.0, 1.0, 4.0, 3.0));\n assert_eq!(a.yxwz(), F32x4::new(2.0, 1.0, 4.0, 3.0));\n assert_eq!(a.zxwz(), F32x4::new(3.0, 1.0, 4.0, 3.0));\n assert_eq!(a.wxwz(), F32x4::new(4.0, 1.0, 4.0, 3.0));\n assert_eq!(a.xywz(), F32x4::new(1.0, 2.0, 4.0, 3.0));\n assert_eq!(a.yywz(), F32x4::new(2.0, 2.0, 4.0, 3.0));\n assert_eq!(a.zywz(), F32x4::new(3.0, 2.0, 4.0, 3.0));\n assert_eq!(a.wywz(), F32x4::new(4.0, 2.0, 4.0, 3.0));\n assert_eq!(a.xzwz(), F32x4::new(1.0, 3.0, 4.0, 3.0));\n assert_eq!(a.yzwz(), F32x4::new(2.0, 3.0, 4.0, 3.0));\n assert_eq!(a.zzwz(), F32x4::new(3.0, 3.0, 4.0, 3.0));\n assert_eq!(a.wzwz(), F32x4::new(4.0, 3.0, 4.0, 3.0));\n assert_eq!(a.xwwz(), F32x4::new(1.0, 4.0, 4.0, 3.0));\n assert_eq!(a.ywwz(), F32x4::new(2.0, 4.0, 4.0, 3.0));\n assert_eq!(a.zwwz(), F32x4::new(3.0, 4.0, 4.0, 3.0));\n assert_eq!(a.wwwz(), F32x4::new(4.0, 4.0, 4.0, 3.0));\n}\n\n#[test]\nfn test_f32x4_concatenations() {\n let a = F32x4::new(4.0, 2.0, 6.0, -1.0);\n let b = F32x4::new(10.0, -3.0, 15.0, 41.0);\n assert_eq!(a.concat_xy_xy(b), F32x4::new(4.0, 2.0, 10.0, -3.0));\n assert_eq!(a.concat_xy_zw(b), F32x4::new(4.0, 2.0, 15.0, 41.0));\n assert_eq!(a.concat_zw_zw(b), F32x4::new(6.0, -1.0, 15.0, 41.0));\n assert_eq!(a.concat_wz_yx(b), F32x4::new(-1.0, 6.0, -3.0, 10.0));\n}\n\n#[test]\nfn test_f32x4_arithmetic_overloads() {\n let a = F32x4::new(4.0, -1.0, 6.0, -32.0);\n let b = F32x4::new(0.5, 0.5, 10.0, 3.0);\n let a_plus_b = F32x4::new(4.5, -0.5, 16.0, -29.0);\n let a_minus_b = F32x4::new(3.5, -1.5, -4.0, -35.0);\n let a_times_b = F32x4::new(2.0, -0.5, 60.0, -96.0);\n assert_eq!(a + b, a_plus_b);\n assert_eq!(a - b, a_minus_b);\n assert_eq!(a * b, a_times_b);\n let mut c = a;\n c += b;\n assert_eq!(c, a_plus_b);\n c = a;\n c -= b;\n assert_eq!(c, a_minus_b);\n c = a;\n c *= b;\n assert_eq!(c, a_times_b);\n assert_eq!(-a, F32x4::new(-4.0, 1.0, -6.0, 32.0));\n}\n\n#[test]\nfn test_f32x4_index_overloads() {\n let mut a = F32x4::new(4.0, 1.0, -32.5, 75.0);\n assert_eq!(a[2], -32.5);\n a[3] = 300.0;\n assert_eq!(a[3], 300.0);\n a[0] *= 0.5;\n assert_eq!(a[0], 2.0);\n}\n\n#[test]\nfn test_f32x4_conversions() {\n let a = F32x4::new(48.0, -4.0, 200.0, 7.0);\n assert_eq!(a.to_i32x4(), I32x4::new(48, -4, 200, 7));\n}\n\n#[test]\nfn test_f32x4_debug() {\n let a = F32x4::new(48.0, -4.0, 200.0, 7.0);\n assert_eq!(\"<48, -4, 200, 7>\", format!(\"{:?}\", a));\n}\n\n// I32x4\n\n#[test]\nfn test_i32x4_constructors() {\n let a = I32x4::new(3, 58, 10, 4);\n assert_eq!((a[0], a[1], a[2], a[3]), (3, 58, 10, 4));\n let b = I32x4::splat(39);\n assert_eq!(b, I32x4::new(39, 39, 39, 39));\n}\n\n#[test]\nfn test_i32x4_basic_ops() {\n let a = I32x4::new(6, 29, -40, 2);\n let b = I32x4::new(10, -5, 10, 46);\n assert_eq!(a.min(b), I32x4::new(6, -5, -40, 2));\n}\n\n#[test]\nfn test_i32x4_packed_comparisons() {\n let a = I32x4::new(59, 1, 5, 63);\n let b = I32x4::new(-59, 1, 5, 104);\n assert_eq!(a.packed_eq(b), U32x4::new(0, !0, !0, 0));\n}\n\n#[test]\nfn test_i32x4_swizzles() {\n let a = I32x4::new(1, 2, 3, 4);\n assert_eq!(a.xxxx(), I32x4::splat(1));\n assert_eq!(a.yyyy(), I32x4::splat(2));\n assert_eq!(a.zzzz(), I32x4::splat(3));\n assert_eq!(a.wwww(), I32x4::splat(4));\n\n assert_eq!(a.yxxx(), I32x4::new(2, 1, 1, 1));\n assert_eq!(a.zxxx(), I32x4::new(3, 1, 1, 1));\n assert_eq!(a.wxxx(), I32x4::new(4, 1, 1, 1));\n assert_eq!(a.xyxx(), I32x4::new(1, 2, 1, 1));\n assert_eq!(a.yyxx(), I32x4::new(2, 2, 1, 1));\n assert_eq!(a.zyxx(), I32x4::new(3, 2, 1, 1));\n assert_eq!(a.wyxx(), I32x4::new(4, 2, 1, 1));\n assert_eq!(a.xzxx(), I32x4::new(1, 3, 1, 1));\n assert_eq!(a.yzxx(), I32x4::new(2, 3, 1, 1));\n assert_eq!(a.zzxx(), I32x4::new(3, 3, 1, 1));\n assert_eq!(a.wzxx(), I32x4::new(4, 3, 1, 1));\n assert_eq!(a.xwxx(), I32x4::new(1, 4, 1, 1));\n assert_eq!(a.ywxx(), I32x4::new(2, 4, 1, 1));\n assert_eq!(a.zwxx(), I32x4::new(3, 4, 1, 1));\n assert_eq!(a.wwxx(), I32x4::new(4, 4, 1, 1));\n assert_eq!(a.xxyx(), I32x4::new(1, 1, 2, 1));\n assert_eq!(a.yxyx(), I32x4::new(2, 1, 2, 1));\n assert_eq!(a.zxyx(), I32x4::new(3, 1, 2, 1));\n assert_eq!(a.wxyx(), I32x4::new(4, 1, 2, 1));\n assert_eq!(a.xyyx(), I32x4::new(1, 2, 2, 1));\n assert_eq!(a.yyyx(), I32x4::new(2, 2, 2, 1));\n assert_eq!(a.zyyx(), I32x4::new(3, 2, 2, 1));\n assert_eq!(a.wyyx(), I32x4::new(4, 2, 2, 1));\n assert_eq!(a.xzyx(), I32x4::new(1, 3, 2, 1));\n assert_eq!(a.yzyx(), I32x4::new(2, 3, 2, 1));\n assert_eq!(a.zzyx(), I32x4::new(3, 3, 2, 1));\n assert_eq!(a.wzyx(), I32x4::new(4, 3, 2, 1));\n assert_eq!(a.xwyx(), I32x4::new(1, 4, 2, 1));\n assert_eq!(a.ywyx(), I32x4::new(2, 4, 2, 1));\n assert_eq!(a.zwyx(), I32x4::new(3, 4, 2, 1));\n assert_eq!(a.wwyx(), I32x4::new(4, 4, 2, 1));\n assert_eq!(a.xxzx(), I32x4::new(1, 1, 3, 1));\n assert_eq!(a.yxzx(), I32x4::new(2, 1, 3, 1));\n assert_eq!(a.zxzx(), I32x4::new(3, 1, 3, 1));\n assert_eq!(a.wxzx(), I32x4::new(4, 1, 3, 1));\n assert_eq!(a.xyzx(), I32x4::new(1, 2, 3, 1));\n assert_eq!(a.yyzx(), I32x4::new(2, 2, 3, 1));\n assert_eq!(a.zyzx(), I32x4::new(3, 2, 3, 1));\n assert_eq!(a.wyzx(), I32x4::new(4, 2, 3, 1));\n assert_eq!(a.xzzx(), I32x4::new(1, 3, 3, 1));\n assert_eq!(a.yzzx(), I32x4::new(2, 3, 3, 1));\n assert_eq!(a.zzzx(), I32x4::new(3, 3, 3, 1));\n assert_eq!(a.wzzx(), I32x4::new(4, 3, 3, 1));\n assert_eq!(a.xwzx(), I32x4::new(1, 4, 3, 1));\n assert_eq!(a.ywzx(), I32x4::new(2, 4, 3, 1));\n assert_eq!(a.zwzx(), I32x4::new(3, 4, 3, 1));\n assert_eq!(a.wwzx(), I32x4::new(4, 4, 3, 1));\n assert_eq!(a.xxwx(), I32x4::new(1, 1, 4, 1));\n assert_eq!(a.yxwx(), I32x4::new(2, 1, 4, 1));\n assert_eq!(a.zxwx(), I32x4::new(3, 1, 4, 1));\n assert_eq!(a.wxwx(), I32x4::new(4, 1, 4, 1));\n assert_eq!(a.xywx(), I32x4::new(1, 2, 4, 1));\n assert_eq!(a.yywx(), I32x4::new(2, 2, 4, 1));\n assert_eq!(a.zywx(), I32x4::new(3, 2, 4, 1));\n assert_eq!(a.wywx(), I32x4::new(4, 2, 4, 1));\n assert_eq!(a.xzwx(), I32x4::new(1, 3, 4, 1));\n assert_eq!(a.yzwx(), I32x4::new(2, 3, 4, 1));\n assert_eq!(a.zzwx(), I32x4::new(3, 3, 4, 1));\n assert_eq!(a.wzwx(), I32x4::new(4, 3, 4, 1));\n assert_eq!(a.xwwx(), I32x4::new(1, 4, 4, 1));\n assert_eq!(a.ywwx(), I32x4::new(2, 4, 4, 1));\n assert_eq!(a.zwwx(), I32x4::new(3, 4, 4, 1));\n assert_eq!(a.wwwx(), I32x4::new(4, 4, 4, 1));\n\n assert_eq!(a.xxxy(), I32x4::new(1, 1, 1, 2));\n assert_eq!(a.yxxy(), I32x4::new(2, 1, 1, 2));\n assert_eq!(a.zxxy(), I32x4::new(3, 1, 1, 2));\n assert_eq!(a.wxxy(), I32x4::new(4, 1, 1, 2));\n assert_eq!(a.xyxy(), I32x4::new(1, 2, 1, 2));\n assert_eq!(a.yyxy(), I32x4::new(2, 2, 1, 2));\n assert_eq!(a.zyxy(), I32x4::new(3, 2, 1, 2));\n assert_eq!(a.wyxy(), I32x4::new(4, 2, 1, 2));\n assert_eq!(a.xzxy(), I32x4::new(1, 3, 1, 2));\n assert_eq!(a.yzxy(), I32x4::new(2, 3, 1, 2));\n assert_eq!(a.zzxy(), I32x4::new(3, 3, 1, 2));\n assert_eq!(a.wzxy(), I32x4::new(4, 3, 1, 2));\n assert_eq!(a.xwxy(), I32x4::new(1, 4, 1, 2));\n assert_eq!(a.ywxy(), I32x4::new(2, 4, 1, 2));\n assert_eq!(a.zwxy(), I32x4::new(3, 4, 1, 2));\n assert_eq!(a.wwxy(), I32x4::new(4, 4, 1, 2));\n assert_eq!(a.xxyy(), I32x4::new(1, 1, 2, 2));\n assert_eq!(a.yxyy(), I32x4::new(2, 1, 2, 2));\n assert_eq!(a.zxyy(), I32x4::new(3, 1, 2, 2));\n assert_eq!(a.wxyy(), I32x4::new(4, 1, 2, 2));\n assert_eq!(a.xyyy(), I32x4::new(1, 2, 2, 2));\n assert_eq!(a.zyyy(), I32x4::new(3, 2, 2, 2));\n assert_eq!(a.wyyy(), I32x4::new(4, 2, 2, 2));\n assert_eq!(a.xzyy(), I32x4::new(1, 3, 2, 2));\n assert_eq!(a.yzyy(), I32x4::new(2, 3, 2, 2));\n assert_eq!(a.zzyy(), I32x4::new(3, 3, 2, 2));\n assert_eq!(a.wzyy(), I32x4::new(4, 3, 2, 2));\n assert_eq!(a.xwyy(), I32x4::new(1, 4, 2, 2));\n assert_eq!(a.ywyy(), I32x4::new(2, 4, 2, 2));\n assert_eq!(a.zwyy(), I32x4::new(3, 4, 2, 2));\n assert_eq!(a.wwyy(), I32x4::new(4, 4, 2, 2));\n assert_eq!(a.xxzy(), I32x4::new(1, 1, 3, 2));\n assert_eq!(a.yxzy(), I32x4::new(2, 1, 3, 2));\n assert_eq!(a.zxzy(), I32x4::new(3, 1, 3, 2));\n assert_eq!(a.wxzy(), I32x4::new(4, 1, 3, 2));\n assert_eq!(a.xyzy(), I32x4::new(1, 2, 3, 2));\n assert_eq!(a.yyzy(), I32x4::new(2, 2, 3, 2));\n assert_eq!(a.zyzy(), I32x4::new(3, 2, 3, 2));\n assert_eq!(a.wyzy(), I32x4::new(4, 2, 3, 2));\n assert_eq!(a.xzzy(), I32x4::new(1, 3, 3, 2));\n assert_eq!(a.yzzy(), I32x4::new(2, 3, 3, 2));\n assert_eq!(a.zzzy(), I32x4::new(3, 3, 3, 2));\n assert_eq!(a.wzzy(), I32x4::new(4, 3, 3, 2));\n assert_eq!(a.xwzy(), I32x4::new(1, 4, 3, 2));\n assert_eq!(a.ywzy(), I32x4::new(2, 4, 3, 2));\n assert_eq!(a.zwzy(), I32x4::new(3, 4, 3, 2));\n assert_eq!(a.wwzy(), I32x4::new(4, 4, 3, 2));\n assert_eq!(a.xxwy(), I32x4::new(1, 1, 4, 2));\n assert_eq!(a.yxwy(), I32x4::new(2, 1, 4, 2));\n assert_eq!(a.zxwy(), I32x4::new(3, 1, 4, 2));\n assert_eq!(a.wxwy(), I32x4::new(4, 1, 4, 2));\n assert_eq!(a.xywy(), I32x4::new(1, 2, 4, 2));\n assert_eq!(a.yywy(), I32x4::new(2, 2, 4, 2));\n assert_eq!(a.zywy(), I32x4::new(3, 2, 4, 2));\n assert_eq!(a.wywy(), I32x4::new(4, 2, 4, 2));\n assert_eq!(a.xzwy(), I32x4::new(1, 3, 4, 2));\n assert_eq!(a.yzwy(), I32x4::new(2, 3, 4, 2));\n assert_eq!(a.zzwy(), I32x4::new(3, 3, 4, 2));\n assert_eq!(a.wzwy(), I32x4::new(4, 3, 4, 2));\n assert_eq!(a.xwwy(), I32x4::new(1, 4, 4, 2));\n assert_eq!(a.ywwy(), I32x4::new(2, 4, 4, 2));\n assert_eq!(a.zwwy(), I32x4::new(3, 4, 4, 2));\n assert_eq!(a.wwwy(), I32x4::new(4, 4, 4, 2));\n\n assert_eq!(a.xxxz(), I32x4::new(1, 1, 1, 3));\n assert_eq!(a.yxxz(), I32x4::new(2, 1, 1, 3));\n assert_eq!(a.zxxz(), I32x4::new(3, 1, 1, 3));\n assert_eq!(a.wxxz(), I32x4::new(4, 1, 1, 3));\n assert_eq!(a.xyxz(), I32x4::new(1, 2, 1, 3));\n assert_eq!(a.yyxz(), I32x4::new(2, 2, 1, 3));\n assert_eq!(a.zyxz(), I32x4::new(3, 2, 1, 3));\n assert_eq!(a.wyxz(), I32x4::new(4, 2, 1, 3));\n assert_eq!(a.xzxz(), I32x4::new(1, 3, 1, 3));\n assert_eq!(a.yzxz(), I32x4::new(2, 3, 1, 3));\n assert_eq!(a.zzxz(), I32x4::new(3, 3, 1, 3));\n assert_eq!(a.wzxz(), I32x4::new(4, 3, 1, 3));\n assert_eq!(a.xwxz(), I32x4::new(1, 4, 1, 3));\n assert_eq!(a.ywxz(), I32x4::new(2, 4, 1, 3));\n assert_eq!(a.zwxz(), I32x4::new(3, 4, 1, 3));\n assert_eq!(a.wwxz(), I32x4::new(4, 4, 1, 3));\n assert_eq!(a.xxyz(), I32x4::new(1, 1, 2, 3));\n assert_eq!(a.yxyz(), I32x4::new(2, 1, 2, 3));\n assert_eq!(a.zxyz(), I32x4::new(3, 1, 2, 3));\n assert_eq!(a.wxyz(), I32x4::new(4, 1, 2, 3));\n assert_eq!(a.xyyz(), I32x4::new(1, 2, 2, 3));\n assert_eq!(a.yyyz(), I32x4::new(2, 2, 2, 3));\n assert_eq!(a.zyyz(), I32x4::new(3, 2, 2, 3));\n assert_eq!(a.wyyz(), I32x4::new(4, 2, 2, 3));\n assert_eq!(a.xzyz(), I32x4::new(1, 3, 2, 3));\n assert_eq!(a.yzyz(), I32x4::new(2, 3, 2, 3));\n assert_eq!(a.zzyz(), I32x4::new(3, 3, 2, 3));\n assert_eq!(a.wzyz(), I32x4::new(4, 3, 2, 3));\n assert_eq!(a.xwyz(), I32x4::new(1, 4, 2, 3));\n assert_eq!(a.ywyz(), I32x4::new(2, 4, 2, 3));\n assert_eq!(a.zwyz(), I32x4::new(3, 4, 2, 3));\n assert_eq!(a.wwyz(), I32x4::new(4, 4, 2, 3));\n assert_eq!(a.xxzz(), I32x4::new(1, 1, 3, 3));\n assert_eq!(a.yxzz(), I32x4::new(2, 1, 3, 3));\n assert_eq!(a.zxzz(), I32x4::new(3, 1, 3, 3));\n assert_eq!(a.wxzz(), I32x4::new(4, 1, 3, 3));\n assert_eq!(a.xyzz(), I32x4::new(1, 2, 3, 3));\n assert_eq!(a.yyzz(), I32x4::new(2, 2, 3, 3));\n assert_eq!(a.zyzz(), I32x4::new(3, 2, 3, 3));\n assert_eq!(a.wyzz(), I32x4::new(4, 2, 3, 3));\n assert_eq!(a.xzzz(), I32x4::new(1, 3, 3, 3));\n assert_eq!(a.yzzz(), I32x4::new(2, 3, 3, 3));\n assert_eq!(a.wzzz(), I32x4::new(4, 3, 3, 3));\n assert_eq!(a.xwzz(), I32x4::new(1, 4, 3, 3));\n assert_eq!(a.ywzz(), I32x4::new(2, 4, 3, 3));\n assert_eq!(a.zwzz(), I32x4::new(3, 4, 3, 3));\n assert_eq!(a.wwzz(), I32x4::new(4, 4, 3, 3));\n assert_eq!(a.xxwz(), I32x4::new(1, 1, 4, 3));\n assert_eq!(a.yxwz(), I32x4::new(2, 1, 4, 3));\n assert_eq!(a.zxwz(), I32x4::new(3, 1, 4, 3));\n assert_eq!(a.wxwz(), I32x4::new(4, 1, 4, 3));\n assert_eq!(a.xywz(), I32x4::new(1, 2, 4, 3));\n assert_eq!(a.yywz(), I32x4::new(2, 2, 4, 3));\n assert_eq!(a.zywz(), I32x4::new(3, 2, 4, 3));\n assert_eq!(a.wywz(), I32x4::new(4, 2, 4, 3));\n assert_eq!(a.xzwz(), I32x4::new(1, 3, 4, 3));\n assert_eq!(a.yzwz(), I32x4::new(2, 3, 4, 3));\n assert_eq!(a.zzwz(), I32x4::new(3, 3, 4, 3));\n assert_eq!(a.wzwz(), I32x4::new(4, 3, 4, 3));\n assert_eq!(a.xwwz(), I32x4::new(1, 4, 4, 3));\n assert_eq!(a.ywwz(), I32x4::new(2, 4, 4, 3));\n assert_eq!(a.zwwz(), I32x4::new(3, 4, 4, 3));\n assert_eq!(a.wwwz(), I32x4::new(4, 4, 4, 3));\n\n assert_eq!(a.xxxw(), I32x4::new(1, 1, 1, 4));\n assert_eq!(a.yxxw(), I32x4::new(2, 1, 1, 4));\n assert_eq!(a.zxxw(), I32x4::new(3, 1, 1, 4));\n assert_eq!(a.wxxw(), I32x4::new(4, 1, 1, 4));\n assert_eq!(a.xyxw(), I32x4::new(1, 2, 1, 4));\n assert_eq!(a.yyxw(), I32x4::new(2, 2, 1, 4));\n assert_eq!(a.zyxw(), I32x4::new(3, 2, 1, 4));\n assert_eq!(a.wyxw(), I32x4::new(4, 2, 1, 4));\n assert_eq!(a.xzxw(), I32x4::new(1, 3, 1, 4));\n assert_eq!(a.yzxw(), I32x4::new(2, 3, 1, 4));\n assert_eq!(a.zzxw(), I32x4::new(3, 3, 1, 4));\n assert_eq!(a.wzxw(), I32x4::new(4, 3, 1, 4));\n assert_eq!(a.xwxw(), I32x4::new(1, 4, 1, 4));\n assert_eq!(a.ywxw(), I32x4::new(2, 4, 1, 4));\n assert_eq!(a.zwxw(), I32x4::new(3, 4, 1, 4));\n assert_eq!(a.wwxw(), I32x4::new(4, 4, 1, 4));\n assert_eq!(a.xxyw(), I32x4::new(1, 1, 2, 4));\n assert_eq!(a.yxyw(), I32x4::new(2, 1, 2, 4));\n assert_eq!(a.zxyw(), I32x4::new(3, 1, 2, 4));\n assert_eq!(a.wxyw(), I32x4::new(4, 1, 2, 4));\n assert_eq!(a.xyyw(), I32x4::new(1, 2, 2, 4));\n assert_eq!(a.yyyw(), I32x4::new(2, 2, 2, 4));\n assert_eq!(a.zyyw(), I32x4::new(3, 2, 2, 4));\n assert_eq!(a.wyyw(), I32x4::new(4, 2, 2, 4));\n assert_eq!(a.xzyw(), I32x4::new(1, 3, 2, 4));\n assert_eq!(a.yzyw(), I32x4::new(2, 3, 2, 4));\n assert_eq!(a.zzyw(), I32x4::new(3, 3, 2, 4));\n assert_eq!(a.wzyw(), I32x4::new(4, 3, 2, 4));\n assert_eq!(a.xwyw(), I32x4::new(1, 4, 2, 4));\n assert_eq!(a.ywyw(), I32x4::new(2, 4, 2, 4));\n assert_eq!(a.zwyw(), I32x4::new(3, 4, 2, 4));\n assert_eq!(a.wwyw(), I32x4::new(4, 4, 2, 4));\n assert_eq!(a.xxzw(), I32x4::new(1, 1, 3, 4));\n assert_eq!(a.yxzw(), I32x4::new(2, 1, 3, 4));\n assert_eq!(a.zxzw(), I32x4::new(3, 1, 3, 4));\n assert_eq!(a.wxzw(), I32x4::new(4, 1, 3, 4));\n assert_eq!(a.xyzw(), I32x4::new(1, 2, 3, 4));\n assert_eq!(a.yyzw(), I32x4::new(2, 2, 3, 4));\n assert_eq!(a.zyzw(), I32x4::new(3, 2, 3, 4));\n assert_eq!(a.wyzw(), I32x4::new(4, 2, 3, 4));\n assert_eq!(a.xzzw(), I32x4::new(1, 3, 3, 4));\n assert_eq!(a.yzzw(), I32x4::new(2, 3, 3, 4));\n assert_eq!(a.zzzw(), I32x4::new(3, 3, 3, 4));\n assert_eq!(a.wzzw(), I32x4::new(4, 3, 3, 4));\n assert_eq!(a.xwzw(), I32x4::new(1, 4, 3, 4));\n assert_eq!(a.ywzw(), I32x4::new(2, 4, 3, 4));\n assert_eq!(a.zwzw(), I32x4::new(3, 4, 3, 4));\n assert_eq!(a.wwzw(), I32x4::new(4, 4, 3, 4));\n assert_eq!(a.xxww(), I32x4::new(1, 1, 4, 4));\n assert_eq!(a.yxww(), I32x4::new(2, 1, 4, 4));\n assert_eq!(a.zxww(), I32x4::new(3, 1, 4, 4));\n assert_eq!(a.wxww(), I32x4::new(4, 1, 4, 4));\n assert_eq!(a.xyww(), I32x4::new(1, 2, 4, 4));\n assert_eq!(a.yyww(), I32x4::new(2, 2, 4, 4));\n assert_eq!(a.zyww(), I32x4::new(3, 2, 4, 4));\n assert_eq!(a.wzww(), I32x4::new(4, 3, 4, 4));\n assert_eq!(a.xzww(), I32x4::new(1, 3, 4, 4));\n assert_eq!(a.yzww(), I32x4::new(2, 3, 4, 4));\n assert_eq!(a.zzww(), I32x4::new(3, 3, 4, 4));\n assert_eq!(a.wzww(), I32x4::new(4, 3, 4, 4));\n assert_eq!(a.xwww(), I32x4::new(1, 4, 4, 4));\n assert_eq!(a.ywww(), I32x4::new(2, 4, 4, 4));\n assert_eq!(a.zwww(), I32x4::new(3, 4, 4, 4));\n}\n\n// Scalar F32x4\n\n#[test]\nfn test_f32x4s_constructors() {\n let a = F32x4S::new(1.0, 2.0, 3.0, 4.0);\n assert_eq!((a[0], a[1], a[2], a[3]), (1.0, 2.0, 3.0, 4.0));\n let b = F32x4S::splat(10.0);\n assert_eq!(b, F32x4S::new(10.0, 10.0, 10.0, 10.0));\n}\n\n#[test]\nfn test_f32x4s_basic_ops() {\n let a = F32x4S::new(1.0, 3.0, 5.0, 7.0);\n let b = F32x4S::new(2.0, 2.0, 6.0, 6.0);\n assert_eq!(a.min(b), F32x4S::new(1.0, 2.0, 5.0, 6.0));\n assert_eq!(a.max(b), F32x4S::new(2.0, 3.0, 6.0, 7.0));\n let c = F32x4S::new(-1.0, 1.3, -20.0, 3.6);\n assert_eq!(c.abs(), F32x4S::new(1.0, 1.3, 20.0, 3.6));\n assert_eq!(c.floor(), F32x4S::new(-1.0, 1.0, -20.0, 3.0));\n assert_eq!(c.ceil(), F32x4S::new(-1.0, 2.0, -20.0, 4.0));\n assert_eq!(c.to_i32x4().to_f32x4(), F32x4S::new(-1.0, 1.0, -20.0, 4.0));\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/x86/mod.rs", "sha256": "9cdab898e39521790403b85e49c7d92ea8fac1f597cbdd0219bb377421e4f2bd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/x86.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse std::cmp::PartialEq;\nuse std::fmt::{self, Debug, Formatter};\nuse std::mem;\nuse std::ops::{Add, BitAnd, BitOr, BitXor, Div, Index, IndexMut, Mul, Not, Shr, Sub};\n\n#[cfg(target_pointer_width = \"32\")]\nuse std::arch::x86::{__m128, __m128i};\n#[cfg(target_pointer_width = \"32\")]\nuse std::arch::x86;\n#[cfg(target_pointer_width = \"64\")]\nuse std::arch::x86_64::{__m128, __m128i};\n#[cfg(target_pointer_width = \"64\")]\nuse std::arch::x86_64 as x86;\n\nmod swizzle_f32x4;\nmod swizzle_i32x4;\n\n// Two 32-bit floats\n\n#[derive(Clone, Copy)]\npub struct F32x2(pub u64);\n\nimpl F32x2 {\n // Constructors\n\n #[inline]\n pub fn new(a: f32, b: f32) -> F32x2 {\n unsafe {\n let a = mem::transmute::<*const f32, *const u32>(&a);\n let b = mem::transmute::<*const f32, *const u32>(&b);\n F32x2((*a as u64) | ((*b as u64) << 32))\n }\n }\n\n #[inline]\n pub fn splat(x: f32) -> F32x2 {\n F32x2::new(x, x)\n }\n\n // Basic operations\n\n #[inline]\n pub fn approx_recip(self) -> F32x2 {\n self.to_f32x4().approx_recip().xy()\n }\n\n #[inline]\n pub fn min(self, other: F32x2) -> F32x2 {\n self.to_f32x4().min(other.to_f32x4()).xy()\n }\n\n #[inline]\n pub fn max(self, other: F32x2) -> F32x2 {\n self.to_f32x4().max(other.to_f32x4()).xy()\n }\n\n #[inline]\n pub fn clamp(self, min: F32x2, max: F32x2) -> F32x2 {\n self.to_f32x4().clamp(min.to_f32x4(), max.to_f32x4()).xy()\n }\n\n #[inline]\n pub fn abs(self) -> F32x2 {\n self.to_f32x4().abs().xy()\n }\n\n #[inline]\n pub fn floor(self) -> F32x2 {\n self.to_f32x4().floor().xy()\n }\n\n #[inline]\n pub fn ceil(self) -> F32x2 {\n self.to_f32x4().ceil().xy()\n }\n\n #[inline]\n pub fn sqrt(self) -> F32x2 {\n self.to_f32x4().sqrt().xy()\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: F32x2) -> U32x2 {\n self.to_f32x4().packed_eq(other.to_f32x4()).xy()\n }\n\n #[inline]\n pub fn packed_gt(self, other: F32x2) -> U32x2 {\n self.to_f32x4().packed_gt(other.to_f32x4()).xy()\n }\n\n #[inline]\n pub fn packed_lt(self, other: F32x2) -> U32x2 {\n self.to_f32x4().packed_lt(other.to_f32x4()).xy()\n }\n\n #[inline]\n pub fn packed_le(self, other: F32x2) -> U32x2 {\n self.to_f32x4().packed_le(other.to_f32x4()).xy()\n }\n\n // Conversions\n\n #[inline]\n pub fn to_f32x4(self) -> F32x4 {\n unsafe {\n let mut result = F32x4::default();\n *mem::transmute::<&mut __m128, &mut u64>(&mut result.0) = self.0;\n result\n }\n }\n\n #[inline]\n pub fn to_i32x2(self) -> I32x2 {\n self.to_i32x4().xy()\n }\n\n #[inline]\n pub fn to_i32x4(self) -> I32x4 {\n self.to_f32x4().to_i32x4()\n }\n\n // Swizzle\n\n #[inline]\n pub fn yx(self) -> F32x2 {\n self.to_f32x4().yx()\n }\n\n // Concatenations\n\n #[inline]\n pub fn concat_xy_xy(self, other: F32x2) -> F32x4 {\n self.to_f32x4().concat_xy_xy(other.to_f32x4())\n }\n}\n\nimpl Default for F32x2 {\n #[inline]\n fn default() -> F32x2 {\n F32x2(0)\n }\n}\n\nimpl Index for F32x2 {\n type Output = f32;\n #[inline]\n fn index(&self, index: usize) -> &f32 {\n unsafe { &mem::transmute::<&u64, &[f32; 2]>(&self.0)[index] }\n }\n}\n\nimpl IndexMut for F32x2 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut f32 {\n unsafe { &mut mem::transmute::<&mut u64, &mut [f32; 2]>(&mut self.0)[index] }\n }\n}\n\nimpl Debug for F32x2 {\n #[inline]\n fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {\n write!(f, \"<{}, {}>\", self[0], self[1])\n }\n}\n\nimpl PartialEq for F32x2 {\n #[inline]\n fn eq(&self, other: &F32x2) -> bool {\n self.packed_eq(*other).all_true()\n }\n}\n\nimpl Add for F32x2 {\n type Output = F32x2;\n #[inline]\n fn add(self, other: F32x2) -> F32x2 {\n (self.to_f32x4() + other.to_f32x4()).xy()\n }\n}\n\nimpl Div for F32x2 {\n type Output = F32x2;\n #[inline]\n fn div(self, other: F32x2) -> F32x2 {\n (self.to_f32x4() / other.to_f32x4()).xy()\n }\n}\n\nimpl Mul for F32x2 {\n type Output = F32x2;\n #[inline]\n fn mul(self, other: F32x2) -> F32x2 {\n (self.to_f32x4() * other.to_f32x4()).xy()\n }\n}\n\nimpl Sub for F32x2 {\n type Output = F32x2;\n #[inline]\n fn sub(self, other: F32x2) -> F32x2 {\n (self.to_f32x4() - other.to_f32x4()).xy()\n }\n}\n\n// Four 32-bit floats\n\n#[derive(Clone, Copy)]\npub struct F32x4(pub __m128);\n\nimpl F32x4 {\n // Constructors\n\n #[inline]\n pub fn new(a: f32, b: f32, c: f32, d: f32) -> F32x4 {\n unsafe {\n let vector = [a, b, c, d];\n F32x4(x86::_mm_loadu_ps(vector.as_ptr()))\n }\n }\n\n #[inline]\n pub fn splat(x: f32) -> F32x4 {\n unsafe { F32x4(x86::_mm_set1_ps(x)) }\n }\n\n // Basic operations\n\n #[inline]\n pub fn approx_recip(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_rcp_ps(self.0)) }\n }\n\n #[inline]\n pub fn min(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(x86::_mm_min_ps(self.0, other.0)) }\n }\n\n #[inline]\n pub fn max(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(x86::_mm_max_ps(self.0, other.0)) }\n }\n\n #[inline]\n pub fn clamp(self, min: F32x4, max: F32x4) -> F32x4 {\n self.max(min).min(max)\n }\n\n #[inline]\n pub fn abs(self) -> F32x4 {\n unsafe {\n let tmp = x86::_mm_srli_epi32(I32x4::splat(-1).0, 1);\n F32x4(x86::_mm_and_ps(x86::_mm_castsi128_ps(tmp), self.0))\n }\n }\n\n #[inline]\n pub fn floor(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_floor_ps(self.0)) }\n }\n\n #[inline]\n pub fn ceil(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_ceil_ps(self.0)) }\n }\n\n #[inline]\n pub fn sqrt(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_sqrt_ps(self.0)) }\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: F32x4) -> U32x4 {\n unsafe {\n U32x4(x86::_mm_castps_si128(x86::_mm_cmpeq_ps(\n self.0, other.0,\n )))\n }\n }\n\n #[inline]\n pub fn packed_gt(self, other: F32x4) -> U32x4 {\n unsafe {\n U32x4(x86::_mm_castps_si128(x86::_mm_cmpgt_ps(\n self.0, other.0,\n )))\n }\n }\n\n #[inline]\n pub fn packed_lt(self, other: F32x4) -> U32x4 {\n other.packed_gt(self)\n }\n\n #[inline]\n pub fn packed_le(self, other: F32x4) -> U32x4 {\n !self.packed_gt(other)\n }\n\n // Conversions\n\n /// Converts these packed floats to integers via rounding.\n #[inline]\n pub fn to_i32x4(self) -> I32x4 {\n unsafe { I32x4(x86::_mm_cvtps_epi32(self.0)) }\n }\n\n // Extraction\n\n #[inline]\n pub fn xy(self) -> F32x2 {\n unsafe {\n let swizzled = self.0;\n F32x2(*mem::transmute::<&__m128, &u64>(&swizzled))\n }\n }\n\n #[inline]\n pub fn xw(self) -> F32x2 {\n self.xwyz().xy()\n }\n\n #[inline]\n pub fn yx(self) -> F32x2 {\n self.yxwz().xy()\n }\n\n #[inline]\n pub fn zy(self) -> F32x2 {\n self.zyxw().xy()\n }\n\n #[inline]\n pub fn zw(self) -> F32x2 {\n self.zwxy().xy()\n }\n\n // Concatenations\n\n #[inline]\n pub fn concat_xy_xy(self, other: F32x4) -> F32x4 {\n unsafe {\n let this = x86::_mm_castps_pd(self.0);\n let other = x86::_mm_castps_pd(other.0);\n let result = x86::_mm_unpacklo_pd(this, other);\n F32x4(x86::_mm_castpd_ps(result))\n }\n }\n\n #[inline]\n pub fn concat_xy_zw(self, other: F32x4) -> F32x4 {\n unsafe {\n let this = x86::_mm_castps_pd(self.0);\n let other = x86::_mm_castps_pd(other.0);\n let result = x86::_mm_shuffle_pd(this, other, 0b10);\n F32x4(x86::_mm_castpd_ps(result))\n }\n }\n\n #[inline]\n pub fn concat_zw_zw(self, other: F32x4) -> F32x4 {\n unsafe {\n let this = x86::_mm_castps_pd(self.0);\n let other = x86::_mm_castps_pd(other.0);\n let result = x86::_mm_unpackhi_pd(this, other);\n F32x4(x86::_mm_castpd_ps(result))\n }\n }\n\n #[inline]\n pub fn concat_wz_yx(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, other.0, 0b0001_1011)) }\n }\n}\n\nimpl Default for F32x4 {\n #[inline]\n fn default() -> F32x4 {\n unsafe { F32x4(x86::_mm_setzero_ps()) }\n }\n}\n\nimpl Index for F32x4 {\n type Output = f32;\n #[inline]\n fn index(&self, index: usize) -> &f32 {\n unsafe { &mem::transmute::<&__m128, &[f32; 4]>(&self.0)[index] }\n }\n}\n\nimpl IndexMut for F32x4 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut f32 {\n unsafe { &mut mem::transmute::<&mut __m128, &mut [f32; 4]>(&mut self.0)[index] }\n }\n}\n\nimpl Debug for F32x4 {\n #[inline]\n fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {\n write!(f, \"<{}, {}, {}, {}>\", self[0], self[1], self[2], self[3])\n }\n}\n\nimpl PartialEq for F32x4 {\n #[inline]\n fn eq(&self, other: &F32x4) -> bool {\n self.packed_eq(*other).all_true()\n }\n}\n\nimpl Add for F32x4 {\n type Output = F32x4;\n #[inline]\n fn add(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(x86::_mm_add_ps(self.0, other.0)) }\n }\n}\n\nimpl Div for F32x4 {\n type Output = F32x4;\n #[inline]\n fn div(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(x86::_mm_div_ps(self.0, other.0)) }\n }\n}\n\nimpl Mul for F32x4 {\n type Output = F32x4;\n #[inline]\n fn mul(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(x86::_mm_mul_ps(self.0, other.0)) }\n }\n}\n\nimpl Sub for F32x4 {\n type Output = F32x4;\n #[inline]\n fn sub(self, other: F32x4) -> F32x4 {\n unsafe { F32x4(x86::_mm_sub_ps(self.0, other.0)) }\n }\n}\n\n// Two 32-bit signed integers\n\n#[derive(Clone, Copy)]\npub struct I32x2(pub u64);\n\nimpl I32x2 {\n // Constructors\n\n #[inline]\n pub fn new(a: i32, b: i32) -> I32x2 {\n unsafe {\n let a = mem::transmute::<*const i32, *const u32>(&a);\n let b = mem::transmute::<*const i32, *const u32>(&b);\n I32x2((*a as u64) | ((*b as u64) << 32))\n }\n }\n\n #[inline]\n pub fn splat(x: i32) -> I32x2 {\n I32x2::new(x, x)\n }\n\n // Accessors\n\n #[inline]\n pub fn x(self) -> i32 {\n self[0]\n }\n\n #[inline]\n pub fn y(self) -> i32 {\n self[1]\n }\n\n // Concatenations\n\n #[inline]\n pub fn concat_xy_xy(self, other: I32x2) -> I32x4 {\n self.to_i32x4().concat_xy_xy(other.to_i32x4())\n }\n\n // Conversions\n\n #[inline]\n pub fn to_i32x4(self) -> I32x4 {\n unsafe {\n let mut result = I32x4::default();\n *mem::transmute::<&mut __m128i, &mut u64>(&mut result.0) = self.0;\n result\n }\n }\n\n #[inline]\n pub fn to_f32x4(self) -> F32x4 {\n self.to_i32x4().to_f32x4()\n }\n\n /// Converts these packed integers to floats.\n #[inline]\n pub fn to_f32x2(self) -> F32x2 {\n self.to_f32x4().xy()\n }\n\n // Basic operations\n\n #[inline]\n pub fn max(self, other: I32x2) -> I32x2 {\n self.to_i32x4().max(other.to_i32x4()).xy()\n }\n\n #[inline]\n pub fn min(self, other: I32x2) -> I32x2 {\n self.to_i32x4().min(other.to_i32x4()).xy()\n }\n\n // Comparisons\n\n // TODO(pcwalton): Use the `U32x2` type!\n #[inline]\n pub fn packed_eq(self, other: I32x2) -> U32x4 {\n self.to_i32x4().packed_eq(other.to_i32x4())\n }\n\n #[inline]\n pub fn packed_gt(self, other: I32x2) -> U32x4 {\n self.to_i32x4().packed_gt(other.to_i32x4())\n }\n\n #[inline]\n pub fn packed_le(self, other: I32x2) -> U32x4 {\n self.to_i32x4().packed_le(other.to_i32x4())\n }\n}\n\nimpl Default for I32x2 {\n #[inline]\n fn default() -> I32x2 {\n I32x2(0)\n }\n}\n\nimpl Index for I32x2 {\n type Output = i32;\n #[inline]\n fn index(&self, index: usize) -> &i32 {\n unsafe { &mem::transmute::<&u64, &[i32; 2]>(&self.0)[index] }\n }\n}\n\nimpl IndexMut for I32x2 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut i32 {\n unsafe { &mut mem::transmute::<&mut u64, &mut [i32; 2]>(&mut self.0)[index] }\n }\n}\n\nimpl Add for I32x2 {\n type Output = I32x2;\n #[inline]\n fn add(self, other: I32x2) -> I32x2 {\n (self.to_i32x4() + other.to_i32x4()).xy()\n }\n}\n\nimpl Sub for I32x2 {\n type Output = I32x2;\n #[inline]\n fn sub(self, other: I32x2) -> I32x2 {\n (self.to_i32x4() - other.to_i32x4()).xy()\n }\n}\n\nimpl Mul for I32x2 {\n type Output = I32x2;\n #[inline]\n fn mul(self, other: I32x2) -> I32x2 {\n (self.to_i32x4() * other.to_i32x4()).xy()\n }\n}\n\nimpl Debug for I32x2 {\n #[inline]\n fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {\n write!(f, \"<{}, {}>\", self[0], self[1])\n }\n}\n\nimpl PartialEq for I32x2 {\n #[inline]\n fn eq(&self, other: &I32x2) -> bool {\n self.packed_eq(*other).all_true()\n }\n}\n\n// Four 32-bit signed integers\n\n#[derive(Clone, Copy)]\npub struct I32x4(pub __m128i);\n\nimpl I32x4 {\n // Constructors\n\n #[inline]\n pub fn new(a: i32, b: i32, c: i32, d: i32) -> I32x4 {\n unsafe {\n let vector = [a, b, c, d];\n I32x4(x86::_mm_loadu_si128(vector.as_ptr() as *const __m128i))\n }\n }\n\n #[inline]\n pub fn splat(x: i32) -> I32x4 {\n unsafe { I32x4(x86::_mm_set1_epi32(x)) }\n }\n\n // Extraction\n\n #[inline]\n pub fn xy(self) -> I32x2 {\n unsafe {\n let swizzled = self.0;\n I32x2(*mem::transmute::<&__m128i, &u64>(&swizzled))\n }\n }\n\n #[inline]\n pub fn xw(self) -> I32x2 {\n self.xwyz().xy()\n }\n\n #[inline]\n pub fn yx(self) -> I32x2 {\n self.yxwz().xy()\n }\n\n #[inline]\n pub fn zy(self) -> I32x2 {\n self.zyxw().xy()\n }\n\n #[inline]\n pub fn zw(self) -> I32x2 {\n self.zwxy().xy()\n }\n\n // Concatenations\n\n #[inline]\n pub fn concat_xy_xy(self, other: I32x4) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_pd(self.0);\n let other = x86::_mm_castsi128_pd(other.0);\n let result = x86::_mm_unpacklo_pd(this, other);\n I32x4(x86::_mm_castpd_si128(result))\n }\n }\n\n #[inline]\n pub fn concat_zw_zw(self, other: I32x4) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_pd(self.0);\n let other = x86::_mm_castsi128_pd(other.0);\n let result = x86::_mm_unpackhi_pd(this, other);\n I32x4(x86::_mm_castpd_si128(result))\n }\n }\n\n // Conversions\n\n /// Converts these packed integers to floats.\n #[inline]\n pub fn to_f32x4(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_cvtepi32_ps(self.0)) }\n }\n\n /// Converts these packed signed integers to unsigned integers.\n ///\n /// Overflowing values will wrap around.\n #[inline]\n pub fn to_u32x4(self) -> U32x4 {\n U32x4(self.0)\n }\n\n // Basic operations\n\n #[inline]\n pub fn max(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(x86::_mm_max_epi32(self.0, other.0)) }\n }\n\n #[inline]\n pub fn min(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(x86::_mm_min_epi32(self.0, other.0)) }\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: I32x4) -> U32x4 {\n unsafe { U32x4(x86::_mm_cmpeq_epi32(self.0, other.0)) }\n }\n\n // Comparisons\n\n #[inline]\n pub fn packed_gt(self, other: I32x4) -> U32x4 {\n unsafe { U32x4(x86::_mm_cmpgt_epi32(self.0, other.0)) }\n }\n\n #[inline]\n pub fn packed_lt(self, other: I32x4) -> U32x4 {\n other.packed_gt(self)\n }\n\n #[inline]\n pub fn packed_le(self, other: I32x4) -> U32x4 {\n !self.packed_gt(other)\n }\n}\n\nimpl Default for I32x4 {\n #[inline]\n fn default() -> I32x4 {\n unsafe { I32x4(x86::_mm_setzero_si128()) }\n }\n}\n\nimpl Index for I32x4 {\n type Output = i32;\n #[inline]\n fn index(&self, index: usize) -> &i32 {\n unsafe { &mem::transmute::<&__m128i, &[i32; 4]>(&self.0)[index] }\n }\n}\n\nimpl IndexMut for I32x4 {\n #[inline]\n fn index_mut(&mut self, index: usize) -> &mut i32 {\n unsafe { &mut mem::transmute::<&mut __m128i, &mut [i32; 4]>(&mut self.0)[index] }\n }\n}\n\nimpl Add for I32x4 {\n type Output = I32x4;\n #[inline]\n fn add(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(x86::_mm_add_epi32(self.0, other.0)) }\n }\n}\n\nimpl Sub for I32x4 {\n type Output = I32x4;\n #[inline]\n fn sub(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(x86::_mm_sub_epi32(self.0, other.0)) }\n }\n}\n\nimpl Mul for I32x4 {\n type Output = I32x4;\n #[inline]\n fn mul(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(x86::_mm_mullo_epi32(self.0, other.0)) }\n }\n}\n\nimpl BitAnd for I32x4 {\n type Output = I32x4;\n #[inline]\n fn bitand(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(x86::_mm_and_si128(self.0, other.0)) }\n }\n}\n\nimpl BitOr for I32x4 {\n type Output = I32x4;\n #[inline]\n fn bitor(self, other: I32x4) -> I32x4 {\n unsafe { I32x4(x86::_mm_or_si128(self.0, other.0)) }\n }\n}\n\nimpl Debug for I32x4 {\n #[inline]\n fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {\n write!(f, \"<{}, {}, {}, {}>\", self[0], self[1], self[2], self[3])\n }\n}\n\nimpl PartialEq for I32x4 {\n #[inline]\n fn eq(&self, other: &I32x4) -> bool {\n self.packed_eq(*other).all_true()\n }\n}\n\n// Two 32-bit unsigned integers\n\n#[derive(Clone, Copy)]\npub struct U32x2(pub u64);\n\nimpl U32x2 {\n #[inline]\n pub fn new(x: u32, y: u32) -> U32x2 {\n U32x2(x as u64 | ((y as u64) << 32))\n }\n\n #[inline]\n pub fn splat(x: u32) -> U32x2 {\n U32x2::new(x, x)\n }\n\n /// Returns true if both booleans in this vector are true.\n ///\n /// The result is *undefined* if both values in this vector are not booleans. A boolean is a\n /// value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_true(self) -> bool {\n self.0 == !0\n }\n\n /// Returns true if both booleans in this vector are false.\n ///\n /// The result is *undefined* if both values in this vector are not booleans. A boolean is a\n /// value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_false(self) -> bool {\n self.0 == 0\n }\n\n #[inline]\n pub fn to_i32x2(self) -> I32x2 {\n I32x2(self.0)\n }\n}\n\nimpl Not for U32x2 {\n type Output = U32x2;\n #[inline]\n fn not(self) -> U32x2 {\n U32x2(!self.0)\n }\n}\n\nimpl BitAnd for U32x2 {\n type Output = U32x2;\n #[inline]\n fn bitand(self, other: U32x2) -> U32x2 {\n U32x2(self.0 & other.0)\n }\n}\n\nimpl BitOr for U32x2 {\n type Output = U32x2;\n #[inline]\n fn bitor(self, other: U32x2) -> U32x2 {\n U32x2(self.0 | other.0)\n }\n}\n\n// Four 32-bit unsigned integers\n\n#[derive(Clone, Copy)]\npub struct U32x4(pub __m128i);\n\nimpl U32x4 {\n // Constructors\n\n #[inline]\n pub fn new(a: u32, b: u32, c: u32, d: u32) -> U32x4 {\n unsafe {\n let vector = [a, b, c, d];\n U32x4(x86::_mm_loadu_si128(vector.as_ptr() as *const __m128i))\n }\n }\n\n #[inline]\n pub fn splat(x: u32) -> U32x4 {\n unsafe { U32x4(x86::_mm_set1_epi32(x as i32)) }\n }\n\n // Conversions\n\n /// Converts these packed unsigned integers to signed integers.\n ///\n /// Overflowing values will wrap around.\n #[inline]\n pub fn to_i32x4(self) -> I32x4 {\n I32x4(self.0)\n }\n\n // Basic operations\n\n /// Returns true if all four booleans in this vector are true.\n ///\n /// The result is *undefined* if all four values in this vector are not booleans. A boolean is\n /// a value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_true(self) -> bool {\n unsafe { x86::_mm_movemask_ps(x86::_mm_castsi128_ps(self.0)) == 0x0f }\n }\n\n /// Returns true if all four booleans in this vector are false.\n ///\n /// The result is *undefined* if all four values in this vector are not booleans. A boolean is\n /// a value with all bits set or all bits clear (i.e. !0 or 0).\n #[inline]\n pub fn all_false(self) -> bool {\n unsafe { x86::_mm_movemask_ps(x86::_mm_castsi128_ps(self.0)) == 0x00 }\n }\n\n // Extraction\n\n #[inline]\n pub fn xy(self) -> U32x2 {\n unsafe {\n let swizzled = self.0;\n U32x2(*mem::transmute::<&__m128i, &u64>(&swizzled))\n }\n }\n\n // Packed comparisons\n\n #[inline]\n pub fn packed_eq(self, other: U32x4) -> U32x4 {\n unsafe { U32x4(x86::_mm_cmpeq_epi32(self.0, other.0)) }\n }\n}\n\nimpl Debug for U32x4 {\n #[inline]\n fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {\n write!(f, \"<{}, {}, {}, {}>\", self[0], self[1], self[2], self[3])\n }\n}\n\nimpl Index for U32x4 {\n type Output = u32;\n #[inline]\n fn index(&self, index: usize) -> &u32 {\n unsafe { &mem::transmute::<&__m128i, &[u32; 4]>(&self.0)[index] }\n }\n}\n\nimpl PartialEq for U32x4 {\n #[inline]\n fn eq(&self, other: &U32x4) -> bool {\n self.packed_eq(*other).all_true()\n }\n}\n\nimpl Not for U32x4 {\n type Output = U32x4;\n #[inline]\n fn not(self) -> U32x4 {\n self ^ U32x4::splat(!0)\n }\n}\n\nimpl BitXor for U32x4 {\n type Output = U32x4;\n #[inline]\n fn bitxor(self, other: U32x4) -> U32x4 {\n unsafe { U32x4(x86::_mm_xor_si128(self.0, other.0)) }\n }\n}\n\nimpl Shr for U32x4 {\n type Output = U32x4;\n #[inline]\n fn shr(self, amount: u32) -> U32x4 {\n unsafe { U32x4(x86::_mm_srl_epi32(self.0, U32x4::new(amount, 0, 0, 0).0)) }\n }\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/x86/swizzle_f32x4.rs", "sha256": "59868ea641f48fbf09de20bf145dd101e50254bc7890c860a42ecadb61d0c1e1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/x86/swizzle_f32x4.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::x86::F32x4;\n\n#[cfg(target_pointer_width = \"32\")]\nuse std::arch::x86;\n#[cfg(target_pointer_width = \"64\")]\nuse std::arch::x86_64 as x86;\n\nimpl F32x4 {\n #[inline]\n pub fn xxxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 0)) }\n }\n\n #[inline]\n pub fn yxxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 1)) }\n }\n\n #[inline]\n pub fn zxxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 2)) }\n }\n\n #[inline]\n pub fn wxxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 3)) }\n }\n\n #[inline]\n pub fn xyxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 4)) }\n }\n\n #[inline]\n pub fn yyxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 5)) }\n }\n\n #[inline]\n pub fn zyxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 6)) }\n }\n\n #[inline]\n pub fn wyxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 7)) }\n }\n\n #[inline]\n pub fn xzxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 8)) }\n }\n\n #[inline]\n pub fn yzxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 9)) }\n }\n\n #[inline]\n pub fn zzxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 10)) }\n }\n\n #[inline]\n pub fn wzxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 11)) }\n }\n\n #[inline]\n pub fn xwxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 12)) }\n }\n\n #[inline]\n pub fn ywxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 13)) }\n }\n\n #[inline]\n pub fn zwxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 14)) }\n }\n\n #[inline]\n pub fn wwxx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 15)) }\n }\n\n #[inline]\n pub fn xxyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 16)) }\n }\n\n #[inline]\n pub fn yxyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 17)) }\n }\n\n #[inline]\n pub fn zxyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 18)) }\n }\n\n #[inline]\n pub fn wxyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 19)) }\n }\n\n #[inline]\n pub fn xyyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 20)) }\n }\n\n #[inline]\n pub fn yyyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 21)) }\n }\n\n #[inline]\n pub fn zyyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 22)) }\n }\n\n #[inline]\n pub fn wyyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 23)) }\n }\n\n #[inline]\n pub fn xzyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 24)) }\n }\n\n #[inline]\n pub fn yzyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 25)) }\n }\n\n #[inline]\n pub fn zzyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 26)) }\n }\n\n #[inline]\n pub fn wzyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 27)) }\n }\n\n #[inline]\n pub fn xwyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 28)) }\n }\n\n #[inline]\n pub fn ywyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 29)) }\n }\n\n #[inline]\n pub fn zwyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 30)) }\n }\n\n #[inline]\n pub fn wwyx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 31)) }\n }\n\n #[inline]\n pub fn xxzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 32)) }\n }\n\n #[inline]\n pub fn yxzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 33)) }\n }\n\n #[inline]\n pub fn zxzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 34)) }\n }\n\n #[inline]\n pub fn wxzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 35)) }\n }\n\n #[inline]\n pub fn xyzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 36)) }\n }\n\n #[inline]\n pub fn yyzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 37)) }\n }\n\n #[inline]\n pub fn zyzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 38)) }\n }\n\n #[inline]\n pub fn wyzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 39)) }\n }\n\n #[inline]\n pub fn xzzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 40)) }\n }\n\n #[inline]\n pub fn yzzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 41)) }\n }\n\n #[inline]\n pub fn zzzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 42)) }\n }\n\n #[inline]\n pub fn wzzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 43)) }\n }\n\n #[inline]\n pub fn xwzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 44)) }\n }\n\n #[inline]\n pub fn ywzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 45)) }\n }\n\n #[inline]\n pub fn zwzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 46)) }\n }\n\n #[inline]\n pub fn wwzx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 47)) }\n }\n\n #[inline]\n pub fn xxwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 48)) }\n }\n\n #[inline]\n pub fn yxwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 49)) }\n }\n\n #[inline]\n pub fn zxwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 50)) }\n }\n\n #[inline]\n pub fn wxwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 51)) }\n }\n\n #[inline]\n pub fn xywx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 52)) }\n }\n\n #[inline]\n pub fn yywx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 53)) }\n }\n\n #[inline]\n pub fn zywx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 54)) }\n }\n\n #[inline]\n pub fn wywx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 55)) }\n }\n\n #[inline]\n pub fn xzwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 56)) }\n }\n\n #[inline]\n pub fn yzwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 57)) }\n }\n\n #[inline]\n pub fn zzwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 58)) }\n }\n\n #[inline]\n pub fn wzwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 59)) }\n }\n\n #[inline]\n pub fn xwwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 60)) }\n }\n\n #[inline]\n pub fn ywwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 61)) }\n }\n\n #[inline]\n pub fn zwwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 62)) }\n }\n\n #[inline]\n pub fn wwwx(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 63)) }\n }\n\n #[inline]\n pub fn xxxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 64)) }\n }\n\n #[inline]\n pub fn yxxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 65)) }\n }\n\n #[inline]\n pub fn zxxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 66)) }\n }\n\n #[inline]\n pub fn wxxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 67)) }\n }\n\n #[inline]\n pub fn xyxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 68)) }\n }\n\n #[inline]\n pub fn yyxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 69)) }\n }\n\n #[inline]\n pub fn zyxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 70)) }\n }\n\n #[inline]\n pub fn wyxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 71)) }\n }\n\n #[inline]\n pub fn xzxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 72)) }\n }\n\n #[inline]\n pub fn yzxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 73)) }\n }\n\n #[inline]\n pub fn zzxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 74)) }\n }\n\n #[inline]\n pub fn wzxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 75)) }\n }\n\n #[inline]\n pub fn xwxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 76)) }\n }\n\n #[inline]\n pub fn ywxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 77)) }\n }\n\n #[inline]\n pub fn zwxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 78)) }\n }\n\n #[inline]\n pub fn wwxy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 79)) }\n }\n\n #[inline]\n pub fn xxyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 80)) }\n }\n\n #[inline]\n pub fn yxyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 81)) }\n }\n\n #[inline]\n pub fn zxyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 82)) }\n }\n\n #[inline]\n pub fn wxyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 83)) }\n }\n\n #[inline]\n pub fn xyyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 84)) }\n }\n\n #[inline]\n pub fn yyyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 85)) }\n }\n\n #[inline]\n pub fn zyyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 86)) }\n }\n\n #[inline]\n pub fn wyyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 87)) }\n }\n\n #[inline]\n pub fn xzyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 88)) }\n }\n\n #[inline]\n pub fn yzyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 89)) }\n }\n\n #[inline]\n pub fn zzyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 90)) }\n }\n\n #[inline]\n pub fn wzyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 91)) }\n }\n\n #[inline]\n pub fn xwyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 92)) }\n }\n\n #[inline]\n pub fn ywyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 93)) }\n }\n\n #[inline]\n pub fn zwyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 94)) }\n }\n\n #[inline]\n pub fn wwyy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 95)) }\n }\n\n #[inline]\n pub fn xxzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 96)) }\n }\n\n #[inline]\n pub fn yxzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 97)) }\n }\n\n #[inline]\n pub fn zxzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 98)) }\n }\n\n #[inline]\n pub fn wxzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 99)) }\n }\n\n #[inline]\n pub fn xyzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 100)) }\n }\n\n #[inline]\n pub fn yyzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 101)) }\n }\n\n #[inline]\n pub fn zyzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 102)) }\n }\n\n #[inline]\n pub fn wyzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 103)) }\n }\n\n #[inline]\n pub fn xzzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 104)) }\n }\n\n #[inline]\n pub fn yzzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 105)) }\n }\n\n #[inline]\n pub fn zzzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 106)) }\n }\n\n #[inline]\n pub fn wzzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 107)) }\n }\n\n #[inline]\n pub fn xwzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 108)) }\n }\n\n #[inline]\n pub fn ywzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 109)) }\n }\n\n #[inline]\n pub fn zwzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 110)) }\n }\n\n #[inline]\n pub fn wwzy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 111)) }\n }\n\n #[inline]\n pub fn xxwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 112)) }\n }\n\n #[inline]\n pub fn yxwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 113)) }\n }\n\n #[inline]\n pub fn zxwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 114)) }\n }\n\n #[inline]\n pub fn wxwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 115)) }\n }\n\n #[inline]\n pub fn xywy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 116)) }\n }\n\n #[inline]\n pub fn yywy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 117)) }\n }\n\n #[inline]\n pub fn zywy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 118)) }\n }\n\n #[inline]\n pub fn wywy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 119)) }\n }\n\n #[inline]\n pub fn xzwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 120)) }\n }\n\n #[inline]\n pub fn yzwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 121)) }\n }\n\n #[inline]\n pub fn zzwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 122)) }\n }\n\n #[inline]\n pub fn wzwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 123)) }\n }\n\n #[inline]\n pub fn xwwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 124)) }\n }\n\n #[inline]\n pub fn ywwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 125)) }\n }\n\n #[inline]\n pub fn zwwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 126)) }\n }\n\n #[inline]\n pub fn wwwy(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 127)) }\n }\n\n #[inline]\n pub fn xxxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 128)) }\n }\n\n #[inline]\n pub fn yxxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 129)) }\n }\n\n #[inline]\n pub fn zxxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 130)) }\n }\n\n #[inline]\n pub fn wxxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 131)) }\n }\n\n #[inline]\n pub fn xyxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 132)) }\n }\n\n #[inline]\n pub fn yyxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 133)) }\n }\n\n #[inline]\n pub fn zyxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 134)) }\n }\n\n #[inline]\n pub fn wyxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 135)) }\n }\n\n #[inline]\n pub fn xzxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 136)) }\n }\n\n #[inline]\n pub fn yzxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 137)) }\n }\n\n #[inline]\n pub fn zzxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 138)) }\n }\n\n #[inline]\n pub fn wzxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 139)) }\n }\n\n #[inline]\n pub fn xwxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 140)) }\n }\n\n #[inline]\n pub fn ywxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 141)) }\n }\n\n #[inline]\n pub fn zwxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 142)) }\n }\n\n #[inline]\n pub fn wwxz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 143)) }\n }\n\n #[inline]\n pub fn xxyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 144)) }\n }\n\n #[inline]\n pub fn yxyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 145)) }\n }\n\n #[inline]\n pub fn zxyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 146)) }\n }\n\n #[inline]\n pub fn wxyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 147)) }\n }\n\n #[inline]\n pub fn xyyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 148)) }\n }\n\n #[inline]\n pub fn yyyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 149)) }\n }\n\n #[inline]\n pub fn zyyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 150)) }\n }\n\n #[inline]\n pub fn wyyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 151)) }\n }\n\n #[inline]\n pub fn xzyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 152)) }\n }\n\n #[inline]\n pub fn yzyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 153)) }\n }\n\n #[inline]\n pub fn zzyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 154)) }\n }\n\n #[inline]\n pub fn wzyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 155)) }\n }\n\n #[inline]\n pub fn xwyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 156)) }\n }\n\n #[inline]\n pub fn ywyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 157)) }\n }\n\n #[inline]\n pub fn zwyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 158)) }\n }\n\n #[inline]\n pub fn wwyz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 159)) }\n }\n\n #[inline]\n pub fn xxzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 160)) }\n }\n\n #[inline]\n pub fn yxzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 161)) }\n }\n\n #[inline]\n pub fn zxzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 162)) }\n }\n\n #[inline]\n pub fn wxzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 163)) }\n }\n\n #[inline]\n pub fn xyzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 164)) }\n }\n\n #[inline]\n pub fn yyzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 165)) }\n }\n\n #[inline]\n pub fn zyzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 166)) }\n }\n\n #[inline]\n pub fn wyzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 167)) }\n }\n\n #[inline]\n pub fn xzzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 168)) }\n }\n\n #[inline]\n pub fn yzzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 169)) }\n }\n\n #[inline]\n pub fn zzzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 170)) }\n }\n\n #[inline]\n pub fn wzzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 171)) }\n }\n\n #[inline]\n pub fn xwzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 172)) }\n }\n\n #[inline]\n pub fn ywzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 173)) }\n }\n\n #[inline]\n pub fn zwzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 174)) }\n }\n\n #[inline]\n pub fn wwzz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 175)) }\n }\n\n #[inline]\n pub fn xxwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 176)) }\n }\n\n #[inline]\n pub fn yxwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 177)) }\n }\n\n #[inline]\n pub fn zxwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 178)) }\n }\n\n #[inline]\n pub fn wxwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 179)) }\n }\n\n #[inline]\n pub fn xywz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 180)) }\n }\n\n #[inline]\n pub fn yywz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 181)) }\n }\n\n #[inline]\n pub fn zywz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 182)) }\n }\n\n #[inline]\n pub fn wywz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 183)) }\n }\n\n #[inline]\n pub fn xzwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 184)) }\n }\n\n #[inline]\n pub fn yzwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 185)) }\n }\n\n #[inline]\n pub fn zzwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 186)) }\n }\n\n #[inline]\n pub fn wzwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 187)) }\n }\n\n #[inline]\n pub fn xwwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 188)) }\n }\n\n #[inline]\n pub fn ywwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 189)) }\n }\n\n #[inline]\n pub fn zwwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 190)) }\n }\n\n #[inline]\n pub fn wwwz(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 191)) }\n }\n\n #[inline]\n pub fn xxxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 192)) }\n }\n\n #[inline]\n pub fn yxxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 193)) }\n }\n\n #[inline]\n pub fn zxxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 194)) }\n }\n\n #[inline]\n pub fn wxxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 195)) }\n }\n\n #[inline]\n pub fn xyxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 196)) }\n }\n\n #[inline]\n pub fn yyxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 197)) }\n }\n\n #[inline]\n pub fn zyxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 198)) }\n }\n\n #[inline]\n pub fn wyxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 199)) }\n }\n\n #[inline]\n pub fn xzxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 200)) }\n }\n\n #[inline]\n pub fn yzxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 201)) }\n }\n\n #[inline]\n pub fn zzxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 202)) }\n }\n\n #[inline]\n pub fn wzxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 203)) }\n }\n\n #[inline]\n pub fn xwxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 204)) }\n }\n\n #[inline]\n pub fn ywxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 205)) }\n }\n\n #[inline]\n pub fn zwxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 206)) }\n }\n\n #[inline]\n pub fn wwxw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 207)) }\n }\n\n #[inline]\n pub fn xxyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 208)) }\n }\n\n #[inline]\n pub fn yxyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 209)) }\n }\n\n #[inline]\n pub fn zxyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 210)) }\n }\n\n #[inline]\n pub fn wxyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 211)) }\n }\n\n #[inline]\n pub fn xyyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 212)) }\n }\n\n #[inline]\n pub fn yyyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 213)) }\n }\n\n #[inline]\n pub fn zyyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 214)) }\n }\n\n #[inline]\n pub fn wyyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 215)) }\n }\n\n #[inline]\n pub fn xzyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 216)) }\n }\n\n #[inline]\n pub fn yzyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 217)) }\n }\n\n #[inline]\n pub fn zzyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 218)) }\n }\n\n #[inline]\n pub fn wzyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 219)) }\n }\n\n #[inline]\n pub fn xwyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 220)) }\n }\n\n #[inline]\n pub fn ywyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 221)) }\n }\n\n #[inline]\n pub fn zwyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 222)) }\n }\n\n #[inline]\n pub fn wwyw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 223)) }\n }\n\n #[inline]\n pub fn xxzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 224)) }\n }\n\n #[inline]\n pub fn yxzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 225)) }\n }\n\n #[inline]\n pub fn zxzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 226)) }\n }\n\n #[inline]\n pub fn wxzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 227)) }\n }\n\n #[inline]\n pub fn xyzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 228)) }\n }\n\n #[inline]\n pub fn yyzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 229)) }\n }\n\n #[inline]\n pub fn zyzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 230)) }\n }\n\n #[inline]\n pub fn wyzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 231)) }\n }\n\n #[inline]\n pub fn xzzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 232)) }\n }\n\n #[inline]\n pub fn yzzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 233)) }\n }\n\n #[inline]\n pub fn zzzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 234)) }\n }\n\n #[inline]\n pub fn wzzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 235)) }\n }\n\n #[inline]\n pub fn xwzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 236)) }\n }\n\n #[inline]\n pub fn ywzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 237)) }\n }\n\n #[inline]\n pub fn zwzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 238)) }\n }\n\n #[inline]\n pub fn wwzw(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 239)) }\n }\n\n #[inline]\n pub fn xxww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 240)) }\n }\n\n #[inline]\n pub fn yxww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 241)) }\n }\n\n #[inline]\n pub fn zxww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 242)) }\n }\n\n #[inline]\n pub fn wxww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 243)) }\n }\n\n #[inline]\n pub fn xyww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 244)) }\n }\n\n #[inline]\n pub fn yyww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 245)) }\n }\n\n #[inline]\n pub fn zyww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 246)) }\n }\n\n #[inline]\n pub fn wyww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 247)) }\n }\n\n #[inline]\n pub fn xzww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 248)) }\n }\n\n #[inline]\n pub fn yzww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 249)) }\n }\n\n #[inline]\n pub fn zzww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 250)) }\n }\n\n #[inline]\n pub fn wzww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 251)) }\n }\n\n #[inline]\n pub fn xwww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 252)) }\n }\n\n #[inline]\n pub fn ywww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 253)) }\n }\n\n #[inline]\n pub fn zwww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 254)) }\n }\n\n #[inline]\n pub fn wwww(self) -> F32x4 {\n unsafe { F32x4(x86::_mm_shuffle_ps(self.0, self.0, 255)) }\n }\n}\n" }, { "package": "pathfinder_simd@0.5.6", "sourcePath": "src/x86/swizzle_i32x4.rs", "sha256": "000ff6f0fb425f439b2d5ee0afd128b97873b4bb207b3bf6b30ec5e4466111ea", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// pathfinder/simd/src/x86/swizzle_i32x4.rs\n//\n// Copyright © 2019 The Pathfinder Project Developers.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::x86::I32x4;\n\n#[cfg(target_pointer_width = \"32\")]\nuse std::arch::x86;\n#[cfg(target_pointer_width = \"64\")]\nuse std::arch::x86_64 as x86;\n\nimpl I32x4 {\n #[inline]\n pub fn xxxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 0,\n )))\n }\n }\n\n #[inline]\n pub fn yxxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 1,\n )))\n }\n }\n\n #[inline]\n pub fn zxxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 2,\n )))\n }\n }\n\n #[inline]\n pub fn wxxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 3,\n )))\n }\n }\n\n #[inline]\n pub fn xyxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 4,\n )))\n }\n }\n\n #[inline]\n pub fn yyxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 5,\n )))\n }\n }\n\n #[inline]\n pub fn zyxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 6,\n )))\n }\n }\n\n #[inline]\n pub fn wyxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 7,\n )))\n }\n }\n\n #[inline]\n pub fn xzxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 8,\n )))\n }\n }\n\n #[inline]\n pub fn yzxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 9,\n )))\n }\n }\n\n #[inline]\n pub fn zzxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 10,\n )))\n }\n }\n\n #[inline]\n pub fn wzxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 11,\n )))\n }\n }\n\n #[inline]\n pub fn xwxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 12,\n )))\n }\n }\n\n #[inline]\n pub fn ywxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 13,\n )))\n }\n }\n\n #[inline]\n pub fn zwxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 14,\n )))\n }\n }\n\n #[inline]\n pub fn wwxx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 15,\n )))\n }\n }\n\n #[inline]\n pub fn xxyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 16,\n )))\n }\n }\n\n #[inline]\n pub fn yxyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 17,\n )))\n }\n }\n\n #[inline]\n pub fn zxyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 18,\n )))\n }\n }\n\n #[inline]\n pub fn wxyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 19,\n )))\n }\n }\n\n #[inline]\n pub fn xyyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 20,\n )))\n }\n }\n\n #[inline]\n pub fn yyyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 21,\n )))\n }\n }\n\n #[inline]\n pub fn zyyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 22,\n )))\n }\n }\n\n #[inline]\n pub fn wyyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 23,\n )))\n }\n }\n\n #[inline]\n pub fn xzyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 24,\n )))\n }\n }\n\n #[inline]\n pub fn yzyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 25,\n )))\n }\n }\n\n #[inline]\n pub fn zzyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 26,\n )))\n }\n }\n\n #[inline]\n pub fn wzyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 27,\n )))\n }\n }\n\n #[inline]\n pub fn xwyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 28,\n )))\n }\n }\n\n #[inline]\n pub fn ywyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 29,\n )))\n }\n }\n\n #[inline]\n pub fn zwyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 30,\n )))\n }\n }\n\n #[inline]\n pub fn wwyx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 31,\n )))\n }\n }\n\n #[inline]\n pub fn xxzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 32,\n )))\n }\n }\n\n #[inline]\n pub fn yxzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 33,\n )))\n }\n }\n\n #[inline]\n pub fn zxzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 34,\n )))\n }\n }\n\n #[inline]\n pub fn wxzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 35,\n )))\n }\n }\n\n #[inline]\n pub fn xyzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 36,\n )))\n }\n }\n\n #[inline]\n pub fn yyzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 37,\n )))\n }\n }\n\n #[inline]\n pub fn zyzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 38,\n )))\n }\n }\n\n #[inline]\n pub fn wyzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 39,\n )))\n }\n }\n\n #[inline]\n pub fn xzzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 40,\n )))\n }\n }\n\n #[inline]\n pub fn yzzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 41,\n )))\n }\n }\n\n #[inline]\n pub fn zzzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 42,\n )))\n }\n }\n\n #[inline]\n pub fn wzzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 43,\n )))\n }\n }\n\n #[inline]\n pub fn xwzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 44,\n )))\n }\n }\n\n #[inline]\n pub fn ywzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 45,\n )))\n }\n }\n\n #[inline]\n pub fn zwzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 46,\n )))\n }\n }\n\n #[inline]\n pub fn wwzx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 47,\n )))\n }\n }\n\n #[inline]\n pub fn xxwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 48,\n )))\n }\n }\n\n #[inline]\n pub fn yxwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 49,\n )))\n }\n }\n\n #[inline]\n pub fn zxwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 50,\n )))\n }\n }\n\n #[inline]\n pub fn wxwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 51,\n )))\n }\n }\n\n #[inline]\n pub fn xywx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 52,\n )))\n }\n }\n\n #[inline]\n pub fn yywx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 53,\n )))\n }\n }\n\n #[inline]\n pub fn zywx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 54,\n )))\n }\n }\n\n #[inline]\n pub fn wywx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 55,\n )))\n }\n }\n\n #[inline]\n pub fn xzwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 56,\n )))\n }\n }\n\n #[inline]\n pub fn yzwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 57,\n )))\n }\n }\n\n #[inline]\n pub fn zzwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 58,\n )))\n }\n }\n\n #[inline]\n pub fn wzwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 59,\n )))\n }\n }\n\n #[inline]\n pub fn xwwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 60,\n )))\n }\n }\n\n #[inline]\n pub fn ywwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 61,\n )))\n }\n }\n\n #[inline]\n pub fn zwwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 62,\n )))\n }\n }\n\n #[inline]\n pub fn wwwx(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 63,\n )))\n }\n }\n\n #[inline]\n pub fn xxxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 64,\n )))\n }\n }\n\n #[inline]\n pub fn yxxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 65,\n )))\n }\n }\n\n #[inline]\n pub fn zxxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 66,\n )))\n }\n }\n\n #[inline]\n pub fn wxxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 67,\n )))\n }\n }\n\n #[inline]\n pub fn xyxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 68,\n )))\n }\n }\n\n #[inline]\n pub fn yyxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 69,\n )))\n }\n }\n\n #[inline]\n pub fn zyxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 70,\n )))\n }\n }\n\n #[inline]\n pub fn wyxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 71,\n )))\n }\n }\n\n #[inline]\n pub fn xzxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 72,\n )))\n }\n }\n\n #[inline]\n pub fn yzxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 73,\n )))\n }\n }\n\n #[inline]\n pub fn zzxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 74,\n )))\n }\n }\n\n #[inline]\n pub fn wzxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 75,\n )))\n }\n }\n\n #[inline]\n pub fn xwxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 76,\n )))\n }\n }\n\n #[inline]\n pub fn ywxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 77,\n )))\n }\n }\n\n #[inline]\n pub fn zwxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 78,\n )))\n }\n }\n\n #[inline]\n pub fn wwxy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 79,\n )))\n }\n }\n\n #[inline]\n pub fn xxyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 80,\n )))\n }\n }\n\n #[inline]\n pub fn yxyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 81,\n )))\n }\n }\n\n #[inline]\n pub fn zxyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 82,\n )))\n }\n }\n\n #[inline]\n pub fn wxyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 83,\n )))\n }\n }\n\n #[inline]\n pub fn xyyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 84,\n )))\n }\n }\n\n #[inline]\n pub fn yyyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 85,\n )))\n }\n }\n\n #[inline]\n pub fn zyyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 86,\n )))\n }\n }\n\n #[inline]\n pub fn wyyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 87,\n )))\n }\n }\n\n #[inline]\n pub fn xzyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 88,\n )))\n }\n }\n\n #[inline]\n pub fn yzyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 89,\n )))\n }\n }\n\n #[inline]\n pub fn zzyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 90,\n )))\n }\n }\n\n #[inline]\n pub fn wzyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 91,\n )))\n }\n }\n\n #[inline]\n pub fn xwyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 92,\n )))\n }\n }\n\n #[inline]\n pub fn ywyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 93,\n )))\n }\n }\n\n #[inline]\n pub fn zwyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 94,\n )))\n }\n }\n\n #[inline]\n pub fn wwyy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 95,\n )))\n }\n }\n\n #[inline]\n pub fn xxzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 96,\n )))\n }\n }\n\n #[inline]\n pub fn yxzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 97,\n )))\n }\n }\n\n #[inline]\n pub fn zxzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 98,\n )))\n }\n }\n\n #[inline]\n pub fn wxzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 99,\n )))\n }\n }\n\n #[inline]\n pub fn xyzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 100,\n )))\n }\n }\n\n #[inline]\n pub fn yyzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 101,\n )))\n }\n }\n\n #[inline]\n pub fn zyzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 102,\n )))\n }\n }\n\n #[inline]\n pub fn wyzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 103,\n )))\n }\n }\n\n #[inline]\n pub fn xzzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 104,\n )))\n }\n }\n\n #[inline]\n pub fn yzzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 105,\n )))\n }\n }\n\n #[inline]\n pub fn zzzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 106,\n )))\n }\n }\n\n #[inline]\n pub fn wzzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 107,\n )))\n }\n }\n\n #[inline]\n pub fn xwzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 108,\n )))\n }\n }\n\n #[inline]\n pub fn ywzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 109,\n )))\n }\n }\n\n #[inline]\n pub fn zwzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 110,\n )))\n }\n }\n\n #[inline]\n pub fn wwzy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 111,\n )))\n }\n }\n\n #[inline]\n pub fn xxwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 112,\n )))\n }\n }\n\n #[inline]\n pub fn yxwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 113,\n )))\n }\n }\n\n #[inline]\n pub fn zxwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 114,\n )))\n }\n }\n\n #[inline]\n pub fn wxwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 115,\n )))\n }\n }\n\n #[inline]\n pub fn xywy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 116,\n )))\n }\n }\n\n #[inline]\n pub fn yywy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 117,\n )))\n }\n }\n\n #[inline]\n pub fn zywy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 118,\n )))\n }\n }\n\n #[inline]\n pub fn wywy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 119,\n )))\n }\n }\n\n #[inline]\n pub fn xzwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 120,\n )))\n }\n }\n\n #[inline]\n pub fn yzwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 121,\n )))\n }\n }\n\n #[inline]\n pub fn zzwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 122,\n )))\n }\n }\n\n #[inline]\n pub fn wzwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 123,\n )))\n }\n }\n\n #[inline]\n pub fn xwwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 124,\n )))\n }\n }\n\n #[inline]\n pub fn ywwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 125,\n )))\n }\n }\n\n #[inline]\n pub fn zwwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 126,\n )))\n }\n }\n\n #[inline]\n pub fn wwwy(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 127,\n )))\n }\n }\n\n #[inline]\n pub fn xxxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 128,\n )))\n }\n }\n\n #[inline]\n pub fn yxxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 129,\n )))\n }\n }\n\n #[inline]\n pub fn zxxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 130,\n )))\n }\n }\n\n #[inline]\n pub fn wxxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 131,\n )))\n }\n }\n\n #[inline]\n pub fn xyxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 132,\n )))\n }\n }\n\n #[inline]\n pub fn yyxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 133,\n )))\n }\n }\n\n #[inline]\n pub fn zyxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 134,\n )))\n }\n }\n\n #[inline]\n pub fn wyxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 135,\n )))\n }\n }\n\n #[inline]\n pub fn xzxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 136,\n )))\n }\n }\n\n #[inline]\n pub fn yzxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 137,\n )))\n }\n }\n\n #[inline]\n pub fn zzxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 138,\n )))\n }\n }\n\n #[inline]\n pub fn wzxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 139,\n )))\n }\n }\n\n #[inline]\n pub fn xwxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 140,\n )))\n }\n }\n\n #[inline]\n pub fn ywxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 141,\n )))\n }\n }\n\n #[inline]\n pub fn zwxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 142,\n )))\n }\n }\n\n #[inline]\n pub fn wwxz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 143,\n )))\n }\n }\n\n #[inline]\n pub fn xxyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 144,\n )))\n }\n }\n\n #[inline]\n pub fn yxyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 145,\n )))\n }\n }\n\n #[inline]\n pub fn zxyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 146,\n )))\n }\n }\n\n #[inline]\n pub fn wxyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 147,\n )))\n }\n }\n\n #[inline]\n pub fn xyyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 148,\n )))\n }\n }\n\n #[inline]\n pub fn yyyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 149,\n )))\n }\n }\n\n #[inline]\n pub fn zyyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 150,\n )))\n }\n }\n\n #[inline]\n pub fn wyyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 151,\n )))\n }\n }\n\n #[inline]\n pub fn xzyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 152,\n )))\n }\n }\n\n #[inline]\n pub fn yzyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 153,\n )))\n }\n }\n\n #[inline]\n pub fn zzyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 154,\n )))\n }\n }\n\n #[inline]\n pub fn wzyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 155,\n )))\n }\n }\n\n #[inline]\n pub fn xwyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 156,\n )))\n }\n }\n\n #[inline]\n pub fn ywyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 157,\n )))\n }\n }\n\n #[inline]\n pub fn zwyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 158,\n )))\n }\n }\n\n #[inline]\n pub fn wwyz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 159,\n )))\n }\n }\n\n #[inline]\n pub fn xxzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 160,\n )))\n }\n }\n\n #[inline]\n pub fn yxzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 161,\n )))\n }\n }\n\n #[inline]\n pub fn zxzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 162,\n )))\n }\n }\n\n #[inline]\n pub fn wxzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 163,\n )))\n }\n }\n\n #[inline]\n pub fn xyzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 164,\n )))\n }\n }\n\n #[inline]\n pub fn yyzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 165,\n )))\n }\n }\n\n #[inline]\n pub fn zyzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 166,\n )))\n }\n }\n\n #[inline]\n pub fn wyzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 167,\n )))\n }\n }\n\n #[inline]\n pub fn xzzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 168,\n )))\n }\n }\n\n #[inline]\n pub fn yzzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 169,\n )))\n }\n }\n\n #[inline]\n pub fn zzzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 170,\n )))\n }\n }\n\n #[inline]\n pub fn wzzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 171,\n )))\n }\n }\n\n #[inline]\n pub fn xwzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 172,\n )))\n }\n }\n\n #[inline]\n pub fn ywzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 173,\n )))\n }\n }\n\n #[inline]\n pub fn zwzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 174,\n )))\n }\n }\n\n #[inline]\n pub fn wwzz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 175,\n )))\n }\n }\n\n #[inline]\n pub fn xxwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 176,\n )))\n }\n }\n\n #[inline]\n pub fn yxwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 177,\n )))\n }\n }\n\n #[inline]\n pub fn zxwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 178,\n )))\n }\n }\n\n #[inline]\n pub fn wxwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 179,\n )))\n }\n }\n\n #[inline]\n pub fn xywz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 180,\n )))\n }\n }\n\n #[inline]\n pub fn yywz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 181,\n )))\n }\n }\n\n #[inline]\n pub fn zywz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 182,\n )))\n }\n }\n\n #[inline]\n pub fn wywz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 183,\n )))\n }\n }\n\n #[inline]\n pub fn xzwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 184,\n )))\n }\n }\n\n #[inline]\n pub fn yzwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 185,\n )))\n }\n }\n\n #[inline]\n pub fn zzwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 186,\n )))\n }\n }\n\n #[inline]\n pub fn wzwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 187,\n )))\n }\n }\n\n #[inline]\n pub fn xwwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 188,\n )))\n }\n }\n\n #[inline]\n pub fn ywwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 189,\n )))\n }\n }\n\n #[inline]\n pub fn zwwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 190,\n )))\n }\n }\n\n #[inline]\n pub fn wwwz(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 191,\n )))\n }\n }\n\n #[inline]\n pub fn xxxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 192,\n )))\n }\n }\n\n #[inline]\n pub fn yxxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 193,\n )))\n }\n }\n\n #[inline]\n pub fn zxxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 194,\n )))\n }\n }\n\n #[inline]\n pub fn wxxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 195,\n )))\n }\n }\n\n #[inline]\n pub fn xyxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 196,\n )))\n }\n }\n\n #[inline]\n pub fn yyxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 197,\n )))\n }\n }\n\n #[inline]\n pub fn zyxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 198,\n )))\n }\n }\n\n #[inline]\n pub fn wyxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 199,\n )))\n }\n }\n\n #[inline]\n pub fn xzxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 200,\n )))\n }\n }\n\n #[inline]\n pub fn yzxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 201,\n )))\n }\n }\n\n #[inline]\n pub fn zzxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 202,\n )))\n }\n }\n\n #[inline]\n pub fn wzxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 203,\n )))\n }\n }\n\n #[inline]\n pub fn xwxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 204,\n )))\n }\n }\n\n #[inline]\n pub fn ywxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 205,\n )))\n }\n }\n\n #[inline]\n pub fn zwxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 206,\n )))\n }\n }\n\n #[inline]\n pub fn wwxw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 207,\n )))\n }\n }\n\n #[inline]\n pub fn xxyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 208,\n )))\n }\n }\n\n #[inline]\n pub fn yxyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 209,\n )))\n }\n }\n\n #[inline]\n pub fn zxyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 210,\n )))\n }\n }\n\n #[inline]\n pub fn wxyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 211,\n )))\n }\n }\n\n #[inline]\n pub fn xyyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 212,\n )))\n }\n }\n\n #[inline]\n pub fn yyyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 213,\n )))\n }\n }\n\n #[inline]\n pub fn zyyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 214,\n )))\n }\n }\n\n #[inline]\n pub fn wyyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 215,\n )))\n }\n }\n\n #[inline]\n pub fn xzyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 216,\n )))\n }\n }\n\n #[inline]\n pub fn yzyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 217,\n )))\n }\n }\n\n #[inline]\n pub fn zzyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 218,\n )))\n }\n }\n\n #[inline]\n pub fn wzyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 219,\n )))\n }\n }\n\n #[inline]\n pub fn xwyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 220,\n )))\n }\n }\n\n #[inline]\n pub fn ywyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 221,\n )))\n }\n }\n\n #[inline]\n pub fn zwyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 222,\n )))\n }\n }\n\n #[inline]\n pub fn wwyw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 223,\n )))\n }\n }\n\n #[inline]\n pub fn xxzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 224,\n )))\n }\n }\n\n #[inline]\n pub fn yxzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 225,\n )))\n }\n }\n\n #[inline]\n pub fn zxzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 226,\n )))\n }\n }\n\n #[inline]\n pub fn wxzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 227,\n )))\n }\n }\n\n #[inline]\n pub fn xyzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 228,\n )))\n }\n }\n\n #[inline]\n pub fn yyzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 229,\n )))\n }\n }\n\n #[inline]\n pub fn zyzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 230,\n )))\n }\n }\n\n #[inline]\n pub fn wyzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 231,\n )))\n }\n }\n\n #[inline]\n pub fn xzzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 232,\n )))\n }\n }\n\n #[inline]\n pub fn yzzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 233,\n )))\n }\n }\n\n #[inline]\n pub fn zzzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 234,\n )))\n }\n }\n\n #[inline]\n pub fn wzzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 235,\n )))\n }\n }\n\n #[inline]\n pub fn xwzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 236,\n )))\n }\n }\n\n #[inline]\n pub fn ywzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 237,\n )))\n }\n }\n\n #[inline]\n pub fn zwzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 238,\n )))\n }\n }\n\n #[inline]\n pub fn wwzw(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 239,\n )))\n }\n }\n\n #[inline]\n pub fn xxww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 240,\n )))\n }\n }\n\n #[inline]\n pub fn yxww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 241,\n )))\n }\n }\n\n #[inline]\n pub fn zxww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 242,\n )))\n }\n }\n\n #[inline]\n pub fn wxww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 243,\n )))\n }\n }\n\n #[inline]\n pub fn xyww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 244,\n )))\n }\n }\n\n #[inline]\n pub fn yyww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 245,\n )))\n }\n }\n\n #[inline]\n pub fn zyww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 246,\n )))\n }\n }\n\n #[inline]\n pub fn wyww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 247,\n )))\n }\n }\n\n #[inline]\n pub fn xzww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 248,\n )))\n }\n }\n\n #[inline]\n pub fn yzww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 249,\n )))\n }\n }\n\n #[inline]\n pub fn zzww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 250,\n )))\n }\n }\n\n #[inline]\n pub fn wzww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 251,\n )))\n }\n }\n\n #[inline]\n pub fn xwww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 252,\n )))\n }\n }\n\n #[inline]\n pub fn ywww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 253,\n )))\n }\n }\n\n #[inline]\n pub fn zwww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 254,\n )))\n }\n }\n\n #[inline]\n pub fn wwww(self) -> I32x4 {\n unsafe {\n let this = x86::_mm_castsi128_ps(self.0);\n I32x4(x86::_mm_castps_si128(x86::_mm_shuffle_ps(\n this, this, 255,\n )))\n }\n }\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "README.md", "sha256": "a6221fce50cc09c13b03a7973fb1d4dbf59ce7f46ca06830bbd46d201ed3b2d8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# pin-project-lite\n\n[![crates.io](https://img.shields.io/crates/v/pin-project-lite?style=flat-square&logo=rust)](https://crates.io/crates/pin-project-lite)\n[![docs.rs](https://img.shields.io/badge/docs.rs-pin--project--lite-blue?style=flat-square&logo=docs.rs)](https://docs.rs/pin-project-lite)\n[![license](https://img.shields.io/badge/license-Apache--2.0_OR_MIT-blue?style=flat-square)](#license)\n[![msrv](https://img.shields.io/badge/msrv-1.37-blue?style=flat-square&logo=rust)](https://www.rust-lang.org)\n[![github actions](https://img.shields.io/github/actions/workflow/status/taiki-e/pin-project-lite/ci.yml?branch=main&style=flat-square&logo=github)](https://github.com/taiki-e/pin-project-lite/actions)\n\n\n\nA lightweight version of [pin-project] written with declarative macros.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\npin-project-lite = \"0.2\"\n```\n\n## Examples\n\n[`pin_project!`] macro creates a projection type covering all the fields of\nstruct.\n\n```rust\nuse std::pin::Pin;\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n struct Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n }\n}\n\nimpl Struct {\n fn method(self: Pin<&mut Self>) {\n let this = self.project();\n let _: Pin<&mut T> = this.pinned; // Pinned reference to the field\n let _: &mut U = this.unpinned; // Normal reference to the field\n }\n}\n```\n\nTo use [`pin_project!`] on enums, you need to name the projection type\nreturned from the method.\n\n```rust\nuse std::pin::Pin;\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project = EnumProj]\n enum Enum {\n Variant { #[pin] pinned: T, unpinned: U },\n }\n}\n\nimpl Enum {\n fn method(self: Pin<&mut Self>) {\n match self.project() {\n EnumProj::Variant { pinned, unpinned } => {\n let _: Pin<&mut T> = pinned;\n let _: &mut U = unpinned;\n }\n }\n }\n}\n```\n\n## [pin-project] vs pin-project-lite\n\nHere are some similarities and differences compared to [pin-project].\n\n### Similar: Safety\n\npin-project-lite guarantees safety in much the same way as [pin-project].\nBoth are completely safe unless you write other unsafe code.\n\n### Different: Minimal design\n\nThis library does not tackle as expansive of a range of use cases as\n[pin-project] does. If your use case is not already covered, please use\n[pin-project].\n\n### Different: No proc-macro related dependencies\n\nThis is the **only** reason to use this crate. However, **if you already\nhave proc-macro related dependencies in your crate's dependency graph, there\nis no benefit from using this crate.** (Note: There is almost no difference\nin the amount of code generated between [pin-project] and pin-project-lite.)\n\n### Different: No useful error messages\n\nThis macro does not handle any invalid input. So error messages are not to\nbe useful in most cases. If you do need useful error messages, then upon\nerror you can pass the same input to [pin-project] to receive a helpful\ndescription of the compile error.\n\n### Different: No support for custom Unpin implementation\n\npin-project supports this by [`UnsafeUnpin`][unsafe-unpin]. (`!Unpin` is supported by both [pin-project][not-unpin] and [pin-project-lite][not-unpin-lite].)\n\n### Different: No support for tuple structs and tuple variants\n\npin-project supports this.\n\n[not-unpin]: https://docs.rs/pin-project/latest/pin_project/attr.pin_project.html#unpin\n[pin-project]: https://github.com/taiki-e/pin-project\n[unsafe-unpin]: https://docs.rs/pin-project/latest/pin_project/attr.pin_project.html#unsafeunpin\n\n\n\n[not-unpin-lite]: https://docs.rs/pin-project-lite/latest/pin_project_lite/macro.pin_project.html#unpin\n[`pin_project!`]: https://docs.rs/pin-project-lite/latest/pin_project_lite/macro.pin_project.html\n\n## License\n\nLicensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or\n[MIT license](LICENSE-MIT) at your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "src/lib.rs", "sha256": "09a84ff233021dabb75200ef71ebcb1c5a47c94484ce294ef5dcf43797873ba4", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*!\n\n\n\nA lightweight version of [pin-project] written with declarative macros.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\npin-project-lite = \"0.2\"\n```\n\n## Examples\n\n[`pin_project!`] macro creates a projection type covering all the fields of\nstruct.\n\n```\nuse std::pin::Pin;\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n struct Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n }\n}\n\nimpl Struct {\n fn method(self: Pin<&mut Self>) {\n let this = self.project();\n let _: Pin<&mut T> = this.pinned; // Pinned reference to the field\n let _: &mut U = this.unpinned; // Normal reference to the field\n }\n}\n```\n\nTo use [`pin_project!`] on enums, you need to name the projection type\nreturned from the method.\n\n```\nuse std::pin::Pin;\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project = EnumProj]\n enum Enum {\n Variant { #[pin] pinned: T, unpinned: U },\n }\n}\n\nimpl Enum {\n fn method(self: Pin<&mut Self>) {\n match self.project() {\n EnumProj::Variant { pinned, unpinned } => {\n let _: Pin<&mut T> = pinned;\n let _: &mut U = unpinned;\n }\n }\n }\n}\n```\n\n## [pin-project] vs pin-project-lite\n\nHere are some similarities and differences compared to [pin-project].\n\n### Similar: Safety\n\npin-project-lite guarantees safety in much the same way as [pin-project].\nBoth are completely safe unless you write other unsafe code.\n\n### Different: Minimal design\n\nThis library does not tackle as expansive of a range of use cases as\n[pin-project] does. If your use case is not already covered, please use\n[pin-project].\n\n### Different: No proc-macro related dependencies\n\nThis is the **only** reason to use this crate. However, **if you already\nhave proc-macro related dependencies in your crate's dependency graph, there\nis no benefit from using this crate.** (Note: There is almost no difference\nin the amount of code generated between [pin-project] and pin-project-lite.)\n\n### Different: No useful error messages\n\nThis macro does not handle any invalid input. So error messages are not to\nbe useful in most cases. If you do need useful error messages, then upon\nerror you can pass the same input to [pin-project] to receive a helpful\ndescription of the compile error.\n\n### Different: No support for custom Unpin implementation\n\npin-project supports this by [`UnsafeUnpin`][unsafe-unpin]. (`!Unpin` is supported by both [pin-project][not-unpin] and [pin-project-lite][not-unpin-lite].)\n\n### Different: No support for tuple structs and tuple variants\n\npin-project supports this.\n\n[not-unpin]: https://docs.rs/pin-project/latest/pin_project/attr.pin_project.html#unpin\n[pin-project]: https://github.com/taiki-e/pin-project\n[unsafe-unpin]: https://docs.rs/pin-project/latest/pin_project/attr.pin_project.html#unsafeunpin\n\n\n\n[not-unpin-lite]: pin_project#unpin\n*/\n\n#![no_std]\n#![doc(test(\n no_crate_inject,\n attr(allow(\n dead_code,\n unused_variables,\n clippy::undocumented_unsafe_blocks,\n clippy::unused_trait_names,\n ))\n))]\n// #![warn(unsafe_op_in_unsafe_fn)] // requires Rust 1.52\n#![warn(\n // Lints that may help when writing public library.\n missing_debug_implementations,\n missing_docs,\n clippy::alloc_instead_of_core,\n clippy::exhaustive_enums,\n clippy::exhaustive_structs,\n clippy::impl_trait_in_params,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n // clippy::missing_inline_in_public_items,\n)]\n\n/// A macro that creates a projection type covering all the fields of struct.\n///\n/// This macro creates a projection type according to the following rules:\n///\n/// - For the field that uses `#[pin]` attribute, makes the pinned reference to the field.\n/// - For the other fields, makes the unpinned reference to the field.\n///\n/// And the following methods are implemented on the original type:\n///\n/// ```\n/// # use std::pin::Pin;\n/// # type Projection<'a> = &'a ();\n/// # type ProjectionRef<'a> = &'a ();\n/// # trait Dox {\n/// fn project(self: Pin<&mut Self>) -> Projection<'_>;\n/// fn project_ref(self: Pin<&Self>) -> ProjectionRef<'_>;\n/// # }\n/// ```\n///\n/// By passing an attribute with the same name as the method to the macro,\n/// you can name the projection type returned from the method. This allows you\n/// to use pattern matching on the projected types.\n///\n/// ```\n/// # use pin_project_lite::pin_project;\n/// # use std::pin::Pin;\n/// pin_project! {\n/// #[project = EnumProj]\n/// enum Enum {\n/// Variant { #[pin] field: T },\n/// }\n/// }\n///\n/// impl Enum {\n/// fn method(self: Pin<&mut Self>) {\n/// let this: EnumProj<'_, T> = self.project();\n/// match this {\n/// EnumProj::Variant { field } => {\n/// let _: Pin<&mut T> = field;\n/// }\n/// }\n/// }\n/// }\n/// ```\n///\n/// By passing the `#[project_replace = MyProjReplace]` attribute you may create an additional\n/// method which allows the contents of `Pin<&mut Self>` to be replaced while simultaneously moving\n/// out all unpinned fields in `Self`.\n///\n/// ```\n/// # use std::pin::Pin;\n/// # type MyProjReplace = ();\n/// # trait Dox {\n/// fn project_replace(self: Pin<&mut Self>, replacement: Self) -> MyProjReplace;\n/// # }\n/// ```\n///\n/// Also, note that the projection types returned by `project` and `project_ref` have\n/// an additional lifetime at the beginning of generics.\n///\n/// ```text\n/// let this: EnumProj<'_, T> = self.project();\n/// ^^\n/// ```\n///\n/// The visibility of the projected types and projection methods is based on the\n/// original type. However, if the visibility of the original type is `pub`, the\n/// visibility of the projected types and the projection methods is downgraded\n/// to `pub(crate)`.\n///\n/// # Safety\n///\n/// `pin_project!` macro guarantees safety in much the same way as [pin-project] crate.\n/// Both are completely safe unless you write other unsafe code.\n///\n/// See [pin-project] crate for more details.\n///\n/// # Examples\n///\n/// ```\n/// use std::pin::Pin;\n///\n/// use pin_project_lite::pin_project;\n///\n/// pin_project! {\n/// struct Struct {\n/// #[pin]\n/// pinned: T,\n/// unpinned: U,\n/// }\n/// }\n///\n/// impl Struct {\n/// fn method(self: Pin<&mut Self>) {\n/// let this = self.project();\n/// let _: Pin<&mut T> = this.pinned; // Pinned reference to the field\n/// let _: &mut U = this.unpinned; // Normal reference to the field\n/// }\n/// }\n/// ```\n///\n/// To use `pin_project!` on enums, you need to name the projection type\n/// returned from the method.\n///\n/// ```\n/// use std::pin::Pin;\n///\n/// use pin_project_lite::pin_project;\n///\n/// pin_project! {\n/// #[project = EnumProj]\n/// enum Enum {\n/// Struct {\n/// #[pin]\n/// field: T,\n/// },\n/// Unit,\n/// }\n/// }\n///\n/// impl Enum {\n/// fn method(self: Pin<&mut Self>) {\n/// match self.project() {\n/// EnumProj::Struct { field } => {\n/// let _: Pin<&mut T> = field;\n/// }\n/// EnumProj::Unit => {}\n/// }\n/// }\n/// }\n/// ```\n///\n/// If you want to call the `project()` method multiple times or later use the\n/// original [`Pin`] type, it needs to use [`.as_mut()`][`Pin::as_mut`] to avoid\n/// consuming the [`Pin`].\n///\n/// ```\n/// use std::pin::Pin;\n///\n/// use pin_project_lite::pin_project;\n///\n/// pin_project! {\n/// struct Struct {\n/// #[pin]\n/// field: T,\n/// }\n/// }\n///\n/// impl Struct {\n/// fn call_project_twice(mut self: Pin<&mut Self>) {\n/// // `project` consumes `self`, so reborrow the `Pin<&mut Self>` via `as_mut`.\n/// self.as_mut().project();\n/// self.as_mut().project();\n/// }\n/// }\n/// ```\n///\n/// # `!Unpin`\n///\n/// If you want to make sure `Unpin` is not implemented, use the `#[project(!Unpin)]`\n/// attribute.\n///\n/// ```\n/// use pin_project_lite::pin_project;\n///\n/// pin_project! {\n/// #[project(!Unpin)]\n/// struct Struct {\n/// #[pin]\n/// field: T,\n/// }\n/// }\n/// ```\n///\n/// This is equivalent to using `#[pin]` attribute for a [`PhantomPinned`] field.\n///\n/// ```\n/// use std::marker::PhantomPinned;\n///\n/// use pin_project_lite::pin_project;\n///\n/// pin_project! {\n/// struct Struct {\n/// field: T,\n/// #[pin]\n/// _pin: PhantomPinned,\n/// }\n/// }\n/// ```\n///\n/// Note that using [`PhantomPinned`] without `#[pin]` or `#[project(!Unpin)]`\n/// attribute has no effect.\n///\n/// # Pinned Drop\n///\n/// In order to correctly implement pin projections, a type's [`Drop`] impl must not move out of any\n/// structurally pinned fields. Unfortunately, [`Drop::drop`] takes `&mut Self`, not `Pin<&mut Self>`.\n///\n/// To implement [`Drop`] for type that has pin, add an `impl PinnedDrop` block at the end of the\n/// [`pin_project`] macro block. PinnedDrop has the following interface:\n///\n/// ```\n/// # use std::pin::Pin;\n/// trait PinnedDrop {\n/// fn drop(this: Pin<&mut Self>);\n/// }\n/// ```\n///\n/// Note that the argument to `PinnedDrop::drop` cannot be named `self`.\n///\n/// `pin_project!` implements the actual [`Drop`] trait via PinnedDrop you implemented. To\n/// explicitly drop a type that implements PinnedDrop, use the [drop] function just like dropping a\n/// type that directly implements [`Drop`].\n///\n/// `PinnedDrop::drop` will never be called more than once, just like [`Drop::drop`].\n///\n/// ```\n/// use pin_project_lite::pin_project;\n///\n/// pin_project! {\n/// pub struct Struct<'a> {\n/// was_dropped: &'a mut bool,\n/// #[pin]\n/// field: u8,\n/// }\n///\n/// impl PinnedDrop for Struct<'_> {\n/// fn drop(this: Pin<&mut Self>) { // <----- NOTE: this is not `self`\n/// **this.project().was_dropped = true;\n/// }\n/// }\n/// }\n///\n/// let mut was_dropped = false;\n/// drop(Struct { was_dropped: &mut was_dropped, field: 42 });\n/// assert!(was_dropped);\n/// ```\n///\n/// [`PhantomPinned`]: core::marker::PhantomPinned\n/// [`Pin::as_mut`]: core::pin::Pin::as_mut\n/// [`Pin`]: core::pin::Pin\n/// [pin-project]: https://github.com/taiki-e/pin-project\n#[macro_export]\nmacro_rules! pin_project {\n ($($tt:tt)*) => {\n $crate::__pin_project_internal! {\n [][][][][]\n $($tt)*\n }\n };\n}\n\n// limitations:\n// - no support for tuple structs and tuple variant (wontfix).\n// - no support for multiple trait/lifetime bounds.\n// - no support for `Self` in where clauses. (wontfix)\n// - no support for overlapping lifetime names. (wontfix)\n// - no interoperability with other field attributes.\n// - no useful error messages. (wontfix)\n// etc...\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_expand {\n (\n [$($proj_mut_ident:ident)?]\n [$($proj_ref_ident:ident)?]\n [$($proj_replace_ident:ident)?]\n [$($proj_not_unpin_mark:ident)?]\n [$proj_vis:vis]\n [$(#[$attrs:meta])* $vis:vis $struct_ty_ident:ident $ident:ident]\n [$($def_generics:tt)*]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n {\n $($body_data:tt)*\n }\n $($(#[$drop_impl_attrs:meta])* impl $($pinned_drop:tt)*)?\n ) => {\n $crate::__pin_project_reconstruct! {\n [$(#[$attrs])* $vis $struct_ty_ident $ident]\n [$($def_generics)*] [$($impl_generics)*]\n [$($ty_generics)*] [$(where $($where_clause)*)?]\n {\n $($body_data)*\n }\n }\n\n $crate::__pin_project_make_proj_ty! {\n [$($proj_mut_ident)?]\n [$proj_vis $struct_ty_ident $ident]\n [__pin_project_make_proj_field_mut]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n {\n $($body_data)*\n }\n }\n $crate::__pin_project_make_proj_ty! {\n [$($proj_ref_ident)?]\n [$proj_vis $struct_ty_ident $ident]\n [__pin_project_make_proj_field_ref]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n {\n $($body_data)*\n }\n }\n $crate::__pin_project_make_proj_replace_ty! {\n [$($proj_replace_ident)?]\n [$proj_vis $struct_ty_ident]\n [__pin_project_make_proj_field_replace]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n {\n $($body_data)*\n }\n }\n\n $crate::__pin_project_constant! {\n [$(#[$attrs])* $vis $struct_ty_ident $ident]\n [$($proj_mut_ident)?] [$($proj_ref_ident)?] [$($proj_replace_ident)?]\n [$($proj_not_unpin_mark)?]\n [$proj_vis]\n [$($def_generics)*] [$($impl_generics)*]\n [$($ty_generics)*] [$(where $($where_clause)*)?]\n {\n $($body_data)*\n }\n $($(#[$drop_impl_attrs])* impl $($pinned_drop)*)?\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_constant {\n (\n [$(#[$attrs:meta])* $vis:vis struct $ident:ident]\n [$($proj_mut_ident:ident)?] [$($proj_ref_ident:ident)?] [$($proj_replace_ident:ident)?]\n [$($proj_not_unpin_mark:ident)?]\n [$proj_vis:vis]\n [$($def_generics:tt)*]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n {\n $(\n $(#[$pin:ident])?\n $field_vis:vis $field:ident: $field_ty:ty\n ),+ $(,)?\n }\n $($(#[$drop_impl_attrs:meta])* impl $($pinned_drop:tt)*)?\n ) => {\n #[allow(\n explicit_outlives_requirements, // https://github.com/rust-lang/rust/issues/60993\n single_use_lifetimes, // https://github.com/rust-lang/rust/issues/55058\n // This lint warns of `clippy::*` generated by external macros.\n // We allow this lint for compatibility with older compilers.\n clippy::unknown_clippy_lints,\n clippy::absolute_paths,\n clippy::min_ident_chars,\n clippy::redundant_pub_crate, // This lint warns `pub(crate)` field in private struct.\n clippy::single_char_lifetime_names,\n clippy::used_underscore_binding\n )]\n const _: () = {\n $crate::__pin_project_make_proj_ty! {\n [$($proj_mut_ident)? Projection]\n [$proj_vis struct $ident]\n [__pin_project_make_proj_field_mut]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n {\n $(\n $(#[$pin])?\n $field_vis $field: $field_ty\n ),+\n }\n }\n $crate::__pin_project_make_proj_ty! {\n [$($proj_ref_ident)? ProjectionRef]\n [$proj_vis struct $ident]\n [__pin_project_make_proj_field_ref]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n {\n $(\n $(#[$pin])?\n $field_vis $field: $field_ty\n ),+\n }\n }\n\n impl<$($impl_generics)*> $ident <$($ty_generics)*>\n $(where\n $($where_clause)*)?\n {\n $crate::__pin_project_struct_make_proj_method! {\n [$($proj_mut_ident)? Projection]\n [$proj_vis]\n [project get_unchecked_mut mut]\n [$($ty_generics)*]\n {\n $(\n $(#[$pin])?\n $field_vis $field\n ),+\n }\n }\n $crate::__pin_project_struct_make_proj_method! {\n [$($proj_ref_ident)? ProjectionRef]\n [$proj_vis]\n [project_ref get_ref]\n [$($ty_generics)*]\n {\n $(\n $(#[$pin])?\n $field_vis $field\n ),+\n }\n }\n $crate::__pin_project_struct_make_proj_replace_method! {\n [$($proj_replace_ident)?]\n [$proj_vis]\n [ProjectionReplace]\n [$($ty_generics)*]\n {\n $(\n $(#[$pin])?\n $field_vis $field\n ),+\n }\n }\n }\n\n $crate::__pin_project_make_unpin_impl! {\n [$($proj_not_unpin_mark)?]\n [$vis $ident]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n $(\n $field: $crate::__pin_project_make_unpin_bound!(\n $(#[$pin])? $field_ty\n )\n ),+\n }\n\n $crate::__pin_project_make_drop_impl! {\n [$ident]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n $($(#[$drop_impl_attrs])* impl $($pinned_drop)*)?\n }\n\n // Ensure that it's impossible to use pin projections on a #[repr(packed)] struct.\n //\n // Taking a reference to a packed field is UB, and applying\n // `#[forbid(unaligned_references)]` makes sure that doing this is a hard error.\n //\n // If the struct ends up having #[repr(packed)] applied somehow,\n // this will generate an (unfriendly) error message. Under all reasonable\n // circumstances, we'll detect the #[repr(packed)] attribute, and generate\n // a much nicer error above.\n //\n // See https://github.com/taiki-e/pin-project/pull/34 for more details.\n //\n // Note:\n // - Lint-based tricks aren't perfect, but they're much better than nothing:\n // https://github.com/taiki-e/pin-project-lite/issues/26\n //\n // - Enable both unaligned_references and safe_packed_borrows lints\n // because unaligned_references lint does not exist in older compilers:\n // https://github.com/taiki-e/pin-project-lite/pull/55\n // https://github.com/rust-lang/rust/pull/82525\n #[forbid(unaligned_references, safe_packed_borrows)]\n fn __assert_not_repr_packed <$($impl_generics)*> (this: &$ident <$($ty_generics)*>)\n $(where\n $($where_clause)*)?\n {\n $(\n let _ = &this.$field;\n )+\n }\n };\n };\n (\n [$(#[$attrs:meta])* $vis:vis enum $ident:ident]\n [$($proj_mut_ident:ident)?] [$($proj_ref_ident:ident)?] [$($proj_replace_ident:ident)?]\n [$($proj_not_unpin_mark:ident)?]\n [$proj_vis:vis]\n [$($def_generics:tt)*]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n {\n $(\n $(#[$variant_attrs:meta])*\n $variant:ident $({\n $(\n $(#[$pin:ident])?\n $field:ident: $field_ty:ty\n ),+ $(,)?\n })?\n ),+ $(,)?\n }\n $($(#[$drop_impl_attrs:meta])* impl $($pinned_drop:tt)*)?\n ) => {\n #[allow(\n single_use_lifetimes, // https://github.com/rust-lang/rust/issues/55058\n // This lint warns of `clippy::*` generated by external macros.\n // We allow this lint for compatibility with older compilers.\n clippy::unknown_clippy_lints,\n clippy::absolute_paths,\n clippy::min_ident_chars,\n clippy::single_char_lifetime_names,\n clippy::used_underscore_binding\n )]\n const _: () = {\n impl<$($impl_generics)*> $ident <$($ty_generics)*>\n $(where\n $($where_clause)*)?\n {\n $crate::__pin_project_enum_make_proj_method! {\n [$($proj_mut_ident)?]\n [$proj_vis]\n [project get_unchecked_mut mut]\n [$($ty_generics)*]\n {\n $(\n $variant $({\n $(\n $(#[$pin])?\n $field\n ),+\n })?\n ),+\n }\n }\n $crate::__pin_project_enum_make_proj_method! {\n [$($proj_ref_ident)?]\n [$proj_vis]\n [project_ref get_ref]\n [$($ty_generics)*]\n {\n $(\n $variant $({\n $(\n $(#[$pin])?\n $field\n ),+\n })?\n ),+\n }\n }\n $crate::__pin_project_enum_make_proj_replace_method! {\n [$($proj_replace_ident)?]\n [$proj_vis]\n [$($ty_generics)*]\n {\n $(\n $variant $({\n $(\n $(#[$pin])?\n $field\n ),+\n })?\n ),+\n }\n }\n }\n\n $crate::__pin_project_make_unpin_impl! {\n [$($proj_not_unpin_mark)?]\n [$vis $ident]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n $(\n $variant: ($(\n $(\n $crate::__pin_project_make_unpin_bound!(\n $(#[$pin])? $field_ty\n )\n ),+\n )?)\n ),+\n }\n\n $crate::__pin_project_make_drop_impl! {\n [$ident]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n $($(#[$drop_impl_attrs])* impl $($pinned_drop)*)?\n }\n\n // We don't need to check for '#[repr(packed)]',\n // since it does not apply to enums.\n };\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_reconstruct {\n (\n [$(#[$attrs:meta])* $vis:vis struct $ident:ident]\n [$($def_generics:tt)*] [$($impl_generics:tt)*]\n [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n {\n $(\n $(#[$pin:ident])?\n $field_vis:vis $field:ident: $field_ty:ty\n ),+ $(,)?\n }\n ) => {\n $(#[$attrs])*\n $vis struct $ident $($def_generics)*\n $(where\n $($where_clause)*)?\n {\n $(\n $field_vis $field: $field_ty\n ),+\n }\n };\n (\n [$(#[$attrs:meta])* $vis:vis enum $ident:ident]\n [$($def_generics:tt)*] [$($impl_generics:tt)*]\n [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n {\n $(\n $(#[$variant_attrs:meta])*\n $variant:ident $({\n $(\n $(#[$pin:ident])?\n $field:ident: $field_ty:ty\n ),+ $(,)?\n })?\n ),+ $(,)?\n }\n ) => {\n $(#[$attrs])*\n $vis enum $ident $($def_generics)*\n $(where\n $($where_clause)*)?\n {\n $(\n $(#[$variant_attrs])*\n $variant $({\n $(\n $field: $field_ty\n ),+\n })?\n ),+\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_proj_ty {\n ([] $($field:tt)*) => {};\n (\n [$proj_ty_ident:ident $default_ident:ident]\n [$proj_vis:vis struct $ident:ident]\n $($field:tt)*\n ) => {};\n (\n [$proj_ty_ident:ident]\n [$proj_vis:vis struct $ident:ident]\n [$__pin_project_make_proj_field:ident]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n {\n $(\n $(#[$pin:ident])?\n $field_vis:vis $field:ident: $field_ty:ty\n ),+ $(,)?\n }\n ) => {\n $crate::__pin_project_make_proj_ty_body! {\n [$proj_ty_ident]\n [$proj_vis struct $ident]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n [\n $(\n $field_vis $field: $crate::$__pin_project_make_proj_field!(\n $(#[$pin])? $field_ty\n )\n ),+\n ]\n }\n };\n (\n [$proj_ty_ident:ident]\n [$proj_vis:vis enum $ident:ident]\n [$__pin_project_make_proj_field:ident]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n {\n $(\n $(#[$variant_attrs:meta])*\n $variant:ident $({\n $(\n $(#[$pin:ident])?\n $field:ident: $field_ty:ty\n ),+ $(,)?\n })?\n ),+ $(,)?\n }\n ) => {\n $crate::__pin_project_make_proj_ty_body! {\n [$proj_ty_ident]\n [$proj_vis enum $ident]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n [\n $(\n $variant $({\n $(\n $field: $crate::$__pin_project_make_proj_field!(\n $(#[$pin])? $field_ty\n )\n ),+\n })?\n ),+\n ]\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_proj_ty_body {\n (\n [$proj_ty_ident:ident]\n [$proj_vis:vis $struct_ty_ident:ident $ident:ident]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n [$($body_data:tt)+]\n ) => {\n #[doc(hidden)] // Workaround for rustc bug: see https://github.com/taiki-e/pin-project-lite/issues/77#issuecomment-1671540180 for more.\n #[allow(\n dead_code, // This lint warns unused fields/variants.\n single_use_lifetimes, // https://github.com/rust-lang/rust/issues/55058\n // This lint warns of `clippy::*` generated by external macros.\n // We allow this lint for compatibility with older compilers.\n clippy::unknown_clippy_lints,\n clippy::absolute_paths,\n clippy::min_ident_chars,\n clippy::mut_mut, // This lint warns `&mut &mut `. (only needed for project)\n clippy::redundant_pub_crate, // This lint warns `pub(crate)` field in private struct.\n clippy::ref_option_ref, // This lint warns `&Option<&>`. (only needed for project_ref)\n clippy::single_char_lifetime_names,\n clippy::type_repetition_in_bounds // https://github.com/rust-lang/rust-clippy/issues/4326\n )]\n $proj_vis $struct_ty_ident $proj_ty_ident <'__pin, $($impl_generics)*>\n where\n $ident <$($ty_generics)*>: '__pin\n $(, $($where_clause)*)?\n {\n $($body_data)+\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_proj_replace_ty {\n ([] $($field:tt)*) => {};\n (\n [$proj_ty_ident:ident]\n [$proj_vis:vis struct]\n [$__pin_project_make_proj_field:ident]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n {\n $(\n $(#[$pin:ident])?\n $field_vis:vis $field:ident: $field_ty:ty\n ),+ $(,)?\n }\n ) => {\n $crate::__pin_project_make_proj_replace_ty_body! {\n [$proj_ty_ident]\n [$proj_vis struct]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n [\n $(\n $field_vis $field: $crate::$__pin_project_make_proj_field!(\n $(#[$pin])? $field_ty\n )\n ),+\n ]\n }\n };\n (\n [$proj_ty_ident:ident]\n [$proj_vis:vis enum]\n [$__pin_project_make_proj_field:ident]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n {\n $(\n $(#[$variant_attrs:meta])*\n $variant:ident $({\n $(\n $(#[$pin:ident])?\n $field:ident: $field_ty:ty\n ),+ $(,)?\n })?\n ),+ $(,)?\n }\n ) => {\n $crate::__pin_project_make_proj_replace_ty_body! {\n [$proj_ty_ident]\n [$proj_vis enum]\n [$($impl_generics)*] [$($ty_generics)*] [$(where $($where_clause)*)?]\n [\n $(\n $variant $({\n $(\n $field: $crate::$__pin_project_make_proj_field!(\n $(#[$pin])? $field_ty\n )\n ),+\n })?\n ),+\n ]\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_proj_replace_ty_body {\n (\n [$proj_ty_ident:ident]\n [$proj_vis:vis $struct_ty_ident:ident]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n [$($body_data:tt)+]\n ) => {\n #[doc(hidden)] // Workaround for rustc bug: see https://github.com/taiki-e/pin-project-lite/issues/77#issuecomment-1671540180 for more.\n #[allow(\n dead_code, // This lint warns unused fields/variants.\n single_use_lifetimes, // https://github.com/rust-lang/rust/issues/55058\n // This lint warns of `clippy::*` generated by external macros.\n // We allow this lint for compatibility with older compilers.\n clippy::unknown_clippy_lints,\n clippy::absolute_paths,\n clippy::min_ident_chars,\n clippy::mut_mut, // This lint warns `&mut &mut `. (only needed for project)\n clippy::redundant_pub_crate, // This lint warns `pub(crate)` field in private struct.\n clippy::single_char_lifetime_names,\n clippy::type_repetition_in_bounds // https://github.com/rust-lang/rust-clippy/issues/4326\n )]\n $proj_vis $struct_ty_ident $proj_ty_ident <$($impl_generics)*>\n where\n $($($where_clause)*)?\n {\n $($body_data)+\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_proj_replace_block {\n (\n [$($proj_path:tt)+]\n {\n $(\n $(#[$pin:ident])?\n $field_vis:vis $field:ident\n ),+\n }\n ) => {\n let result = $($proj_path)* {\n $(\n $field: $crate::__pin_project_make_replace_field_proj!(\n $(#[$pin])? $field\n )\n ),+\n };\n\n {\n ( $(\n $crate::__pin_project_make_unsafe_drop_in_place_guard!(\n $(#[$pin])? $field\n ),\n )* );\n }\n\n result\n };\n ([$($proj_path:tt)+]) => { $($proj_path)* };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_struct_make_proj_method {\n ([] $($variant:tt)*) => {};\n (\n [$proj_ty_ident:ident $_ignored_default_arg:ident]\n [$proj_vis:vis]\n [$method_ident:ident $get_method:ident $($mut:ident)?]\n [$($ty_generics:tt)*]\n $($variant:tt)*\n ) => {\n $crate::__pin_project_struct_make_proj_method! {\n [$proj_ty_ident]\n [$proj_vis]\n [$method_ident $get_method $($mut)?]\n [$($ty_generics)*]\n $($variant)*\n }\n };\n (\n [$proj_ty_ident:ident]\n [$proj_vis:vis]\n [$method_ident:ident $get_method:ident $($mut:ident)?]\n [$($ty_generics:tt)*]\n {\n $(\n $(#[$pin:ident])?\n $field_vis:vis $field:ident\n ),+\n }\n ) => {\n #[doc(hidden)] // Workaround for rustc bug: see https://github.com/taiki-e/pin-project-lite/issues/77#issuecomment-1671540180 for more.\n #[inline]\n $proj_vis fn $method_ident<'__pin>(\n self: $crate::__private::Pin<&'__pin $($mut)? Self>,\n ) -> $proj_ty_ident <'__pin, $($ty_generics)*> {\n unsafe {\n let Self { $($field),* } = self.$get_method();\n $proj_ty_ident {\n $(\n $field: $crate::__pin_project_make_unsafe_field_proj!(\n $(#[$pin])? $field\n )\n ),+\n }\n }\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_struct_make_proj_replace_method {\n ([] $($field:tt)*) => {};\n (\n [$proj_ty_ident:ident]\n [$proj_vis:vis]\n [$_proj_ty_ident:ident]\n [$($ty_generics:tt)*]\n {\n $(\n $(#[$pin:ident])?\n $field_vis:vis $field:ident\n ),+\n }\n ) => {\n #[doc(hidden)] // Workaround for rustc bug: see https://github.com/taiki-e/pin-project-lite/issues/77#issuecomment-1671540180 for more.\n #[inline]\n $proj_vis fn project_replace(\n self: $crate::__private::Pin<&mut Self>,\n replacement: Self,\n ) -> $proj_ty_ident <$($ty_generics)*> {\n unsafe {\n let __self_ptr: *mut Self = self.get_unchecked_mut();\n\n // Destructors will run in reverse order, so next create a guard to overwrite\n // `self` with the replacement value without calling destructors.\n let __guard = $crate::__private::UnsafeOverwriteGuard::new(__self_ptr, replacement);\n\n let Self { $($field),* } = &mut *__self_ptr;\n\n $crate::__pin_project_make_proj_replace_block! {\n [$proj_ty_ident]\n {\n $(\n $(#[$pin])?\n $field\n ),+\n }\n }\n }\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_enum_make_proj_method {\n ([] $($variant:tt)*) => {};\n (\n [$proj_ty_ident:ident]\n [$proj_vis:vis]\n [$method_ident:ident $get_method:ident $($mut:ident)?]\n [$($ty_generics:tt)*]\n {\n $(\n $variant:ident $({\n $(\n $(#[$pin:ident])?\n $field:ident\n ),+\n })?\n ),+\n }\n ) => {\n #[doc(hidden)] // Workaround for rustc bug: see https://github.com/taiki-e/pin-project-lite/issues/77#issuecomment-1671540180 for more.\n #[inline]\n $proj_vis fn $method_ident<'__pin>(\n self: $crate::__private::Pin<&'__pin $($mut)? Self>,\n ) -> $proj_ty_ident <'__pin, $($ty_generics)*> {\n unsafe {\n match self.$get_method() {\n $(\n Self::$variant $({\n $($field),+\n })? => {\n $proj_ty_ident::$variant $({\n $(\n $field: $crate::__pin_project_make_unsafe_field_proj!(\n $(#[$pin])? $field\n )\n ),+\n })?\n }\n ),+\n }\n }\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_enum_make_proj_replace_method {\n ([] $($field:tt)*) => {};\n (\n [$proj_ty_ident:ident]\n [$proj_vis:vis]\n [$($ty_generics:tt)*]\n {\n $(\n $variant:ident $({\n $(\n $(#[$pin:ident])?\n $field:ident\n ),+\n })?\n ),+\n }\n ) => {\n #[doc(hidden)] // Workaround for rustc bug: see https://github.com/taiki-e/pin-project-lite/issues/77#issuecomment-1671540180 for more.\n #[inline]\n $proj_vis fn project_replace(\n self: $crate::__private::Pin<&mut Self>,\n replacement: Self,\n ) -> $proj_ty_ident <$($ty_generics)*> {\n unsafe {\n let __self_ptr: *mut Self = self.get_unchecked_mut();\n\n // Destructors will run in reverse order, so next create a guard to overwrite\n // `self` with the replacement value without calling destructors.\n let __guard = $crate::__private::UnsafeOverwriteGuard::new(__self_ptr, replacement);\n\n match &mut *__self_ptr {\n $(\n Self::$variant $({\n $($field),+\n })? => {\n $crate::__pin_project_make_proj_replace_block! {\n [$proj_ty_ident :: $variant]\n $({\n $(\n $(#[$pin])?\n $field\n ),+\n })?\n }\n }\n ),+\n }\n }\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_unpin_impl {\n (\n []\n [$vis:vis $ident:ident]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n $($field:tt)*\n ) => {\n // Automatically create the appropriate conditional `Unpin` implementation.\n //\n // Basically this is equivalent to the following code:\n // ```\n // impl Unpin for Struct where T: Unpin {}\n // ```\n //\n // However, if struct is public and there is a private type field,\n // this would cause an E0446 (private type in public interface).\n //\n // When RFC 2145 is implemented (rust-lang/rust#48054),\n // this will become a lint, rather than a hard error.\n //\n // As a workaround for this, we generate a new struct, containing all of the pinned\n // fields from our #[pin_project] type. This struct is declared within\n // a function, which makes it impossible to be named by user code.\n // This guarantees that it will use the default auto-trait impl for Unpin -\n // that is, it will implement Unpin iff all of its fields implement Unpin.\n // This type can be safely declared as 'public', satisfying the privacy\n // checker without actually allowing user code to access it.\n //\n // This allows users to apply the #[pin_project] attribute to types\n // regardless of the privacy of the types of their fields.\n //\n // See also https://github.com/taiki-e/pin-project/pull/53.\n #[allow(non_snake_case)]\n $vis struct __Origin<'__pin, $($impl_generics)*>\n $(where\n $($where_clause)*)?\n {\n __dummy_lifetime: $crate::__private::PhantomData<&'__pin ()>,\n $($field)*\n }\n impl<'__pin, $($impl_generics)*> $crate::__private::Unpin for $ident <$($ty_generics)*>\n where\n $crate::__private::PinnedFieldsOf<__Origin<'__pin, $($ty_generics)*>>:\n $crate::__private::Unpin\n $(, $($where_clause)*)?\n {\n }\n };\n (\n [$proj_not_unpin_mark:ident]\n [$vis:vis $ident:ident]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n $($field:tt)*\n ) => {\n // TODO: Using `: Sized` here allow emulating real negative_impls...\n // https://github.com/taiki-e/pin-project/issues/340#issuecomment-2428002670\n #[doc(hidden)]\n impl<'__pin, $($impl_generics)*> $crate::__private::Unpin for $ident <$($ty_generics)*>\n where\n (\n $crate::__private::PhantomData<&'__pin ()>,\n $crate::__private::PhantomPinned,\n ): $crate::__private::Unpin\n $(, $($where_clause)*)?\n {\n }\n }\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_drop_impl {\n (\n [$_ident:ident]\n [$($_impl_generics:tt)*] [$($_ty_generics:tt)*] [$(where $($_where_clause:tt)*)?]\n $(#[$drop_impl_attrs:meta])*\n impl $(<\n $( $lifetime:lifetime $(: $lifetime_bound:lifetime)? ),* $(,)?\n $( $generics:ident\n $(: $generics_bound:path)?\n $(: ?$generics_unsized_bound:path)?\n $(: $generics_lifetime_bound:lifetime)?\n ),*\n >)? PinnedDrop for $self_ty:ty\n $(where\n $( $where_clause_ty:ty\n $(: $where_clause_bound:path)?\n $(: ?$where_clause_unsized_bound:path)?\n $(: $where_clause_lifetime_bound:lifetime)?\n ),* $(,)?\n )?\n {\n $(#[$drop_fn_attrs:meta])*\n fn drop($($arg:ident)+: Pin<&mut Self>) {\n $($tt:tt)*\n }\n }\n ) => {\n $(#[$drop_impl_attrs])*\n impl $(<\n $( $lifetime $(: $lifetime_bound)? ,)*\n $( $generics\n $(: $generics_bound)?\n $(: ?$generics_unsized_bound)?\n $(: $generics_lifetime_bound)?\n ),*\n >)? $crate::__private::Drop for $self_ty\n $(where\n $( $where_clause_ty\n $(: $where_clause_bound)?\n $(: ?$where_clause_unsized_bound)?\n $(: $where_clause_lifetime_bound)?\n ),*\n )?\n {\n $(#[$drop_fn_attrs])*\n fn drop(&mut self) {\n // Implementing `__DropInner::__drop_inner` is safe, but calling it is not safe.\n // This is because destructors can be called multiple times in safe code and\n // [double dropping is unsound](https://github.com/rust-lang/rust/pull/62360).\n //\n // `__drop_inner` is defined as a safe method, but this is fine since\n // `__drop_inner` is not accessible by the users and we call `__drop_inner` only\n // once.\n //\n // Users can implement [`Drop`] safely using `pin_project!` and can drop a\n // type that implements `PinnedDrop` using the [`drop`] function safely.\n fn __drop_inner $(<\n $( $lifetime $(: $lifetime_bound)? ,)*\n $( $generics\n $(: $generics_bound)?\n $(: ?$generics_unsized_bound)?\n $(: $generics_lifetime_bound)?\n ),*\n >)? (\n $($arg)+: $crate::__private::Pin<&mut $self_ty>,\n )\n $(where\n $( $where_clause_ty\n $(: $where_clause_bound)?\n $(: ?$where_clause_unsized_bound)?\n $(: $where_clause_lifetime_bound)?\n ),*\n )?\n {\n // A dummy `__drop_inner` function to prevent users call outer `__drop_inner`.\n fn __drop_inner() {}\n $($tt)*\n }\n\n // Safety - we're in 'drop', so we know that 'self' will\n // never move again.\n let pinned_self: $crate::__private::Pin<&mut Self>\n = unsafe { $crate::__private::Pin::new_unchecked(self) };\n // We call `__drop_inner` only once. Since `__DropInner::__drop_inner`\n // is not accessible by the users, it is never called again.\n __drop_inner(pinned_self);\n }\n }\n };\n (\n [$ident:ident]\n [$($impl_generics:tt)*] [$($ty_generics:tt)*] [$(where $($where_clause:tt)*)?]\n ) => {\n // Ensure that struct does not implement `Drop`.\n //\n // There are two possible cases:\n // 1. The user type does not implement Drop. In this case,\n // the first blanket impl will not apply to it. This code\n // will compile, as there is only one impl of MustNotImplDrop for the user type\n // 2. The user type does impl Drop. This will make the blanket impl applicable,\n // which will then conflict with the explicit MustNotImplDrop impl below.\n // This will result in a compilation error, which is exactly what we want.\n trait MustNotImplDrop {}\n #[allow(clippy::drop_bounds, drop_bounds)]\n impl MustNotImplDrop for T {}\n impl<$($impl_generics)*> MustNotImplDrop for $ident <$($ty_generics)*>\n $(where\n $($where_clause)*)?\n {\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_unpin_bound {\n (#[pin] $field_ty:ty) => {\n $field_ty\n };\n ($field_ty:ty) => {\n $crate::__private::AlwaysUnpin<$field_ty>\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_unsafe_field_proj {\n (#[pin] $field:ident) => {\n $crate::__private::Pin::new_unchecked($field)\n };\n ($field:ident) => {\n $field\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_replace_field_proj {\n (#[pin] $field:ident) => {\n $crate::__private::PhantomData\n };\n ($field:ident) => {\n $crate::__private::ptr::read($field)\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_unsafe_drop_in_place_guard {\n (#[pin] $field:ident) => {\n $crate::__private::UnsafeDropInPlaceGuard::new($field)\n };\n ($field:ident) => {\n ()\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_proj_field_mut {\n (#[pin] $field_ty:ty) => {\n $crate::__private::Pin<&'__pin mut ($field_ty)>\n };\n ($field_ty:ty) => {\n &'__pin mut ($field_ty)\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_proj_field_ref {\n (#[pin] $field_ty:ty) => {\n $crate::__private::Pin<&'__pin ($field_ty)>\n };\n ($field_ty:ty) => {\n &'__pin ($field_ty)\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_make_proj_field_replace {\n (#[pin] $field_ty:ty) => {\n $crate::__private::PhantomData<$field_ty>\n };\n ($field_ty:ty) => {\n $field_ty\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_internal {\n // parsing proj_mut_ident\n (\n []\n [$($proj_ref_ident:ident)?]\n [$($proj_replace_ident:ident)?]\n [$( ! $proj_not_unpin_mark:ident)?]\n [$($attrs:tt)*]\n\n #[project = $proj_mut_ident:ident]\n $($tt:tt)*\n ) => {\n $crate::__pin_project_internal! {\n [$proj_mut_ident]\n [$($proj_ref_ident)?]\n [$($proj_replace_ident)?]\n [$( ! $proj_not_unpin_mark)?]\n [$($attrs)*]\n $($tt)*\n }\n };\n // parsing proj_ref_ident\n (\n [$($proj_mut_ident:ident)?]\n []\n [$($proj_replace_ident:ident)?]\n [$( ! $proj_not_unpin_mark:ident)?]\n [$($attrs:tt)*]\n\n #[project_ref = $proj_ref_ident:ident]\n $($tt:tt)*\n ) => {\n $crate::__pin_project_internal! {\n [$($proj_mut_ident)?]\n [$proj_ref_ident]\n [$($proj_replace_ident)?]\n [$( ! $proj_not_unpin_mark)?]\n [$($attrs)*]\n $($tt)*\n }\n };\n // parsing proj_replace_ident\n (\n [$($proj_mut_ident:ident)?]\n [$($proj_ref_ident:ident)?]\n []\n [$( ! $proj_not_unpin_mark:ident)?]\n [$($attrs:tt)*]\n\n #[project_replace = $proj_replace_ident:ident]\n $($tt:tt)*\n ) => {\n $crate::__pin_project_internal! {\n [$($proj_mut_ident)?]\n [$($proj_ref_ident)?]\n [$proj_replace_ident]\n [$( ! $proj_not_unpin_mark)?]\n [$($attrs)*]\n $($tt)*\n }\n };\n // parsing !Unpin\n (\n [$($proj_mut_ident:ident)?]\n [$($proj_ref_ident:ident)?]\n [$($proj_replace_ident:ident)?]\n []\n [$($attrs:tt)*]\n\n #[project( ! $proj_not_unpin_mark:ident)]\n $($tt:tt)*\n ) => {\n $crate::__pin_project_internal! {\n [$($proj_mut_ident)?]\n [$($proj_ref_ident)?]\n [$($proj_replace_ident)?]\n [ ! $proj_not_unpin_mark]\n [$($attrs)*]\n $($tt)*\n }\n };\n // this is actually part of a recursive step that picks off a single non-`pin_project_lite` attribute\n // there could be more to parse\n (\n [$($proj_mut_ident:ident)?]\n [$($proj_ref_ident:ident)?]\n [$($proj_replace_ident:ident)?]\n [$( ! $proj_not_unpin_mark:ident)?]\n [$($attrs:tt)*]\n\n #[$($attr:tt)*]\n $($tt:tt)*\n ) => {\n $crate::__pin_project_internal! {\n [$($proj_mut_ident)?]\n [$($proj_ref_ident)?]\n [$($proj_replace_ident)?]\n [$( ! $proj_not_unpin_mark)?]\n [$($attrs)* #[$($attr)*]]\n $($tt)*\n }\n };\n // now determine visibility\n // if public, downgrade\n (\n [$($proj_mut_ident:ident)?]\n [$($proj_ref_ident:ident)?]\n [$($proj_replace_ident:ident)?]\n [$( ! $proj_not_unpin_mark:ident)?]\n [$($attrs:tt)*]\n pub $struct_ty_ident:ident $ident:ident\n $($tt:tt)*\n ) => {\n $crate::__pin_project_parse_generics! {\n [$($proj_mut_ident)?]\n [$($proj_ref_ident)?]\n [$($proj_replace_ident)?]\n [$($proj_not_unpin_mark)?]\n [$($attrs)*]\n [pub $struct_ty_ident $ident pub(crate)]\n $($tt)*\n }\n };\n (\n [$($proj_mut_ident:ident)?]\n [$($proj_ref_ident:ident)?]\n [$($proj_replace_ident:ident)?]\n [$( ! $proj_not_unpin_mark:ident)?]\n [$($attrs:tt)*]\n $vis:vis $struct_ty_ident:ident $ident:ident\n $($tt:tt)*\n ) => {\n $crate::__pin_project_parse_generics! {\n [$($proj_mut_ident)?]\n [$($proj_ref_ident)?]\n [$($proj_replace_ident)?]\n [$($proj_not_unpin_mark)?]\n [$($attrs)*]\n [$vis $struct_ty_ident $ident $vis]\n $($tt)*\n }\n };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! __pin_project_parse_generics {\n (\n [$($proj_mut_ident:ident)?]\n [$($proj_ref_ident:ident)?]\n [$($proj_replace_ident:ident)?]\n [$($proj_not_unpin_mark:ident)?]\n [$($attrs:tt)*]\n [$vis:vis $struct_ty_ident:ident $ident:ident $proj_vis:vis]\n $(<\n $( $lifetime:lifetime $(: $lifetime_bound:lifetime)? ),* $(,)?\n $( $generics:ident\n $(: $generics_bound:path)?\n $(: ?$generics_unsized_bound:path)?\n $(: $generics_lifetime_bound:lifetime)?\n $(= $generics_default:ty)?\n ),* $(,)?\n >)?\n $(where\n $( $where_clause_ty:ty\n $(: $where_clause_bound:path)?\n $(: ?$where_clause_unsized_bound:path)?\n $(: $where_clause_lifetime_bound:lifetime)?\n ),* $(,)?\n )?\n {\n $($body_data:tt)*\n }\n $($(#[$drop_impl_attrs:meta])* impl $($pinned_drop:tt)*)?\n ) => {\n $crate::__pin_project_expand! {\n [$($proj_mut_ident)?]\n [$($proj_ref_ident)?]\n [$($proj_replace_ident)?]\n [$($proj_not_unpin_mark)?]\n [$proj_vis]\n [$($attrs)* $vis $struct_ty_ident $ident]\n [$(<\n $( $lifetime $(: $lifetime_bound)? ,)*\n $( $generics\n $(: $generics_bound)?\n $(: ?$generics_unsized_bound)?\n $(: $generics_lifetime_bound)?\n $(= $generics_default)?\n ),*\n >)?]\n [$(\n $( $lifetime $(: $lifetime_bound)? ,)*\n $( $generics\n $(: $generics_bound)?\n $(: ?$generics_unsized_bound)?\n $(: $generics_lifetime_bound)?\n ),*\n )?]\n [$( $( $lifetime ,)* $( $generics ),* )?]\n [$(where $( $where_clause_ty\n $(: $where_clause_bound)?\n $(: ?$where_clause_unsized_bound)?\n $(: $where_clause_lifetime_bound)?\n ),* )?]\n {\n $($body_data)*\n }\n $($(#[$drop_impl_attrs])* impl $($pinned_drop)*)?\n }\n };\n}\n\n// Not public API.\n#[doc(hidden)]\n#[allow(missing_debug_implementations)]\npub mod __private {\n use core::mem::ManuallyDrop;\n #[doc(hidden)]\n pub use core::{\n marker::{PhantomData, PhantomPinned, Unpin},\n ops::Drop,\n pin::Pin,\n ptr,\n };\n\n // Workaround for issue on unstable negative_impls feature that allows unsound overlapping Unpin\n // implementations and rustc bug that leaks unstable negative_impls into stable.\n // See https://github.com/taiki-e/pin-project/issues/340#issuecomment-2432146009 for details.\n #[doc(hidden)]\n pub type PinnedFieldsOf =\n as PinnedFieldsOfHelperTrait>::Actual;\n // We cannot use as IntoIterator>::Item or similar since we should allow ?Sized in T.\n #[doc(hidden)]\n pub trait PinnedFieldsOfHelperTrait {\n type Actual: ?Sized;\n }\n #[doc(hidden)]\n pub struct PinnedFieldsOfHelperStruct(T);\n impl PinnedFieldsOfHelperTrait for PinnedFieldsOfHelperStruct {\n type Actual = T;\n }\n\n // This is an internal helper struct used by `pin_project!`.\n #[doc(hidden)]\n pub struct AlwaysUnpin(PhantomData);\n impl Unpin for AlwaysUnpin {}\n\n // This is an internal helper used to ensure a value is dropped.\n #[doc(hidden)]\n pub struct UnsafeDropInPlaceGuard(*mut T);\n impl UnsafeDropInPlaceGuard {\n #[doc(hidden)]\n pub unsafe fn new(ptr: *mut T) -> Self {\n Self(ptr)\n }\n }\n impl Drop for UnsafeDropInPlaceGuard {\n fn drop(&mut self) {\n // SAFETY: the caller of `UnsafeDropInPlaceGuard::new` must guarantee\n // that `ptr` is valid for drop when this guard is destructed.\n unsafe {\n ptr::drop_in_place(self.0);\n }\n }\n }\n\n // This is an internal helper used to ensure a value is overwritten without\n // its destructor being called.\n #[doc(hidden)]\n pub struct UnsafeOverwriteGuard {\n target: *mut T,\n value: ManuallyDrop,\n }\n impl UnsafeOverwriteGuard {\n #[doc(hidden)]\n pub unsafe fn new(target: *mut T, value: T) -> Self {\n Self { target, value: ManuallyDrop::new(value) }\n }\n }\n impl Drop for UnsafeOverwriteGuard {\n fn drop(&mut self) {\n // SAFETY: the caller of `UnsafeOverwriteGuard::new` must guarantee\n // that `target` is valid for writes when this guard is destructed.\n unsafe {\n ptr::write(self.target, ptr::read(&*self.value));\n }\n }\n }\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/auxiliary/mod.rs", "sha256": "52a78b77be05c67737c56865fbce4477eaacc6482d6377af2a5822f05a2ce061", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![allow(dead_code, unused_macros)]\n\nmacro_rules! assert_unpin {\n ($ty:ty) => {\n static_assertions::assert_impl_all!($ty: Unpin);\n };\n}\nmacro_rules! assert_not_unpin {\n ($ty:ty) => {\n static_assertions::assert_not_impl_all!($ty: Unpin);\n };\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/compiletest.rs", "sha256": "48cfd92c7271345d57a6c222a7a61e00449f38e6ba379130813c45b31a5589c9", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![cfg(not(miri))]\n\n#[rustversion::attr(not(nightly), ignore)]\n#[test]\nfn ui() {\n let t = trybuild::TestCases::new();\n t.compile_fail(\"tests/ui/**/*.rs\");\n t.pass(\"tests/run-pass/**/*.rs\");\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/drop_order.rs", "sha256": "9b13b6461e44be1c1beca6ec47387de83efcb988326e4ff832183b8fad17a3f4", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n// Refs: https://doc.rust-lang.org/reference/destructors.html\n\nuse std::{cell::Cell, panic, pin::Pin, thread};\n\nuse pin_project_lite::pin_project;\n\nstruct D<'a>(&'a Cell, usize);\n\nimpl Drop for D<'_> {\n fn drop(&mut self) {\n if !thread::panicking() {\n let old = self.0.replace(self.1);\n assert_eq!(old, self.1 - 1);\n }\n }\n}\n\npin_project! {\n#[project = StructPinnedProj]\n#[project_ref = StructPinnedProjRef]\n#[project_replace = StructPinnedProjReplace]\nstruct StructPinned<'a> {\n #[pin]\n f1: D<'a>,\n #[pin]\n f2: D<'a>,\n}\n}\n\npin_project! {\n#[project = StructUnpinnedProj]\n#[project_ref = StructUnpinnedProjRef]\n#[project_replace = StructUnpinnedProjReplace]\nstruct StructUnpinned<'a> {\n f1: D<'a>,\n f2: D<'a>,\n}\n}\n\npin_project! {\n#[project_replace = EnumProjReplace]\nenum Enum<'a> {\n #[allow(dead_code)] // false positive that fixed in Rust 1.38\n StructPinned {\n #[pin]\n f1: D<'a>,\n #[pin]\n f2: D<'a>,\n },\n #[allow(dead_code)] // false positive that fixed in Rust 1.38\n StructUnpinned {\n f1: D<'a>,\n f2: D<'a>,\n },\n}\n}\n\n#[test]\nfn struct_pinned() {\n {\n let c = Cell::new(0);\n let _x = StructPinned { f1: D(&c, 1), f2: D(&c, 2) };\n }\n {\n let c = Cell::new(0);\n let mut x = StructPinned { f1: D(&c, 1), f2: D(&c, 2) };\n let y = Pin::new(&mut x);\n let _z = y.project_replace(StructPinned { f1: D(&c, 3), f2: D(&c, 4) });\n }\n}\n\n#[test]\nfn struct_unpinned() {\n {\n let c = Cell::new(0);\n let _x = StructUnpinned { f1: D(&c, 1), f2: D(&c, 2) };\n }\n {\n let c = Cell::new(0);\n let mut x = StructUnpinned { f1: D(&c, 1), f2: D(&c, 2) };\n let y = Pin::new(&mut x);\n let _z = y.project_replace(StructUnpinned { f1: D(&c, 3), f2: D(&c, 4) });\n }\n}\n\n#[test]\nfn enum_struct() {\n {\n let c = Cell::new(0);\n let _x = Enum::StructPinned { f1: D(&c, 1), f2: D(&c, 2) };\n }\n {\n let c = Cell::new(0);\n let mut x = Enum::StructPinned { f1: D(&c, 1), f2: D(&c, 2) };\n let y = Pin::new(&mut x);\n let _z = y.project_replace(Enum::StructPinned { f1: D(&c, 3), f2: D(&c, 4) });\n }\n\n {\n let c = Cell::new(0);\n let _x = Enum::StructUnpinned { f1: D(&c, 1), f2: D(&c, 2) };\n }\n {\n let c = Cell::new(0);\n let mut x = Enum::StructUnpinned { f1: D(&c, 1), f2: D(&c, 2) };\n let y = Pin::new(&mut x);\n let _z = y.project_replace(Enum::StructUnpinned { f1: D(&c, 3), f2: D(&c, 4) });\n }\n}\n\n// https://github.com/rust-lang/rust/issues/47949\n// https://github.com/taiki-e/pin-project/pull/194#discussion_r419098111\n#[allow(clippy::many_single_char_names)]\n#[test]\nfn project_replace_panic() {\n pin_project! {\n #[project_replace = SProjReplace]\n struct S {\n #[pin]\n pinned: T,\n unpinned: U,\n }\n }\n\n struct D<'a>(&'a mut bool, bool);\n impl Drop for D<'_> {\n fn drop(&mut self) {\n *self.0 = true;\n if self.1 {\n panic!();\n }\n }\n }\n\n let (mut a, mut b, mut c, mut d) = (false, false, false, false);\n let res = panic::catch_unwind(panic::AssertUnwindSafe(|| {\n let mut x = S { pinned: D(&mut a, true), unpinned: D(&mut b, false) };\n let _y = Pin::new(&mut x)\n .project_replace(S { pinned: D(&mut c, false), unpinned: D(&mut d, false) });\n // Previous `x.pinned` was dropped and panicked when `project_replace` is\n // called, so this is unreachable.\n unreachable!();\n }));\n assert!(res.is_err());\n assert!(a);\n assert!(b);\n assert!(c);\n assert!(d);\n\n let (mut a, mut b, mut c, mut d) = (false, false, false, false);\n let res = panic::catch_unwind(panic::AssertUnwindSafe(|| {\n let mut x = S { pinned: D(&mut a, false), unpinned: D(&mut b, true) };\n {\n let _y = Pin::new(&mut x)\n .project_replace(S { pinned: D(&mut c, false), unpinned: D(&mut d, false) });\n // `_y` (previous `x.unpinned`) live to the end of this scope, so\n // this is not unreachable.\n // unreachable!();\n }\n unreachable!();\n }));\n assert!(res.is_err());\n assert!(a);\n assert!(b);\n assert!(c);\n assert!(d);\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/default/enum.rs", "sha256": "ebe6b6403e35459420f6fe979243c382972d2d582376d69f6c630ab5ba2599b5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n #[project_replace = EnumProjReplace]\n enum Enum {\n Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n },\n Unit,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/default/struct.rs", "sha256": "05767cd68ec5cddc481cff5ad677b5146f7d6fd1effcf1ff131d24c334d85032", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n struct Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/multifields/enum.rs", "sha256": "d38eb301c86307edbf5cb07ed434b64a10a68aaef32dfb8b15029b5cc5586b0b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n#[project_replace = EnumProjReplace]\nenum Enum {\n Struct {\n #[pin]\n pinned1: T,\n #[pin]\n pinned2: T,\n unpinned1: U,\n unpinned2: U,\n },\n Unit,\n}\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/multifields/struct.rs", "sha256": "1fe2b911e8782eaac5952d9e0914473d46d82db45b1da859bdc6b8dc3978e549", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n#[project_replace = StructProjReplace]\nstruct Struct {\n #[pin]\n pinned1: T,\n #[pin]\n pinned2: T,\n unpinned1: U,\n unpinned2: U,\n}\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/naming/enum-all.rs", "sha256": "ebe6b6403e35459420f6fe979243c382972d2d582376d69f6c630ab5ba2599b5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n #[project_replace = EnumProjReplace]\n enum Enum {\n Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n },\n Unit,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/naming/enum-mut.rs", "sha256": "bf215d055b29a7f9fdb84cabf34576d9dec8e87c33cebe2b553d1fe5e5cb0882", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project = EnumProj]\n enum Enum {\n Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n },\n Unit,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/naming/enum-none.rs", "sha256": "115968816e1a1a6b3634b8594dde4815d4641d817d69ebdd90879b96a8a2808e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n enum Enum {\n Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n },\n Unit,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/naming/enum-ref.rs", "sha256": "e09ce2ca812b9fcf7dc59b486e5b50b84db08860a2ceda63ba094d7351bb391f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project_ref = EnumProjRef]\n enum Enum {\n Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n },\n Unit,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/naming/struct-all.rs", "sha256": "0d6b64bc687fbc19329a5558461329e3d2cfa367ff8adb7be76efe2d3160d33e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project = StructProj]\n #[project_ref = StructProjRef]\n #[project_replace = StructProjReplace]\n struct Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/naming/struct-mut.rs", "sha256": "36edd8db816c122c80ccb139b4305cad8edfdf9b7ee4aebacafa1180613a5075", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project = StructProj]\n struct Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/naming/struct-none.rs", "sha256": "05767cd68ec5cddc481cff5ad677b5146f7d6fd1effcf1ff131d24c334d85032", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n struct Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/naming/struct-ref.rs", "sha256": "dd249875f962d16dd9ec821acf0c2b5c1ed0ed05569f6e874ccd5caca2759acf", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project_ref = StructProjRef]\n struct Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/not_unpin/enum.rs", "sha256": "c65aac4d2083e83d2dfc3bec8ce243e3e9169ef5f986f077f0a8b4226d39f7c3", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project(!Unpin)]\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n enum Enum {\n Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n },\n Unit,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/not_unpin/struct.rs", "sha256": "99d77727393e4a050d2ceb99710172dfdd13dfcb6e6416f63de39e5df3a9f03f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project = StructProj]\n #[project(!Unpin)]\n #[project_ref = StructProjRef]\n struct Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/pinned_drop/enum.rs", "sha256": "8f590791cb4693bb077c562f0d93128f9452e6a5f60c631b5fd624a221883ebd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n enum Enum {\n Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n },\n Unit,\n }\n impl PinnedDrop for Enum {\n fn drop(this: Pin<&mut Self>) {\n let _ = this;\n }\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/pinned_drop/struct.rs", "sha256": "8da271d7d3182b33be297d985a1e3b57d6067406db21eca8ac2c448ef1a0869a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n struct Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n }\n impl PinnedDrop for Struct {\n fn drop(this: Pin<&mut Self>) {\n let _ = this;\n }\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/pub/enum.rs", "sha256": "fdbd806e92cfad4e67737d1f782544aa4e3116bae702161c43b5c22bfbf3a208", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n pub enum Enum {\n Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n },\n Unit,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expand/pub/struct.rs", "sha256": "05cb3f03ef9801fecf0a7c86bbd236c97218b34ac01ee507cb7de9a78efaeffa", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n pub struct Struct {\n #[pin]\n pub pinned: T,\n pub unpinned: U,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/expandtest.rs", "sha256": "a4336ab879d79824ab3144ea385a7a00ba77604c3b6639ae53b6283850b421df", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![cfg(not(miri))]\n\n#[rustversion::attr(not(nightly), ignore)]\n#[test]\nfn expandtest() {\n let args = &[\"--all-features\"];\n macrotest::expand_args(\"tests/expand/**/*.rs\", args);\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/include/basic.rs", "sha256": "8c06c765c92d696284dde5b4762488100ce0ee1661ead307023a85d5c82d8ddc", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n// default pin_project! is completely safe.\n\n::pin_project_lite::pin_project! {\n /// Testing default struct.\n #[derive(Debug)]\n #[allow(clippy::exhaustive_structs)] // for the type itself\n pub struct DefaultStruct {\n #[pin]\n pub pinned: T,\n pub unpinned: U,\n }\n}\n\n::pin_project_lite::pin_project! {\n /// Testing named struct.\n #[allow(clippy::exhaustive_structs)] // for the type itself\n #[project = DefaultStructProj]\n #[project_ref = DefaultStructProjRef]\n #[derive(Debug)]\n pub struct DefaultStructNamed {\n #[pin]\n pub pinned: T,\n pub unpinned: U,\n }\n}\n\n::pin_project_lite::pin_project! {\n /// Testing enum.\n #[allow(clippy::exhaustive_enums)] // for the type itself\n #[project = DefaultEnumProj]\n #[project_ref = DefaultEnumProjRef]\n #[derive(Debug)]\n pub enum DefaultEnum {\n /// Struct variant.\n Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n },\n /// Unit variant.\n Unit,\n }\n}\n\n::pin_project_lite::pin_project! {\n /// Testing pinned drop struct.\n #[allow(clippy::exhaustive_structs)] // for the type itself\n #[derive(Debug)]\n pub struct PinnedDropStruct {\n #[pin]\n pub pinned: T,\n pub unpinned: U,\n }\n impl PinnedDrop for PinnedDropStruct {\n fn drop(_this: Pin<&mut Self>) {}\n }\n}\n\n::pin_project_lite::pin_project! {\n /// Testing pinned drop enum.\n #[allow(clippy::absolute_paths, clippy::exhaustive_enums)] // for the type itself\n #[project = PinnedDropEnumProj]\n #[project_ref = PinnedDropEnumProjRef]\n #[derive(Debug)]\n pub enum PinnedDropEnum {\n /// Struct variant.\n Struct {\n #[pin]\n pinned: T,\n unpinned: U,\n },\n /// Unit variant.\n Unit,\n }\n #[allow(clippy::absolute_paths)]\n impl PinnedDrop for PinnedDropEnum {\n fn drop(_this: Pin<&mut Self>) {}\n }\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/proper_unpin.rs", "sha256": "83790eb38ae2fd5aed97fc67da79d91c1071f08d1e5aaef0c71e65d8b5767403", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![allow(dead_code)]\n\n#[macro_use]\nmod auxiliary;\n\npub mod default {\n use std::marker::PhantomPinned;\n\n use pin_project_lite::pin_project;\n\n struct Inner {\n f: T,\n }\n\n assert_unpin!(Inner<()>);\n assert_not_unpin!(Inner);\n\n pin_project! {\n struct Struct {\n #[pin]\n f1: Inner,\n f2: U,\n }\n }\n\n assert_unpin!(Struct<(), ()>);\n assert_unpin!(Struct<(), PhantomPinned>);\n assert_not_unpin!(Struct);\n assert_not_unpin!(Struct);\n\n pin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n enum Enum {\n V1 {\n #[pin]\n f1: Inner,\n f2: U,\n },\n }\n }\n\n assert_unpin!(Enum<(), ()>);\n assert_unpin!(Enum<(), PhantomPinned>);\n assert_not_unpin!(Enum);\n assert_not_unpin!(Enum);\n\n pin_project! {\n #[project(!Unpin)]\n enum NotUnpinEnum {\n V1 {\n #[pin] f1: Inner,\n f2: U,\n }\n }\n }\n\n assert_not_unpin!(NotUnpinEnum<(), ()>);\n\n pin_project! {\n struct TrivialBounds {\n #[pin]\n f: PhantomPinned,\n }\n }\n\n assert_not_unpin!(TrivialBounds);\n\n pin_project! {\n struct PinRef<'a, T, U> {\n #[pin]\n f1: &'a mut Inner,\n f2: U,\n }\n }\n\n assert_unpin!(PinRef<'_, PhantomPinned, PhantomPinned>);\n\n pin_project! {\n #[project(!Unpin)]\n struct NotUnpin {\n #[pin]\n u: U\n }\n }\n\n assert_not_unpin!(NotUnpin<()>);\n\n pin_project! {\n #[project(!Unpin)]\n struct NotUnpinRef<'a, T, U> {\n #[pin]\n f1: &'a mut Inner,\n f2: U\n }\n }\n\n assert_not_unpin!(NotUnpinRef<'_, (), ()>);\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/test.rs", "sha256": "3bfff987a820770bd0126656a77871d5609f21c9c3b6825bd1606561aa37b572", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![allow(dead_code, unreachable_pub, clippy::no_effect_underscore_binding)]\n\n#[macro_use]\nmod auxiliary;\n\nuse std::{\n marker::{PhantomData, PhantomPinned},\n pin::Pin,\n};\n\nuse pin_project_lite::pin_project;\n\n#[test]\nfn projection() {\n pin_project! {\n #[project = StructProj]\n #[project_ref = StructProjRef]\n #[project_replace = StructProjReplace]\n #[derive(Default)]\n struct Struct {\n #[pin]\n f1: T,\n f2: U,\n }\n }\n\n let mut s = Struct { f1: 1, f2: 2 };\n let mut s_orig = Pin::new(&mut s);\n let s = s_orig.as_mut().project();\n\n let _: Pin<&mut i32> = s.f1;\n assert_eq!(*s.f1, 1);\n let _: &mut i32 = s.f2;\n assert_eq!(*s.f2, 2);\n\n assert_eq!(s_orig.as_ref().f1, 1);\n assert_eq!(s_orig.as_ref().f2, 2);\n\n let mut s = Struct { f1: 1, f2: 2 };\n let mut s = Pin::new(&mut s);\n {\n let StructProj { f1, f2 } = s.as_mut().project();\n let _: Pin<&mut i32> = f1;\n let _: &mut i32 = f2;\n }\n {\n let StructProjRef { f1, f2 } = s.as_ref().project_ref();\n let _: Pin<&i32> = f1;\n let _: &i32 = f2;\n }\n\n {\n let StructProjReplace { f1: PhantomData, f2 } =\n s.as_mut().project_replace(Struct::default());\n assert_eq!(f2, 2);\n let StructProj { f1, f2 } = s.project();\n assert_eq!(*f1, 0);\n assert_eq!(*f2, 0);\n }\n\n pin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n #[project_replace = EnumProjReplace]\n #[derive(Eq, PartialEq, Debug)]\n enum Enum {\n Struct {\n #[pin]\n f1: C,\n f2: D,\n },\n Unit,\n }\n }\n\n let mut e = Enum::Struct { f1: 1, f2: 2 };\n let mut e = Pin::new(&mut e);\n\n match e.as_mut().project() {\n EnumProj::Struct { f1, f2 } => {\n let _: Pin<&mut i32> = f1;\n assert_eq!(*f1, 1);\n let _: &mut i32 = f2;\n assert_eq!(*f2, 2);\n }\n EnumProj::Unit => unreachable!(),\n }\n\n assert_eq!(&*e, &Enum::Struct { f1: 1, f2: 2 });\n\n if let EnumProj::Struct { f1, f2 } = e.as_mut().project() {\n let _: Pin<&mut i32> = f1;\n assert_eq!(*f1, 1);\n let _: &mut i32 = f2;\n assert_eq!(*f2, 2);\n }\n\n if let EnumProjReplace::Struct { f1: PhantomData, f2 } = e.as_mut().project_replace(Enum::Unit)\n {\n assert_eq!(f2, 2);\n }\n}\n\n#[test]\nfn enum_project_set() {\n pin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n #[derive(Eq, PartialEq, Debug)]\n enum Enum {\n V1 { #[pin] f: u8 },\n V2 { f: bool },\n }\n }\n\n let mut e = Enum::V1 { f: 25 };\n let mut e_orig = Pin::new(&mut e);\n let e_proj = e_orig.as_mut().project();\n\n match e_proj {\n EnumProj::V1 { f } => {\n let new_e = Enum::V2 { f: f.as_ref().get_ref() == &25 };\n e_orig.set(new_e);\n }\n EnumProj::V2 { .. } => unreachable!(),\n }\n\n assert_eq!(e, Enum::V2 { f: true });\n}\n\n#[test]\nfn where_clause() {\n pin_project! {\n struct Struct\n where\n T: Copy,\n {\n f: T,\n }\n }\n\n pin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n enum Enum\n where\n T: Copy,\n {\n V { f: T },\n }\n }\n}\n\n#[test]\nfn where_clause_and_associated_type_field() {\n pin_project! {\n struct Struct1\n where\n I: Iterator,\n {\n #[pin]\n f1: I,\n f2: I::Item,\n }\n }\n\n pin_project! {\n struct Struct2\n where\n I: Iterator,\n {\n #[pin]\n f1: I,\n f2: J,\n }\n }\n\n pin_project! {\n pub struct Struct3\n where\n T: 'static,\n {\n f: T,\n }\n }\n\n trait Static: 'static {}\n\n impl Static for Struct3 {}\n\n pin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n enum Enum\n where\n I: Iterator,\n {\n V1 { #[pin] f: I },\n V2 { f: I::Item },\n }\n }\n}\n\n#[test]\nfn derive_copy() {\n pin_project! {\n #[derive(Clone, Copy)]\n struct Struct {\n f: T,\n }\n }\n\n fn is_copy() {}\n\n is_copy::>();\n}\n\n#[test]\nfn move_out() {\n struct NotCopy;\n\n pin_project! {\n struct Struct {\n f: NotCopy,\n }\n }\n\n let x = Struct { f: NotCopy };\n let _val: NotCopy = x.f;\n\n pin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n enum Enum {\n V { f: NotCopy },\n }\n }\n\n let x = Enum::V { f: NotCopy };\n #[allow(clippy::infallible_destructuring_match)]\n let _val: NotCopy = match x {\n Enum::V { f } => f,\n };\n}\n\n#[test]\nfn trait_bounds_on_type_generics() {\n pin_project! {\n pub struct Struct1<'a, T: ?Sized> {\n f: &'a mut T,\n }\n }\n\n pin_project! {\n pub struct Struct2<'a, T: ::core::fmt::Debug> {\n f: &'a mut T,\n }\n }\n\n pin_project! {\n pub struct Struct3<'a, T: core::fmt::Debug> {\n f: &'a mut T,\n }\n }\n\n // pin_project! {\n // pub struct Struct4<'a, T: core::fmt::Debug + core::fmt::Display> {\n // f: &'a mut T,\n // }\n // }\n\n // pin_project! {\n // pub struct Struct5<'a, T: core::fmt::Debug + ?Sized> {\n // f: &'a mut T,\n // }\n // }\n\n pin_project! {\n pub struct Struct6<'a, T: core::fmt::Debug = [u8; 16]> {\n f: &'a mut T,\n }\n }\n\n let _: Struct6<'_> = Struct6 { f: &mut [0_u8; 16] };\n\n pin_project! {\n pub struct Struct7 {\n f: T,\n }\n }\n\n trait Static: 'static {}\n\n impl Static for Struct7 {}\n\n pin_project! {\n pub struct Struct8<'a, 'b: 'a> {\n f1: &'a u8,\n f2: &'b u8,\n }\n }\n\n pin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n enum Enum<'a, T: ?Sized> {\n V { f: &'a mut T },\n }\n }\n}\n\n#[test]\nfn private_type_in_public_type() {\n pin_project! {\n pub struct PublicStruct {\n #[pin]\n inner: PrivateStruct,\n }\n }\n\n struct PrivateStruct(T);\n}\n\n#[allow(clippy::needless_lifetimes)]\n#[test]\nfn lifetime_project() {\n pin_project! {\n struct Struct1 {\n #[pin]\n pinned: T,\n unpinned: U,\n }\n }\n\n pin_project! {\n struct Struct2<'a, T, U> {\n #[pin]\n pinned: &'a T,\n unpinned: U,\n }\n }\n\n pin_project! {\n #[project = EnumProj]\n #[project_ref = EnumProjRef]\n enum Enum {\n V {\n #[pin]\n pinned: T,\n unpinned: U,\n },\n }\n }\n\n impl Struct1 {\n fn get_pin_ref<'a>(self: Pin<&'a Self>) -> Pin<&'a T> {\n self.project_ref().pinned\n }\n fn get_pin_mut<'a>(self: Pin<&'a mut Self>) -> Pin<&'a mut T> {\n self.project().pinned\n }\n fn get_pin_ref_elided(self: Pin<&Self>) -> Pin<&T> {\n self.project_ref().pinned\n }\n fn get_pin_mut_elided(self: Pin<&mut Self>) -> Pin<&mut T> {\n self.project().pinned\n }\n }\n\n impl<'b, T, U> Struct2<'b, T, U> {\n fn get_pin_ref<'a>(self: Pin<&'a Self>) -> Pin<&'a &'b T> {\n self.project_ref().pinned\n }\n fn get_pin_mut<'a>(self: Pin<&'a mut Self>) -> Pin<&'a mut &'b T> {\n self.project().pinned\n }\n fn get_pin_ref_elided(self: Pin<&Self>) -> Pin<&&'b T> {\n self.project_ref().pinned\n }\n fn get_pin_mut_elided(self: Pin<&mut Self>) -> Pin<&mut &'b T> {\n self.project().pinned\n }\n }\n\n impl Enum {\n fn get_pin_ref<'a>(self: Pin<&'a Self>) -> Pin<&'a T> {\n match self.project_ref() {\n EnumProjRef::V { pinned, .. } => pinned,\n }\n }\n fn get_pin_mut<'a>(self: Pin<&'a mut Self>) -> Pin<&'a mut T> {\n match self.project() {\n EnumProj::V { pinned, .. } => pinned,\n }\n }\n fn get_pin_ref_elided(self: Pin<&Self>) -> Pin<&T> {\n match self.project_ref() {\n EnumProjRef::V { pinned, .. } => pinned,\n }\n }\n fn get_pin_mut_elided(self: Pin<&mut Self>) -> Pin<&mut T> {\n match self.project() {\n EnumProj::V { pinned, .. } => pinned,\n }\n }\n }\n}\n\nmod visibility {\n use pin_project_lite::pin_project;\n\n pin_project! {\n pub(crate) struct S {\n pub f: u8,\n }\n }\n}\n\n#[test]\nfn visibility() {\n let mut x = visibility::S { f: 0 };\n let x = Pin::new(&mut x);\n let y = x.as_ref().project_ref();\n let _: &u8 = y.f;\n let y = x.project();\n let _: &mut u8 = y.f;\n}\n\n#[test]\nfn trivial_bounds() {\n pin_project! {\n pub struct NoGenerics {\n #[pin]\n f: PhantomPinned,\n }\n }\n\n assert_not_unpin!(NoGenerics);\n}\n\n#[test]\nfn dst() {\n pin_project! {\n pub struct Struct1 {\n f: T,\n }\n }\n\n let mut x = Struct1 { f: 0_u8 };\n let x: Pin<&mut Struct1> = Pin::new(&mut x);\n let _: &mut dyn core::fmt::Debug = x.project().f;\n\n pin_project! {\n pub struct Struct2 {\n #[pin]\n f: T,\n }\n }\n\n let mut x = Struct2 { f: 0_u8 };\n let x: Pin<&mut Struct2> = Pin::new(&mut x);\n let _: Pin<&mut (dyn core::fmt::Debug + Unpin)> = x.project().f;\n\n pin_project! {\n struct Struct3\n where\n T: ?Sized,\n {\n f: T,\n }\n }\n\n pin_project! {\n struct Struct4\n where\n T: ?Sized,\n {\n #[pin]\n f: T,\n }\n }\n\n pin_project! {\n struct Struct11<'a, T: ?Sized, U: ?Sized> {\n f1: &'a mut T,\n f2: U,\n }\n }\n}\n\n#[test]\nfn dyn_type() {\n pin_project! {\n struct Struct1 {\n f: dyn core::fmt::Debug,\n }\n }\n\n pin_project! {\n struct Struct2 {\n #[pin]\n f: dyn core::fmt::Debug,\n }\n }\n\n pin_project! {\n struct Struct3 {\n f: dyn core::fmt::Debug + Send,\n }\n }\n\n pin_project! {\n struct Struct4 {\n #[pin]\n f: dyn core::fmt::Debug + Send,\n }\n }\n}\n\n#[test]\nfn no_infer_outlives() {\n trait Trait {\n type Y;\n }\n\n struct Struct1(A);\n\n impl Trait for Struct1 {\n type Y = Option;\n }\n\n pin_project! {\n struct Struct2 {\n _f: as Trait>::Y,\n }\n }\n}\n\n// https://github.com/taiki-e/pin-project-lite/issues/31\n#[test]\nfn trailing_comma() {\n pub trait T {}\n\n pin_project! {\n pub struct S1<\n A: T,\n B: T,\n > {\n f: (A, B),\n }\n }\n\n pin_project! {\n pub struct S2<\n A,\n B,\n >\n where\n A: T,\n B: T,\n {\n f: (A, B),\n }\n }\n\n pin_project! {\n #[allow(explicit_outlives_requirements)]\n pub struct S3<\n 'a,\n A: 'a,\n B: 'a,\n > {\n f: &'a (A, B),\n }\n }\n\n // pin_project! {\n // pub struct S4<\n // 'a,\n // 'b: 'a, // <-----\n // > {\n // f: &'a &'b (),\n // }\n // }\n}\n\n#[test]\nfn attrs() {\n pin_project! {\n /// dox1\n #[derive(Clone)]\n #[project = StructProj]\n #[project_ref = StructProjRef]\n /// dox2\n #[derive(Debug)]\n /// dox3\n struct Struct {\n // TODO\n // /// dox4\n f: ()\n }\n }\n\n pin_project! {\n #[project = Enum1Proj]\n #[project_ref = Enum1ProjRef]\n enum Enum1 {\n #[cfg(not(any()))]\n V {\n f: ()\n },\n }\n }\n\n pin_project! {\n /// dox1\n #[derive(Clone)]\n #[project(!Unpin)]\n #[project = Enum2Proj]\n #[project_ref = Enum2ProjRef]\n /// dox2\n #[derive(Debug)]\n /// dox3\n enum Enum2 {\n /// dox4\n V1 {\n // TODO\n // /// dox5\n f: ()\n },\n /// dox6\n V2,\n }\n }\n}\n\n#[test]\nfn pinned_drop() {\n pin_project! {\n pub struct Struct1<'a> {\n was_dropped: &'a mut bool,\n #[pin]\n field: u8,\n }\n impl PinnedDrop for Struct1<'_> {\n fn drop(this: Pin<&mut Self>) {\n **this.project().was_dropped = true;\n }\n }\n }\n\n let mut was_dropped = false;\n drop(Struct1 { was_dropped: &mut was_dropped, field: 42 });\n assert!(was_dropped);\n\n pin_project! {\n pub struct Struct2<'a> {\n was_dropped: &'a mut bool,\n #[pin]\n field: u8,\n }\n impl PinnedDrop for Struct2<'_> {\n fn drop(mut this: Pin<&mut Self>) {\n **this.as_mut().project().was_dropped = true;\n }\n }\n }\n\n trait Service {\n type Error;\n }\n\n pin_project! {\n struct Struct3<'a, T, Request>\n where\n T: Service,\n T::Error: std::error::Error,\n {\n was_dropped: &'a mut bool,\n #[pin]\n field: T,\n req: Request,\n }\n\n /// dox1\n impl PinnedDrop for Struct3<'_, T, Request>\n where\n T: Service,\n T::Error: std::error::Error,\n {\n /// dox2\n fn drop(mut this: Pin<&mut Self>) {\n **this.as_mut().project().was_dropped = true;\n }\n }\n }\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/not_unpin/conflict-unpin.rs", "sha256": "23c7e081d86f2000838c32844dce1eee98d1b118eb1a29daf65c705aad5c68fb", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n #[project(!Unpin)]\n struct Foo {\n #[pin]\n f1: T,\n f2: U,\n }\n}\n\nimpl Unpin for Foo where T: Unpin {}\n\npin_project! {\n #[project(!Unpin)]\n struct Bar {\n #[pin]\n f1: T,\n f2: U,\n }\n}\n\nimpl Unpin for Bar {}\n\npin_project! {\n #[project(!Unpin)]\n struct Baz {\n #[pin]\n f1: T,\n f2: U,\n }\n}\n\nimpl Unpin for Baz {}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/not_unpin/negative_impls_stable.rs", "sha256": "f014d5398d505ca5f482905cb9597a2d5fced563b9350229468c3751dd7f92ae", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n// https://github.com/taiki-e/pin-project/issues/340#issuecomment-2428002670\n\npin_project_lite::pin_project! {\n #[project(!Unpin)]\n struct Foo {\n #[pin]\n pinned: Pinned,\n unpinned: Unpinned,\n }\n}\n\nstruct MyPhantomPinned(::core::marker::PhantomPinned);\nimpl Unpin for MyPhantomPinned where for<'cursed> str: Sized {}\nimpl Unpin for Foo {}\n\nfn is_unpin() {}\n\nfn main() {\n is_unpin::>()\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pin_project/conflict-drop.rs", "sha256": "a1964fa136080cc354479ff55b8428e5fa79305681fb2475cc37d6071cd64112", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! { //~ ERROR E0119\n struct Foo {\n #[pin]\n future: T,\n field: U,\n }\n}\n\nimpl Drop for Foo {\n fn drop(&mut self) {}\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pin_project/conflict-unpin.rs", "sha256": "37e7b2f9ab1be70d0521c9dbbc954067f22a2321511e008f0f72e13d0edb4afe", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\n// The same implementation.\n\npin_project! { //~ ERROR E0119\n struct Foo {\n #[pin]\n future: T,\n field: U,\n }\n}\n\n// conflicting implementations\nimpl Unpin for Foo where T: Unpin {} // Conditional Unpin impl\n\n// The implementation that under different conditions.\n\npin_project! { //~ ERROR E0119\n struct Bar {\n #[pin]\n future: T,\n field: U,\n }\n}\n\n// conflicting implementations\nimpl Unpin for Bar {} // Non-conditional Unpin impl\n\npin_project! { //~ ERROR E0119\n struct Baz {\n #[pin]\n future: T,\n field: U,\n }\n}\n\n// conflicting implementations\nimpl Unpin for Baz {} // Conditional Unpin impl\n\npin_project! { //~ ERROR E0119\n #[project(!Unpin)]\n struct Qux {\n #[pin]\n future: T,\n field: U,\n }\n}\n\n// conflicting implementations\nimpl Unpin for Qux {} // Non-conditional Unpin impl\n\npin_project! { //~ ERROR E0119\n #[project(!Unpin)]\n struct Fred {\n #[pin]\n future: T,\n field: U,\n }\n}\n\n// conflicting implementations\nimpl Unpin for Fred {} // Conditional Unpin impl\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pin_project/invalid-bounds.rs", "sha256": "4068e272a4f5d44851e0901dd6cfa806ead000c2e1a507a2b42be51d26d17056", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n struct Generics1 { //~ ERROR no rules expected the token `:`\n field: T,\n }\n}\n\npin_project! {\n struct Generics2 { //~ ERROR no rules expected the token `:`\n field: T,\n }\n}\n\npin_project! {\n struct Generics3 { //~ ERROR expected one of `+`, `,`, `=`, or `>`, found `:`\n field: T,\n }\n}\n\npin_project! {\n struct Generics4 { //~ ERROR expected one of `+`, `,`, `=`, or `>`, found `:`\n field: T,\n }\n}\n\npin_project! {\n struct Generics5 { //~ ERROR expected one of `+`, `,`, `=`, or `>`, found `:`\n field: T,\n }\n}\n\npin_project! {\n struct Generics6 { //~ ERROR no rules expected the token `Sized`\n field: T,\n }\n}\n\npin_project! {\n struct WhereClause1\n where\n T: 'static : Sized //~ ERROR no rules expected the token `:`\n {\n field: T,\n }\n}\n\npin_project! {\n struct WhereClause2\n where\n T: 'static : ?Sized //~ ERROR no rules expected the token `:`\n {\n field: T,\n }\n}\n\npin_project! {\n struct WhereClause3\n where\n T: Sized : 'static //~ ERROR expected `where`, or `{` after struct name, found `:`\n {\n field: T,\n }\n}\n\npin_project! {\n struct WhereClause4\n where\n T: ?Sized : 'static //~ ERROR expected `where`, or `{` after struct name, found `:`\n {\n field: T,\n }\n}\n\npin_project! {\n struct WhereClause5\n where\n T: Sized : ?Sized //~ ERROR expected `where`, or `{` after struct name, found `:`\n {\n field: T,\n }\n}\n\npin_project! {\n struct WhereClause6\n where\n T: ?Sized : Sized //~ ERROR no rules expected the token `Sized`\n {\n field: T,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pin_project/invalid.rs", "sha256": "1c84cc089623dd31f483011ed6543f169af341b028a212fcf00146bc96dd792f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n struct A {\n #[pin()] //~ ERROR no rules expected the token `(`\n pinned: T,\n }\n}\n\npin_project! {\n #[pin] //~ ERROR cannot find attribute `pin` in this scope\n struct B {\n pinned: T,\n }\n}\n\npin_project! {\n struct C {\n #[pin]\n #[pin] //~ ERROR no rules expected the token `#`\n pinned: T,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pin_project/negative_impls_stable.rs", "sha256": "1e70055fbec4bc0a53454fd93a0663f8785440d37dcf359ab29958d97fd51e9d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n// https://github.com/taiki-e/pin-project/issues/340#issuecomment-2428002670\n\npin_project_lite::pin_project! {\n struct Foo {\n #[pin]\n pinned: Pinned,\n unpinned: Unpinned,\n }\n}\n\nstruct MyPhantomPinned(::core::marker::PhantomPinned);\nimpl Unpin for MyPhantomPinned where for<'cursed> str: Sized {}\nimpl Unpin for Foo {}\n\nfn is_unpin() {}\n\nfn main() {\n is_unpin::>()\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pin_project/overlapping_lifetime_names.rs", "sha256": "ac9c29b389e8e067e9e384cf04be69c0e5bb862cf373de0e3b4296fe2007809b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! { //~ ERROR E0263,E0496\n pub struct Foo<'__pin, T> {\n #[pin]\n field: &'__pin mut T,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pin_project/overlapping_unpin_struct.rs", "sha256": "cb3d6d4087b8b4c917bc71e55764775f6bb018a77cda5c6b726afacd66789859", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse std::marker::PhantomPinned;\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n struct Foo {\n #[pin]\n inner: T,\n }\n}\n\nstruct __Origin {}\n\nimpl Unpin for __Origin {}\n\nfn is_unpin() {}\n\nfn main() {\n is_unpin::>(); //~ ERROR E0277\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pin_project/packed.rs", "sha256": "6a24ac14e61e5d3e3b495f49d226318ec500e5aa46f34acf8e4f3e0827bd9604", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! { //~ ERROR reference to packed field is unaligned\n #[repr(packed, C)]\n struct Packed {\n #[pin]\n field: u16,\n }\n}\n\npin_project! { //~ ERROR reference to packed field is unaligned\n #[repr(packed(2))]\n struct PackedN {\n #[pin]\n field: u32,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pin_project/unpin_sneaky.rs", "sha256": "6e6385e067679c3bf5233163b3fa99ff2fc28bcb93fe594a5c987c0db7c0eaa6", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n struct Foo {\n #[pin]\n inner: u8,\n }\n}\n\nimpl Unpin for __Origin {} //~ ERROR E0412,E0321\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pin_project/unsupported.rs", "sha256": "3ad5f584f9e56be3573133fba026a7bb378595d0a4b7b1bcbb6b78e37cef20a7", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n struct Struct1 {} //~ ERROR no rules expected the token `}`\n}\n\npin_project! {\n struct Struct2(); //~ ERROR no rules expected the token `(`\n}\n\npin_project! {\n struct Struct3; //~ ERROR no rules expected the token `;`\n}\n\npin_project! {\n enum Enum { //~ ERROR no rules expected the token `enum`\n A(u8)\n }\n}\n\npin_project! {\n union Union { //~ ERROR no rules expected the token `union`\n x: u8,\n }\n}\n\nfn main() {}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pinned_drop/call-drop-inner.rs", "sha256": "0618b60a450c99b5953b302e26c68df1abe7632af3d9baf3639eb60091e4ff4e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\npin_project! {\n pub struct S {\n #[pin]\n field: u8,\n }\n impl PinnedDrop for S {\n fn drop(this: Pin<&mut Self>) {\n __drop_inner(this);\n }\n }\n}\n\nfn main() {\n let _x = S { field: 0 };\n}\n" }, { "package": "pin-project-lite@0.2.17", "sourcePath": "tests/ui/pinned_drop/conditional-drop-impl.rs", "sha256": "df03f38b2a331f339af2734fdd4ad2f637b2e2189d9928599adebe04ebee517f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse pin_project_lite::pin_project;\n\n// In `Drop` impl, the implementor must specify the same requirement as type definition.\n\nstruct DropImpl {\n f: T,\n}\n\nimpl Drop for DropImpl {\n //~^ ERROR E0367\n fn drop(&mut self) {}\n}\n\npin_project! {\n //~^ ERROR E0367\n struct PinnedDropImpl {\n #[pin]\n f: T,\n }\n\n impl PinnedDrop for PinnedDropImpl {\n fn drop(_this: Pin<&mut Self>) {}\n }\n}\n\nfn main() {}\n" }, { "package": "png@0.17.5", "sourcePath": "README.md", "sha256": "a87e2bc972068409cff0665241349c4eb72291b33d12ed97a09f97e9a2560655", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# PNG Decoder/Encoder\n[![Build Status](https://github.com/image-rs/image-png/workflows/Rust%20CI/badge.svg)](https://github.com/image-rs/image-png/actions)\n[![Documentation](https://docs.rs/png/badge.svg)](https://docs.rs/png)\n[![Crates.io](https://img.shields.io/crates/v/png.svg)](https://crates.io/crates/png)\n![Lines of Code](https://tokei.rs/b1/github/image-rs/image-png)\n[![License](https://img.shields.io/crates/l/png.svg)](https://github.com/image-rs/image-png)\n[![fuzzit](https://app.fuzzit.dev/badge?org_id=image-rs)](https://app.fuzzit.dev/orgs/image-rs/dashboard)\n\nPNG decoder/encoder in pure Rust.\n\nIt contains all features required to handle the entirety of [the PngSuite by\nWillem van Schack][PngSuite].\n\n[PngSuite]: http://www.schaik.com/pngsuite2011/pngsuite.html\n\n## pngcheck\n\nThe `pngcheck` utility is a small demonstration binary that checks and prints\nmetadata on every `.png` image provided via parameter. You can run it (for\nexample on the test directories) with\n\n```bash\ncargo run --release --example pngcheck ./tests/pngsuite/*\n```\n\n## License\n\nLicensed under either of\n\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or https://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or https://opensource.org/licenses/MIT)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any\nadditional terms or conditions.\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "README.md", "sha256": "a66e02c23633b74d19013225a52fe26742b59b9575a5cb0451f53e8e7468b622", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# portable-atomic\n\n[![crates.io](https://img.shields.io/crates/v/portable-atomic?style=flat-square&logo=rust)](https://crates.io/crates/portable-atomic)\n[![docs.rs](https://img.shields.io/badge/docs.rs-portable--atomic-blue?style=flat-square&logo=docs.rs)](https://docs.rs/portable-atomic)\n[![license](https://img.shields.io/badge/license-Apache--2.0_OR_MIT-blue?style=flat-square)](#license)\n[![msrv](https://img.shields.io/badge/msrv-1.34-blue?style=flat-square&logo=rust)](https://www.rust-lang.org)\n[![github actions](https://img.shields.io/github/actions/workflow/status/taiki-e/portable-atomic/ci.yml?branch=main&style=flat-square&logo=github)](https://github.com/taiki-e/portable-atomic/actions)\n\n\n\nPortable atomic types including support for 128-bit atomics, atomic float, etc.\n\n- Provide all atomic integer types (`Atomic{I,U}{8,16,32,64}`) for all targets that can use atomic CAS. (i.e., all targets that can use `std`, and most no-std targets)\n- Provide `AtomicI128` and `AtomicU128`.\n- Provide `AtomicF32` and `AtomicF64`. ([optional, requires the `float` feature](#optional-features-float))\n- Provide `AtomicF16` and `AtomicF128` for [unstable `f16` and `f128`](https://github.com/rust-lang/rust/issues/116909). ([optional, requires the `float` feature and unstable cfgs](#optional-features-float))\n- Provide atomic load/store for targets where atomic is not available at all in the standard library. (RISC-V without A-extension, MSP430, AVR)\n- Provide atomic CAS for targets where atomic CAS is not available in the standard library. (thumbv6m, pre-v6 Arm, RISC-V without A-extension, MSP430, AVR, Xtensa, etc.) (always enabled for MSP430 and AVR, [optional](#optional-features-critical-section) otherwise)\n- Make features that require newer compilers, such as [`fetch_{max,min}`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.fetch_max), [`fetch_update`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.fetch_update), [`as_ptr`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.as_ptr), [`from_ptr`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.from_ptr), [`AtomicBool::fetch_not`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicBool.html#method.fetch_not), [`AtomicPtr::fetch_*`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicPtr.html#method.fetch_and), and [stronger CAS failure ordering](https://github.com/rust-lang/rust/pull/98383) available on Rust 1.34+.\n- Provide workaround for bugs in the standard library's atomic-related APIs, such as [rust-lang/compiler-builtins#1234], [rust-lang/rust#100650], `fence`/`compiler_fence` on MSP430 that cause LLVM error, etc.\n\n\n\nportable-atomic version of `std::sync::Arc` and `std::task::Wake` is provided by the [portable-atomic-util] crate.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nportable-atomic = \"1\"\n```\n\nThe default features are mainly for users who use atomics larger than the pointer width.\nIf you don't need them, disabling the default features may reduce code size and compile time slightly.\n\n```toml\n[dependencies]\nportable-atomic = { version = \"1\", default-features = false }\n```\n\nIf your crate supports no-std environment and requires atomic CAS, enabling the `require-cas` feature will allow the portable-atomic to display a [helpful error message](https://github.com/taiki-e/portable-atomic/pull/100) to users on targets requiring additional action on the user side to provide atomic CAS.\n\n```toml\n[dependencies]\nportable-atomic = { version = \"1.3\", default-features = false, features = [\"require-cas\"] }\n```\n\n(Since 1.8, portable-atomic can display a [helpful error message](https://github.com/taiki-e/portable-atomic/pull/181) even without the `require-cas` feature when the rustc version is 1.78+. However, the `require-cas` feature also allows rejecting builds at an earlier stage, we recommend enabling it unless enabling it causes [problems](https://github.com/matklad/once_cell/pull/267).)\n\n## 128-bit atomics support\n\nNative 128-bit atomic operations are available on x86_64 (Rust 1.59+), AArch64 (Rust 1.59+), riscv64 (Rust 1.59+), Arm64EC (Rust 1.84+), s390x (Rust 1.84+), and powerpc64 (Rust 1.95+), otherwise the fallback implementation is used.\n\nOn x86_64, even if `cmpxchg16b` is not available at compile-time (Note: `cmpxchg16b` target feature is enabled by default only on Apple, Windows (except Windows 7), and Fuchsia targets), run-time detection checks whether `cmpxchg16b` is available. If `cmpxchg16b` is not available at either compile-time or run-time detection, the fallback implementation is used. See also [`portable_atomic_no_outline_atomics`](#optional-cfg-no-outline-atomics) cfg.\n\nThey are usually implemented using inline assembly, and when using Miri or ThreadSanitizer that do not support inline assembly, core intrinsics are used instead of inline assembly if possible.\n\nSee the [`atomic128` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/atomic128/README.md) for details.\n\n## Optional features/cfgs\n\nportable-atomic provides features and cfgs to allow enabling specific APIs and customizing its behavior.\n\nSome options have both a feature and a cfg. When both exist, it indicates that the feature does not follow Cargo's recommendation that [features should be additive](https://doc.rust-lang.org/nightly/cargo/reference/features.html#feature-unification). Therefore, the maintainer's recommendation is to use cfg instead of feature. However, in the embedded ecosystem, it is very common to use features in such places, so these options provide both so you can choose based on your preference.\n\n

\nHow to enable cfg (click to show)\n\nOne of the ways to enable cfg is to set [rustflags in the cargo config](https://doc.rust-lang.org/cargo/reference/config.html#targettriplerustflags):\n\n```toml\n# .cargo/config.toml\n[target.]\nrustflags = [\"--cfg\", \"portable_atomic_unsafe_assume_single_core\"]\n```\n\nOr set environment variable:\n\n```sh\nRUSTFLAGS=\"--cfg portable_atomic_unsafe_assume_single_core\" cargo ...\n```\n\n
\n\n- **`fallback` feature** *(enabled by default)*
\n Enable fallback implementations.\n\n This enables atomic types with larger than the width supported by atomic instructions available on the current target. If the current target [supports 128-bit atomics](#128-bit-atomics-support), this is no-op.\n\n This uses fallback implementation that using global locks by default. The following features/cfgs change this behavior:\n - [`unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg](#optional-features-unsafe-assume-single-core): Use fallback implementations that disabling interrupts instead of using global locks.\n - If your target is single-core and calling interrupt disable instructions is safe, this is a safer and more efficient option.\n - [`unsafe-assume-privileged` feature / `portable_atomic_unsafe_assume_privileged` cfg](#optional-features-unsafe-assume-privileged): Use fallback implementations that using global locks with disabling interrupts.\n - If your target is multi-core and calling interrupt disable instructions is safe, this is a safer option.\n\n- **`float` feature**
\n Provide `AtomicF{32,64}`.\n\n If you want atomic types for unstable float types ([`f16` and `f128`](https://github.com/rust-lang/rust/issues/116909)), enable unstable cfg (`portable_atomic_unstable_f16` cfg for `AtomicF16`, `portable_atomic_unstable_f128` cfg for `AtomicF128`, [there is no possibility that both feature and cfg will be provided for unstable options.](https://github.com/taiki-e/portable-atomic/pull/200#issuecomment-2682252991)).\n\n> [!NOTE]\n> - Atomic float's `fetch_{add,sub,min,max}` are usually implemented using CAS loops, which can be slower than equivalent operations of atomic integers. As an exception, AArch64 with FEAT_LSFE and GPU targets have atomic float instructions and we use them on AArch64 when `lsfe` target feature is available at compile-time. We [plan to use atomic float instructions for GPU targets as well in the future.](https://github.com/taiki-e/portable-atomic/issues/34)\n> - Unstable cfgs are outside of the normal semver guarantees and minor or patch versions of portable-atomic may make breaking changes to them at any time.\n\n- **`std` feature**
\n Use `std`.\n\n- **`require-cas` feature**
\n Emit compile error if atomic CAS is not available. See [Usage](#usage) section for usage of this feature.\n\n- **`serde` feature**
\n Implement `serde::{Serialize,Deserialize}` for atomic types.\n\n Note:\n - The MSRV when this feature is enabled depends on the MSRV of [serde].\n\n- **`critical-section` feature**
\n Use [critical-section] to provide atomic CAS for targets where atomic CAS is not available in the standard library.\n\n `critical-section` support is useful to get atomic CAS when the [`unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)](#optional-features-unsafe-assume-single-core) can't be used,\n such as multi-core targets, unprivileged code running under some RTOS, or environments where disabling interrupts\n needs extra care due to e.g. real-time requirements.\n\n> [!NOTE]\n> - When enabling this feature, you should provide a suitable critical section implementation for the current target, see the [critical-section] documentation for details on how to do so.\n> - With this feature, critical sections are taken for all atomic operations, while with `unsafe-assume-single-core` feature [some operations](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/interrupt/README.md#no-disable-interrupts) don't require disabling interrupts. Therefore, for better performance, if all the `critical-section` implementation for your target does is disable interrupts, prefer using `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg) instead.\n> - It is usually **discouraged** to always enable this feature in libraries that depend on `portable-atomic`.\n>\n> Enabling this feature will prevent the end user from having the chance to take advantage of other (potentially) efficient implementations (implementations provided by `unsafe-assume-single-core` feature mentioned above, implementation proposed in [#60], etc.). Also, targets that are currently unsupported may be supported in the future.\n>\n> The recommended approach for libraries is to leave it up to the end user whether or not to enable this feature. (However, it may make sense to enable this feature by default for libraries specific to a platform where other implementations are known not to work.)\n>\n> See also [this comment](https://github.com/matklad/once_cell/issues/264#issuecomment-2352654806).\n>\n> As an example, the end-user's `Cargo.toml` that uses a crate that provides a critical-section implementation and a crate that depends on portable-atomic as an option would be expected to look like this:\n>\n> ```toml\n> [dependencies]\n> portable-atomic = { version = \"1\", default-features = false, features = [\"critical-section\"] }\n> crate-provides-critical-section-impl = \"...\"\n> crate-uses-portable-atomic-as-feature = { version = \"...\", features = [\"portable-atomic\"] }\n> ```\n>\n> - Enabling both this feature and `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg) will result in a compile error.\n> - Enabling both this feature and `unsafe-assume-privileged` feature (or `portable_atomic_unsafe_assume_privileged` cfg) will result in a compile error.\n> - The MSRV when this feature is enabled depends on the MSRV of [critical-section].\n\n- **`unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg**
\n Assume that the target is single-core and privileged instructions required to disable interrupts are available.\n\n - When this feature/cfg is enabled, this crate provides atomic CAS for targets where atomic CAS is not available in the standard library by disabling interrupts.\n - When both this feature/cfg and enabled-by-default `fallback` feature is enabled, this crate provides atomic types with larger than the width supported by native instructions by disabling interrupts.\n\n> [!WARNING]\n> This feature/cfg is `unsafe`, and note the following safety requirements:\n> - Enabling this feature/cfg for multi-core systems is always **unsound**.\n>\n> - This uses privileged instructions to disable interrupts, so it usually doesn't work on unprivileged mode.\n>\n> Enabling this feature/cfg in an environment where privileged instructions are not available, or if the instructions used are not sufficient to disable interrupts in the system, it is also usually considered **unsound**, although the details are system-dependent.\n>\n> The following are known cases:\n> - On Arm (except for M-Profile architectures), this disables only IRQs by default. For many systems (e.g., GBA) this is enough. If the system need to disable both IRQs and FIQs, you need to enable the `disable-fiq` feature (or `portable_atomic_disable_fiq` cfg) together.\n> - On RISC-V, this generates code for machine-mode (M-mode) by default. If you enable the `s-mode` feature (or `portable_atomic_s_mode` cfg) together, this generates code for supervisor-mode (S-mode). In particular, `qemu-system-riscv*` uses [OpenSBI](https://github.com/riscv-software-src/opensbi) as the default firmware.\n\nConsider using the [`unsafe-assume-privileged` feature (or `portable_atomic_unsafe_assume_privileged` cfg)](#optional-features-unsafe-assume-privileged) for multi-core systems with atomic CAS.\n\nConsider using the [`critical-section` feature](#optional-features-critical-section) for systems that cannot use this feature/cfg.\n\nSee also the [`interrupt` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/interrupt/README.md).\n\n> [!NOTE]\n> - It is **very strongly discouraged** to enable this feature/cfg in libraries that depend on `portable-atomic`.\n>\n> The recommended approach for libraries is to leave it up to the end user whether or not to enable this feature/cfg. (However, it may make sense to enable this feature/cfg by default for libraries specific to a platform where it is guaranteed to always be sound, for example in a hardware abstraction layer targeting a single-core chip.)\n> - Enabling this feature/cfg for unsupported architectures will result in a compile error.\n> - Arm, RISC-V, and Xtensa are currently supported. (Since all MSP430 and AVR are single-core, we always provide atomic CAS for them without this feature/cfg.)\n> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target is not supported yet.\n> - Enabling this feature/cfg for targets where privileged instructions are obviously unavailable (e.g., Linux) will result in a compile error.\n> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target supports privileged instructions but the build rejected.\n> - Enabling both this feature/cfg and `critical-section` feature will result in a compile error.\n> - When both this feature/cfg and `unsafe-assume-privileged` feature (or `portable_atomic_unsafe_assume_privileged` cfg) are enabled, this feature/cfg is preferred.\n\n- **`unsafe-assume-privileged` feature / `portable_atomic_unsafe_assume_privileged` cfg**
\n Similar to `unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg, but only assumes about availability of privileged instructions required to disable interrupts.\n\n - When both this feature/cfg and enabled-by-default `fallback` feature is enabled, this crate provides atomic types with larger than the width supported by native instructions by using global locks with disabling interrupts.\n\n> [!WARNING]\n> This feature/cfg is `unsafe`, and except for being sound in multi-core systems, this has the same safety requirements as [`unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg](#optional-features-unsafe-assume-single-core).\n\n> [!NOTE]\n> - It is **very strongly discouraged** to enable this feature/cfg in libraries that depend on `portable-atomic`.\n>\n> The recommended approach for libraries is to leave it up to the end user whether or not to enable this feature/cfg. (However, it may make sense to enable this feature/cfg by default for libraries specific to a platform where it is guaranteed to always be sound, for example in a hardware abstraction layer.)\n> - Enabling this feature/cfg for unsupported targets will result in a compile error.\n> - This requires atomic CAS (`cfg(target_has_atomic = \"ptr\")` or `cfg_no_atomic_cas!`).\n> - Arm, RISC-V, and Xtensa are currently supported.\n> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target is not supported yet.\n> - Enabling this feature/cfg for targets where privileged instructions are obviously unavailable (e.g., Linux) will result in a compile error.\n> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target supports privileged instructions but the build rejected.\n> - Enabling both this feature/cfg and `critical-section` feature will result in a compile error.\n> - When both this feature/cfg and `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg) are enabled, `unsafe-assume-single-core` is preferred.\n\n- **`portable_atomic_no_outline_atomics` cfg**
\n Disable dynamic dispatching by run-time CPU feature detection.\n\n Dynamic dispatching by run-time CPU feature detection allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE/FEAT_LSE2 (AArch64).\n\n See also the [`atomic128` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/atomic128/README.md).\n\n> [!NOTE]\n> - If the required target features are enabled at compile-time, dynamic dispatching is automatically disabled and the atomic operations are inlined.\n> - This is compatible with no-std (as with all features except `std`).\n> - On some targets, run-time detection is disabled by default mainly for compatibility with incomplete build environments or support for it is experimental, and can be enabled by `portable_atomic_outline_atomics` cfg. (When both cfg are enabled, `*_no_*` cfg is preferred.)\n> - Some AArch64 targets enable LLVM's `outline-atomics` target feature by default, so if you set this cfg, you may want to disable that as well. (However, portable-atomic's outline-atomics does not depend on the compiler-rt symbols, so even if you need to disable LLVM's outline-atomics, you may not need to disable portable-atomic's outline-atomics.)\n> - Dynamic detection is currently only supported in x86_64, AArch64, Arm, RISC-V, Arm64EC, and powerpc64. Enabling this cfg for unsupported architectures will be ignored.\n\n## Related Projects\n\n- [portable-atomic-util]: Synchronization primitives built with portable-atomic. This provides portable-atomic version of `std::sync::Arc` and `std::task::Wake`.\n- [atomic-maybe-uninit]: Atomic operations on potentially uninitialized integers.\n- [atomic-memcpy]: Byte-wise atomic memcpy.\n- [asmtest]: A library for tracking generated assemblies.\n\n[#60]: https://github.com/taiki-e/portable-atomic/issues/60\n[asmtest]: https://github.com/taiki-e/asmtest\n[atomic-maybe-uninit]: https://github.com/taiki-e/atomic-maybe-uninit\n[atomic-memcpy]: https://github.com/taiki-e/atomic-memcpy\n[critical-section]: https://github.com/rust-embedded/critical-section\n[portable-atomic-util]: https://github.com/taiki-e/portable-atomic-util\n[rust-lang/compiler-builtins#1234]: https://github.com/rust-lang/compiler-builtins/pull/1234\n[rust-lang/rust#100650]: https://github.com/rust-lang/rust/issues/100650\n[serde]: https://github.com/serde-rs/serde\n\n\n\n## License\n\nLicensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or\n[MIT license](LICENSE-MIT) at your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "build.rs", "sha256": "9d0b1e071715580497aadb918d4b3b124ee366ba66f15ac05a3d8045f0bc2cf0", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n// The rustc-cfg emitted by the build script are *not* public API.\n\n#![allow(clippy::match_same_arms)] // https://github.com/rust-lang/rust-clippy/issues/12044\n\n#[path = \"src/gen/build.rs\"]\nmod generated;\n#[path = \"version.rs\"]\nmod version;\n\nuse std::{env, str};\n\nuse self::version::{Version, rustc_version};\n\nfn main() {\n println!(\"cargo:rerun-if-changed=build.rs\");\n println!(\"cargo:rerun-if-changed=src/gen/build.rs\");\n println!(\"cargo:rerun-if-changed=version.rs\");\n\n #[cfg(feature = \"unsafe-assume-single-core\")]\n println!(\"cargo:rustc-cfg=portable_atomic_unsafe_assume_single_core\");\n #[cfg(feature = \"unsafe-assume-privileged\")]\n println!(\"cargo:rustc-cfg=portable_atomic_unsafe_assume_privileged\");\n #[cfg(feature = \"s-mode\")]\n println!(\"cargo:rustc-cfg=portable_atomic_s_mode\");\n #[cfg(feature = \"force-amo\")]\n println!(\"cargo:rustc-cfg=portable_atomic_force_amo\");\n #[cfg(feature = \"disable-fiq\")]\n println!(\"cargo:rustc-cfg=portable_atomic_disable_fiq\");\n\n let target = &*env::var(\"TARGET\").expect(\"TARGET not set\");\n let target_arch = &*env::var(\"CARGO_CFG_TARGET_ARCH\").expect(\"CARGO_CFG_TARGET_ARCH not set\");\n let target_os = &*env::var(\"CARGO_CFG_TARGET_OS\").expect(\"CARGO_CFG_TARGET_OS not set\");\n\n let version = match rustc_version() {\n Some(version) => version,\n None => {\n if env::var_os(\"PORTABLE_ATOMIC_DENY_WARNINGS\").is_some() {\n panic!(\"unable to determine rustc version\")\n }\n println!(\n \"cargo:warning={}: unable to determine rustc version; assuming latest stable rustc (1.{})\",\n env!(\"CARGO_PKG_NAME\"),\n Version::LATEST.minor\n );\n Version::LATEST\n }\n };\n\n if version.minor >= 80 {\n println!(\n r#\"cargo:rustc-check-cfg=cfg(target_feature,values(\"lsfe\",\"fast-serialization\",\"load-store-on-cond\",\"distinct-ops\",\"rmw\"))\"#\n );\n\n // Custom cfgs set by build script. Not public API.\n // grep -F 'cargo:rustc-cfg=' build.rs | grep -Ev '^ *//' | sed -E 's/^.*cargo:rustc-cfg=//; s/(=\\\\)?\".*$//' | LC_ALL=C sort -u | tr '\\n' ',' | sed -E 's/,$/\\n/'\n println!(\n \"cargo:rustc-check-cfg=cfg(portable_atomic_atomic_intrinsics,portable_atomic_disable_fiq,portable_atomic_force_amo,portable_atomic_ll_sc_rmw,portable_atomic_no_asm,portable_atomic_no_asm_maybe_uninit,portable_atomic_no_asm_syscall,portable_atomic_no_atomic_64,portable_atomic_no_atomic_cas,portable_atomic_no_atomic_load_store,portable_atomic_no_atomic_min_max,portable_atomic_no_cfg_target_has_atomic,portable_atomic_no_cmpxchg16b_intrinsic,portable_atomic_no_cmpxchg16b_target_feature,portable_atomic_no_const_mut_refs,portable_atomic_no_const_raw_ptr_deref,portable_atomic_no_const_transmute,portable_atomic_no_core_unwind_safe,portable_atomic_no_diagnostic_namespace,portable_atomic_no_strict_provenance,portable_atomic_no_strict_provenance_atomic_ptr,portable_atomic_no_stronger_failure_ordering,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_pre_llvm_16,portable_atomic_pre_llvm_18,portable_atomic_pre_llvm_20,portable_atomic_s_mode,portable_atomic_sanitize_thread,portable_atomic_target_feature,portable_atomic_unsafe_assume_privileged,portable_atomic_unsafe_assume_single_core,portable_atomic_unstable_asm,portable_atomic_unstable_asm_experimental_arch,portable_atomic_unstable_cfg_target_has_atomic,portable_atomic_unstable_isa_attribute)\"\n );\n // TODO: handle multi-line target_feature_fallback\n // grep -F 'target_feature_fallback(\"' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\\(//; s/\",.*$/\"/' | LC_ALL=C sort -u | tr '\\n' ',' | sed -E 's/,$/\\n/'\n println!(\n r#\"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values(\"cmpxchg16b\",\"distinct-ops\",\"fast-serialization\",\"load-store-on-cond\",\"lse\",\"lse128\",\"lse2\",\"lsfe\",\"mclass\",\"miscellaneous-extensions-3\",\"quadword-atomics\",\"rcpc3\",\"rmw\",\"v6\",\"v7\",\"zaamo\",\"zabha\",\"zacas\"))\"#\n );\n }\n\n // https://github.com/rust-lang/rust/pull/123745 (includes https://github.com/rust-lang/cargo/pull/13560) merged in Rust 1.79 (nightly-2024-04-11).\n if !version.probe(79, 2024, 4, 10) {\n // HACK: If --target is specified, rustflags is not applied to the build\n // script itself, so the build script will not be recompiled when rustflags\n // is changed. That in itself is not a problem, but the old Cargo does\n // not rerun the build script as well, which can be problematic.\n // https://github.com/rust-lang/cargo/issues/13003\n // This problem has been fixed in 1.79 so only older versions need a workaround.\n println!(\"cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS\");\n println!(\"cargo:rerun-if-env-changed=RUSTFLAGS\");\n println!(\"cargo:rerun-if-env-changed=CARGO_BUILD_RUSTFLAGS\");\n let mut target_upper = target.replace(|c: char| c == '-' || c == '.', \"_\");\n target_upper.make_ascii_uppercase();\n println!(\"cargo:rerun-if-env-changed=CARGO_TARGET_{}_RUSTFLAGS\", target_upper);\n }\n\n // Note that cfgs are `no_`*, not `has_*`. This allows treating as the latest\n // stable rustc is used when the build script doesn't run. This is useful\n // for non-cargo build systems that don't run the build script.\n\n // atomic_min_max stabilized in Rust 1.45 (nightly-2020-05-30): https://github.com/rust-lang/rust/pull/72324\n if !version.probe(45, 2020, 5, 29) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_atomic_min_max\");\n }\n // track_caller stabilized in Rust 1.46 (nightly-2020-07-02): https://github.com/rust-lang/rust/pull/72445\n if !version.probe(46, 2020, 7, 1) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_track_caller\");\n }\n // unsafe_op_in_unsafe_fn stabilized in Rust 1.52 (nightly-2021-03-11): https://github.com/rust-lang/rust/pull/79208\n if !version.probe(52, 2021, 3, 10) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_unsafe_op_in_unsafe_fn\");\n }\n // const_transmute stabilized in Rust 1.56 (nightly-2021-07-29): https://github.com/rust-lang/rust/pull/85769\n if !version.probe(56, 2021, 7, 28) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_const_transmute\");\n }\n // https://github.com/rust-lang/rust/pull/84662 merged in Rust 1.56 (nightly-2021-08-02).\n if !version.probe(56, 2021, 8, 1) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_core_unwind_safe\");\n }\n // const_raw_ptr_deref stabilized in Rust 1.58 (nightly-2021-11-15): https://github.com/rust-lang/rust/pull/89551\n if !version.probe(58, 2021, 11, 14) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_const_raw_ptr_deref\");\n }\n // https://github.com/rust-lang/rust/pull/98383 merged in Rust 1.64 (nightly-2022-07-19).\n if !version.probe(64, 2022, 7, 18) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_stronger_failure_ordering\");\n }\n // https://github.com/rust-lang/rust/pull/114790 merged in nightly-2023-08-24\n if !version.probe(74, 2023, 8, 23) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_asm_maybe_uninit\");\n }\n // #[diagnostic] stabilized in Rust 1.78 (nightly-2024-03-09): https://github.com/rust-lang/rust/pull/119888\n if !version.probe(78, 2024, 3, 8) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_diagnostic_namespace\");\n }\n // const_mut_refs/const_refs_to_cell stabilized in Rust 1.83 (nightly-2024-09-16): https://github.com/rust-lang/rust/pull/129195\n if !version.probe(83, 2024, 9, 15) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_const_mut_refs\");\n }\n // strict_provenance/exposed_provenance APIs stabilized in Rust 1.84 (nightly-2024-10-22): https://github.com/rust-lang/rust/pull/130350\n if !version.probe(84, 2024, 10, 21) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_strict_provenance\");\n }\n // strict_provenance_atomic_ptr stabilized in Rust 1.91 (nightly-2024-10-22): https://github.com/rust-lang/rust/pull/145467\n if !version.probe(91, 2025, 8, 30) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_strict_provenance_atomic_ptr\");\n }\n\n // For Miri and ThreadSanitizer. (aarch64, arm64ec, s390x, powerpc64)\n // https://github.com/rust-lang/rust/pull/97423 merged in Rust 1.64 (nightly-2022-06-30).\n // https://github.com/rust-lang/rust/pull/141507 merged in Rust 1.89 (nightly-2025-05-31).\n if version.nightly\n && version.probe(64, 2022, 6, 29)\n && !version.probe(89, 2025, 5, 30)\n && (target_arch != \"powerpc64\" || version.llvm >= 15)\n {\n println!(\"cargo:rustc-cfg=portable_atomic_atomic_intrinsics\");\n }\n\n // asm! on AArch64, Arm, RISC-V, x86, and x86_64 stabilized in Rust 1.59 (nightly-2021-12-16): https://github.com/rust-lang/rust/pull/91728\n let no_asm = !version.probe(59, 2021, 12, 15);\n if no_asm {\n if version.nightly\n && version.probe(46, 2020, 6, 20)\n && ((target_arch != \"x86\" && target_arch != \"x86_64\") || version.llvm >= 10)\n && is_allowed_feature(\"asm\")\n {\n // This feature was added in Rust 1.45 (nightly-2020-05-20), but\n // concat! in asm! requires Rust 1.46 (nightly-2020-06-21).\n // x86 intel syntax requires LLVM 10 (since Rust 1.53, the minimum\n // external LLVM version is 10+: https://github.com/rust-lang/rust/pull/83387).\n // The part of this feature we use has not been changed since nightly-2020-06-21\n // until it was stabilized, so it can safely be enabled in nightly for that period.\n println!(\"cargo:rustc-cfg=portable_atomic_unstable_asm\");\n if (target_arch == \"riscv32\" || target_arch == \"riscv64\")\n && !version.probe(56, 2021, 8, 15)\n {\n // Clobbering vector status needed for asm syscall require Rust 1.56 (nightly-2021-08-16): https://github.com/rust-lang/rust/pull/87581\n println!(\"cargo:rustc-cfg=portable_atomic_no_asm_syscall\");\n }\n }\n println!(\"cargo:rustc-cfg=portable_atomic_no_asm\");\n } else {\n match target_arch {\n \"arm64ec\" | \"s390x\" => {\n // asm! on Arm64EC and s390x stabilized in Rust 1.84 (nightly-2024-11-11): https://github.com/rust-lang/rust/pull/131781, https://github.com/rust-lang/rust/pull/131258\n if !version.probe(84, 2024, 11, 10) {\n if version.nightly\n && (target_arch != \"s390x\" || version.probe(71, 2023, 5, 8))\n && is_allowed_feature(\"asm_experimental_arch\")\n {\n // https://github.com/rust-lang/rust/pull/111331 merged in Rust 1.71 (nightly-2023-05-09).\n // The part of this feature we use has not been changed since nightly-2023-05-09\n // until it was stabilized, so it can safely be enabled in nightly for that period.\n println!(\"cargo:rustc-cfg=portable_atomic_unstable_asm_experimental_arch\");\n } else {\n println!(\"cargo:rustc-cfg=portable_atomic_no_asm\");\n }\n }\n }\n \"powerpc64\" => {\n // asm! on PowerPC stabilized in Rust 1.95 (nightly-2026-01-28): https://github.com/rust-lang/rust/pull/147996\n if !version.probe(95, 2026, 1, 27) {\n if version.nightly\n && version.probe(60, 2022, 2, 12)\n && is_allowed_feature(\"asm_experimental_arch\")\n {\n // https://github.com/rust-lang/rust/pull/93868 merged in Rust 1.60 (nightly-2022-02-13).\n // The part of this feature we use has not been changed since nightly-2022-02-13\n // until it was stabilized, so it can safely be enabled in nightly for that period.\n println!(\"cargo:rustc-cfg=portable_atomic_unstable_asm_experimental_arch\");\n } else {\n println!(\"cargo:rustc-cfg=portable_atomic_no_asm\");\n }\n if !version.probe(92, 2025, 9, 22)\n || env::var(\"CARGO_CFG_TARGET_ABI\")\n .unwrap_or_default()\n .split(',')\n .any(|abi| abi == \"spu\")\n {\n // Clobbering ctr register needed for asm syscall require Rust 1.92 (nightly-2025-09-23): https://github.com/rust-lang/rust/pull/146831\n // Cell/B.E. Linux's SPU syscalls support a subset of powerpc64 syscalls with the exact semantics: https://github.com/torvalds/linux/blob/v7.1/arch/powerpc/platforms/cell/spu_callbacks.c#L17\n // but have a different syscall sequence: https://github.com/mirror/newlib-cygwin/blob/newlib-4.4.0/libgloss/spu/linux_syscalls.c#L37\n // This is currently used only for testing.\n println!(\"cargo:rustc-cfg=portable_atomic_no_asm_syscall\");\n }\n }\n }\n _ => {}\n }\n }\n\n // feature(cfg_target_has_atomic) stabilized in Rust 1.60 (nightly-2022-02-11): https://github.com/rust-lang/rust/pull/93824\n if !version.probe(60, 2022, 2, 10) {\n if version.nightly\n && version.probe(40, 2019, 10, 13)\n && is_allowed_feature(\"cfg_target_has_atomic\")\n {\n // The part of this feature we use has not been changed since nightly-2019-10-14\n // until it was stabilized, so it can safely be enabled in nightly for that period.\n println!(\"cargo:rustc-cfg=portable_atomic_unstable_cfg_target_has_atomic\");\n } else {\n println!(\"cargo:rustc-cfg=portable_atomic_no_cfg_target_has_atomic\");\n let target = &*convert_custom_linux_target(target);\n if generated::NO_ATOMIC_CAS.contains(&target) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_atomic_cas\");\n }\n if generated::NO_ATOMIC_64.contains(&target) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_atomic_64\");\n } else {\n // Otherwise, assuming `\"max-atomic-width\" == 64` or `\"max-atomic-width\" == 128`.\n }\n }\n }\n // We don't need to use convert_custom_linux_target here because all linux targets have atomics.\n if generated::NO_ATOMIC.contains(&target) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_atomic_load_store\");\n }\n\n if version.llvm < 20 {\n println!(\"cargo:rustc-cfg=portable_atomic_pre_llvm_20\");\n if version.llvm < 18 {\n println!(\"cargo:rustc-cfg=portable_atomic_pre_llvm_18\");\n if version.llvm < 16 {\n println!(\"cargo:rustc-cfg=portable_atomic_pre_llvm_16\");\n }\n }\n }\n\n if version.nightly {\n // `cfg(sanitize = \"..\")` is not stabilized.\n let sanitize = env::var(\"CARGO_CFG_SANITIZE\").unwrap_or_default();\n if sanitize.contains(\"thread\") {\n // Most kinds of sanitizers are not compatible with asm\n // (https://github.com/google/sanitizers/issues/192),\n // but it seems that ThreadSanitizer is the only one that can cause\n // false positives in our code.\n println!(\"cargo:rustc-cfg=portable_atomic_sanitize_thread\");\n }\n }\n\n match target_arch {\n \"x86_64\" => {\n // cmpxchg16b_target_feature stabilized in Rust 1.69 (nightly-2023-03-01): https://github.com/rust-lang/rust/pull/106774\n if !version.probe(69, 2023, 2, 28) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_cmpxchg16b_target_feature\");\n }\n // For Miri and ThreadSanitizer.\n // https://github.com/rust-lang/rust/pull/109359 (includes https://github.com/rust-lang/stdarch/pull/1358) merged in Rust 1.70 (nightly-2023-03-24).\n if version.nightly && !version.probe(70, 2023, 3, 23) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_cmpxchg16b_intrinsic\");\n }\n\n // cmpxchg16b_target_feature stabilized in Rust 1.69.\n if needs_target_feature_fallback(&version, Some(69)) {\n // x86_64 Apple targets always support CMPXCHG16B:\n // https://github.com/rust-lang/rust/blob/1.68.0/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs#L8\n // https://github.com/rust-lang/rust/blob/1.68.0/compiler/rustc_target/src/spec/apple_base.rs#L69-L70\n // (Windows (except Windows 7, since Rust 1.78) and Fuchsia (since Rust 1.87) targets\n // also enable CMPXCHG16B, but this branch is only used on pre-1.69 that\n // cmpxchg16b_target_feature is unstable.)\n // Script to get builtin targets that support CMPXCHG16B by default:\n // $ (for target in $(rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == \"x86_64\" then .key else empty end'); do rustc --print cfg --target \"${target}\" | grep -Fq '\"cmpxchg16b\"' && printf '%s\\n' \"${target}\"; done)\n let is_apple = env::var(\"CARGO_CFG_TARGET_VENDOR\").unwrap_or_default() == \"apple\";\n let cmpxchg16b = is_apple;\n // LLVM recognizes this also as cx16 target feature: https://godbolt.org/z/KM3jz616j\n // However, it is unlikely that rustc will support that name, so we ignore it.\n target_feature_fallback(\"cmpxchg16b\", cmpxchg16b);\n }\n }\n \"aarch64\" | \"arm64ec\" => {\n // target_feature \"lse2\"/\"lse128\"/\"rcpc3\" is unstable and available on rustc side since nightly-2024-08-30: https://github.com/rust-lang/rust/pull/128192\n if !version.probe(82, 2024, 8, 29) || needs_target_feature_fallback(&version, None) {\n // FEAT_LSE2 doesn't imply FEAT_LSE. FEAT_LSE128 implies FEAT_LSE but not FEAT_LSE2.\n // AArch64 macOS always supports FEAT_LSE and FEAT_LSE2 because M1 is Armv8.4 with all features of Armv8.5 except FEAT_BTI:\n // https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/AArch64/AArch64Processors.td#L1558\n // https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/AArch64/AArch64Processors.td#L1180\n // Script to get builtin targets that support FEAT_LSE/FEAT_LSE2 by default:\n // $ (for target in $(rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == \"aarch64\" or .value.arch == \"arm64ec\" then .key else empty end'); do rustc --print cfg --target \"${target}\" | grep -Fq '\"lse\"' && printf '%s\\n' \"${target}\"; done)\n // $ (for target in $(rustc -Z unstable-options --print all-target-specs-json | jq -r '. | to_entries[] | if .value.arch == \"aarch64\" or .value.arch == \"arm64ec\" then .key else empty end'); do rustc --print cfg --target \"${target}\" | grep -Fq '\"lse2\"' && printf '%s\\n' \"${target}\"; done)\n let is_macos = target_os == \"macos\";\n let mut lse = is_macos;\n target_feature_fallback(\"lse2\", is_macos);\n lse |= target_feature_fallback(\"lse128\", false);\n target_feature_fallback(\"rcpc3\", false);\n // aarch64_target_feature stabilized in Rust 1.61.\n if needs_target_feature_fallback(&version, Some(61)) {\n target_feature_fallback(\"lse\", lse);\n }\n }\n // As of Rust 1.85, target_feature \"lsfe\" is not available on rustc side:\n // https://github.com/rust-lang/rust/blob/1.85.0/compiler/rustc_target/src/target_features.rs\n target_feature_fallback(\"lsfe\", false);\n\n // As of Apple M1/M1 Pro, on Apple hardware, CAS-loop-based RMW is much slower than\n // LL/SC-loop-based RMW: https://github.com/taiki-e/portable-atomic/pull/89\n let is_apple = env::var(\"CARGO_CFG_TARGET_VENDOR\").unwrap_or_default() == \"apple\";\n if is_apple || target_cpu().map_or(false, |cpu| cpu.starts_with(\"apple-\")) {\n println!(\"cargo:rustc-cfg=portable_atomic_ll_sc_rmw\");\n }\n }\n \"arm\" => {\n // For non-Linux/Android pre-v6 Arm (tier 3) with unsafe_assume_single_core enabled.\n // feature(isa_attribute) stabilized in Rust 1.67 (nightly-2022-11-06): https://github.com/rust-lang/rust/pull/102458\n if version.nightly && !version.probe(67, 2022, 11, 5) {\n println!(\"cargo:rustc-cfg=portable_atomic_unstable_isa_attribute\");\n }\n\n if needs_target_feature_fallback(&version, None) {\n // #[cfg(target_feature = \"v7\")] and others don't work on stable.\n // armv7-unknown-linux-gnueabihf\n // ^^\n let mut mclass = false;\n let mut v6 = false;\n let mut v7 = false;\n if let Some(mut subarch) =\n strip_prefix(target, \"arm\").or_else(|| strip_prefix(target, \"thumb\"))\n {\n subarch = strip_prefix(subarch, \"eb\").unwrap_or(subarch); // ignore endianness\n subarch = subarch.split('-').next().unwrap(); // ignore vender/os/env\n let mut i = subarch.splitn(2, '.');\n subarch = i.next().unwrap();\n let suffix = i.next().unwrap_or_default(); // .base/.main suffix\n let mut known = true;\n // See https://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/build.rs for details\n match subarch {\n \"v7\" | \"v7a\" | \"v7neon\" | \"v7s\" | \"v7k\" | \"v8\" | \"v8a\" | \"v9\" | \"v9a\" => {} // aclass\n \"v7r\" | \"v8r\" | \"v9r\" => {} // rclass\n \"v6m\" | \"v7em\" | \"v7m\" | \"v8m\" => mclass = true,\n // arm-linux-androideabi is v5te\n // https://github.com/rust-lang/rust/blob/1.84.0/compiler/rustc_target/src/spec/targets/arm_linux_androideabi.rs#L18\n _ if target == \"arm-linux-androideabi\" => subarch = \"v5te\",\n // armeb-unknown-linux-gnueabi is v8 & aclass\n // https://github.com/rust-lang/rust/blob/1.84.0/compiler/rustc_target/src/spec/targets/armeb_unknown_linux_gnueabi.rs#L18\n _ if target == \"armeb-unknown-linux-gnueabi\" => subarch = \"v8\",\n // Legacy Arm architectures (pre-v7 except v6m) don't have *class target feature.\n \"\" => subarch = \"v6\",\n \"v4t\" | \"v5te\" | \"v6\" | \"v6k\" => {}\n _ => {\n known = false;\n if env::var_os(\"PORTABLE_ATOMIC_DENY_WARNINGS\").is_some() {\n panic!(\"unrecognized Arm subarch: {}\", target)\n }\n println!(\n \"cargo:warning={}: unrecognized Arm subarch: {}\",\n env!(\"CARGO_PKG_NAME\"),\n target\n );\n }\n }\n v6 = known && subarch.starts_with(\"v6\");\n v7 = known && subarch.starts_with(\"v7\");\n if known && (subarch.starts_with(\"v8\") || subarch.starts_with(\"v9\")) {\n // Armv8-M is not considered as v8 by LLVM.\n // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/arm_shared/mod.rs\n if mclass {\n // Armv8-M Mainline is a superset of Armv7-M.\n // Armv8-M Baseline is a superset of Armv6-M.\n // That said, LLVM handles thumbv8m.main without v8m like v6m, not v7m: https://godbolt.org/z/Ph96v9zae\n // TODO: Armv9-M has not yet been released,\n // so it is not clear how it will be handled here.\n v6 = true;\n v7 = suffix == \"main\";\n } else {\n v7 = true;\n }\n }\n }\n v6 |= target_feature_fallback(\"v7\", v7);\n target_feature_fallback(\"v6\", v6);\n target_feature_fallback(\"mclass\", mclass);\n }\n }\n \"riscv32\" | \"riscv64\" => {\n // zabha and zacas imply zaamo in GCC, LLVM 20+, and Rust, but do not in LLVM 19.\n // However, enabling them without zaamo or a is not allowed in LLVM 19, so we can assume\n // zaamo is available when zabha is enabled).\n // https://github.com/llvm/llvm-project/commit/956361ca080a689a96b6552d28681aaf0ad2f494\n // https://github.com/gcc-mirror/gcc/commit/7b2b2e3d660edc8ef3a8cfbdfc2b0fd499459601\n // https://github.com/gcc-mirror/gcc/commit/11c2453a16b725b7fb67778e1ab4636a51a1217d\n // https://github.com/rust-lang/rust/pull/130877\n let mut zaamo = false;\n // target_feature \"zacas\" is available as unstable on rustc side\n // since nightly-2025-02-26 (https://github.com/rust-lang/rust/pull/137417),\n // and stabilized in Rust 1.94 (https://github.com/rust-lang/rust/pull/145948).\n if !version.probe(87, 2025, 2, 25) || needs_target_feature_fallback(&version, Some(94))\n {\n // amocas.{w,d,q} (and amocas.{b,h} if zabha is also available)\n // available as experimental since LLVM 17 https://github.com/llvm/llvm-project/commit/29f630a1ddcbb03caa31b5002f0cbc105ff3a869\n // available non-experimental since LLVM 20 https://github.com/llvm/llvm-project/commit/614aeda93b2225c6eb42b00ba189ba7ca2585c60\n zaamo |= target_feature_fallback(\"zacas\", false);\n }\n // target_feature \"zaamo\"/\"zabha\" is available as unstable on rustc side\n // since nightly-2024-10-02 (https://github.com/rust-lang/rust/pull/130877),\n // and stabilized in Rust 1.94 (https://github.com/rust-lang/rust/pull/145948).\n if !version.probe(83, 2024, 10, 1) || needs_target_feature_fallback(&version, Some(94))\n {\n if version.llvm >= 19 {\n // amo*.{b,h}\n // available since LLVM 19 https://github.com/llvm/llvm-project/commit/89f87c387627150d342722b79c78cea2311cddf7 / https://github.com/llvm/llvm-project/commit/6b7444964a8d028989beee554a1f5c61d16a1cac\n zaamo |= target_feature_fallback(\"zabha\", false);\n }\n // amo*.{w,d}\n target_feature_fallback(\"zaamo\", zaamo);\n }\n }\n \"powerpc64\" => {\n // target_feature \"quadword-atomics\" is unstable and available on rustc side since nightly-2024-09-28: https://github.com/rust-lang/rust/pull/130873\n if !version.probe(83, 2024, 9, 27) || needs_target_feature_fallback(&version, None) {\n let mut pwr8_features = false;\n if let Some(cpu) = target_cpu() {\n if let Some(mut cpu_version) = strip_prefix(&cpu, \"pwr\") {\n cpu_version = strip_suffix(cpu_version, \"x\").unwrap_or(cpu_version); // for pwr5x and pwr6x\n if let Ok(cpu_version) = cpu_version.parse::() {\n pwr8_features = cpu_version >= 8;\n }\n } else {\n // https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/PowerPC/PPC.td#L789\n // https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/PowerPC/PPC.td#L557\n // On the minimum external LLVM version of the oldest rustc version which we can use asm_experimental_arch\n // on this target (see CI config for more), \"future\" is based on pwr10 features.\n // https://github.com/llvm/llvm-project/blob/llvmorg-12.0.0/llvm/lib/Target/PowerPC/PPC.td#L370\n pwr8_features = cpu == \"future\" || cpu == \"ppc64le\";\n }\n } else {\n // powerpc64le is pwr8 by default https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/PowerPC/PPC.td#L789\n // See also https://github.com/rust-lang/rust/issues/59932\n pwr8_features = env::var(\"CARGO_CFG_TARGET_ENDIAN\")\n .expect(\"CARGO_CFG_TARGET_ENDIAN not set\")\n == \"little\";\n }\n // power8 features: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/PowerPC/PPC.td#L484\n // lqarx and stqcx.\n target_feature_fallback(\"quadword-atomics\", pwr8_features);\n }\n }\n \"s390x\" => {\n let mut arch9_features = false; // z196+\n let mut arch13_features = false; // z15+\n if let Some(cpu) = target_cpu() {\n // LLVM and GCC recognize the same names:\n // https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/SystemZ/SystemZProcessors.td\n // https://github.com/gcc-mirror/gcc/blob/releases/gcc-15.2.0/gcc/config/s390/s390.opt#L58-L128\n if let Some(arch_version) = strip_prefix(&cpu, \"arch\") {\n if let Ok(arch_version) = arch_version.parse::() {\n arch9_features = arch_version >= 9;\n arch13_features = arch_version >= 13;\n }\n } else {\n match &*cpu {\n \"z196\" | \"zEC12\" | \"z13\" | \"z14\" => arch9_features = true,\n \"z15\" | \"z16\" | \"z17\" => {\n arch9_features = true;\n arch13_features = true;\n }\n _ => {}\n }\n }\n }\n // target_feature \"miscellaneous-extensions-3\" is available as unstable on rustc side\n // since nightly-2025-06-05 (https://github.com/rust-lang/rust/pull/141250),\n // and stabilized in Rust 1.93 (https://github.com/rust-lang/rust/pull/145656).\n if !version.probe(89, 2025, 6, 4) || needs_target_feature_fallback(&version, Some(93)) {\n // arch13 features: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/SystemZ/SystemZFeatures.td#L303\n // nand (nnr{,g}k), select (sel{,g}r), etc.\n target_feature_fallback(\"miscellaneous-extensions-3\", arch13_features);\n }\n // As of Rust 1.84, target_feature \"fast-serialization\"/\"load-store-on-cond\"/\"distinct-ops\"/\"miscellaneous-extensions-3\" is not available on rustc side:\n // https://github.com/rust-lang/rust/blob/1.84.0/compiler/rustc_target/src/target_features.rs#L547\n // arch9 features: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/SystemZ/SystemZFeatures.td#L103\n // bcr 14,0\n target_feature_fallback(\"fast-serialization\", arch9_features);\n // {l,st}oc{,g}{,r}\n target_feature_fallback(\"load-store-on-cond\", arch9_features);\n // {al,sl,n,o,x}{,g}rk\n target_feature_fallback(\"distinct-ops\", arch9_features);\n }\n \"avr\" => {\n // target_feature \"rmw\" is unstable and available on rustc side since nightly-2026-02-08: https://github.com/rust-lang/rust/pull/146900\n if !version.probe(95, 2026, 2, 7) || needs_target_feature_fallback(&version, None) {\n // https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/AVR/AVRDevices.td\n let mut xmegau = false; // FamilyXMEGAU\n if let Some(cpu) = target_cpu() {\n match &*cpu {\n \"atxmega16a4u\" | \"atxmega16c4\" | \"atxmega32a4u\" | \"atxmega32c3\"\n | \"atxmega32c4\" | \"atxmega32e5\" | \"atxmega16e5\" | \"atxmega8e5\"\n | \"atxmega64a3u\" | \"atxmega64a4u\" | \"atxmega64b1\" | \"atxmega64b3\"\n | \"atxmega64c3\" | \"atxmega64a1u\" | \"atxmega128a3u\" | \"atxmega128b1\"\n | \"atxmega128b3\" | \"atxmega128c3\" | \"atxmega192a3u\" | \"atxmega192c3\"\n | \"atxmega256a3u\" | \"atxmega256a3bu\" | \"atxmega256c3\" | \"atxmega384c3\"\n | \"atxmega128a1u\" | \"atxmega128a4u\" => xmegau = true,\n _ => {}\n }\n }\n target_feature_fallback(\"rmw\", xmegau);\n }\n }\n _ => {}\n }\n}\n\n// HACK: Currently, it seems that the only way to handle unstable target\n// features on the stable is to parse the `-C target-feature` in RUSTFLAGS.\n//\n// - #[cfg(target_feature = \"unstable_target_feature\")] doesn't work on stable.\n// - CARGO_CFG_TARGET_FEATURE excludes unstable target features on stable.\n//\n// As mentioned in the [RFC2045], unstable target features are also passed to LLVM\n// (e.g., https://godbolt.org/z/4rr7rMcfG), so this hack works properly on stable.\n//\n// [RFC2045]: https://rust-lang.github.io/rfcs/2045-target-feature.html#backend-compilation-options\nfn needs_target_feature_fallback(version: &Version, stable: Option) -> bool {\n match stable {\n // In these cases, cfg(target_feature = \"...\") would work, so skip emitting our own fallback target_feature cfg.\n _ if version.nightly => false,\n Some(stabilized) if version.minor >= stabilized => false,\n _ => true,\n }\n}\nfn target_feature_fallback(name: &str, mut has_target_feature: bool) -> bool {\n if let Some(rustflags) = env::var_os(\"CARGO_ENCODED_RUSTFLAGS\") {\n for mut flag in rustflags.to_string_lossy().split('\\x1f') {\n flag = strip_prefix(flag, \"-C\").unwrap_or(flag);\n if let Some(flag) = strip_prefix(flag, \"target-feature=\") {\n for s in flag.split(',') {\n // TODO: Handles cases where a specific target feature\n // implicitly enables another target feature.\n match (s.as_bytes().first(), s.as_bytes().get(1..)) {\n (Some(b'+'), Some(f)) if f == name.as_bytes() => has_target_feature = true,\n (Some(b'-'), Some(f)) if f == name.as_bytes() => has_target_feature = false,\n _ => {}\n }\n }\n }\n }\n }\n if has_target_feature {\n println!(\"cargo:rustc-cfg=portable_atomic_target_feature=\\\"{}\\\"\", name);\n }\n has_target_feature\n}\n\nfn target_cpu() -> Option {\n let rustflags = env::var_os(\"CARGO_ENCODED_RUSTFLAGS\")?;\n let rustflags = rustflags.to_string_lossy();\n let mut cpu = None;\n for mut flag in rustflags.split('\\x1f') {\n flag = strip_prefix(flag, \"-C\").unwrap_or(flag);\n if let Some(flag) = strip_prefix(flag, \"target-cpu=\") {\n cpu = Some(flag);\n }\n }\n cpu.map(str::to_owned)\n}\n\nfn is_allowed_feature(name: &str) -> bool {\n // https://github.com/dtolnay/thiserror/pull/248\n if env::var_os(\"RUSTC_STAGE\").is_some() {\n return false;\n }\n\n // allowed by default\n let mut allowed = true;\n if let Some(rustflags) = env::var_os(\"CARGO_ENCODED_RUSTFLAGS\") {\n for mut flag in rustflags.to_string_lossy().split('\\x1f') {\n flag = strip_prefix(flag, \"-Z\").unwrap_or(flag);\n if let Some(flag) = strip_prefix(flag, \"allow-features=\") {\n // If it is specified multiple times, the last value will be preferred.\n allowed = flag.split(',').any(|allowed| allowed == name);\n }\n }\n }\n allowed\n}\n\n// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/build-common.rs.\n//\n// The target triplets have the form of 'arch-vendor-system'.\n//\n// When building for Linux (e.g. the 'system' part is\n// 'linux-something'), replace the vendor with 'unknown'\n// so that mapping to rust standard targets happens correctly.\nfn convert_custom_linux_target(target: &str) -> String {\n let mut parts: Vec<&str> = target.split('-').collect();\n let system = parts.get(2);\n if system == Some(&\"linux\") {\n parts[1] = \"unknown\";\n }\n parts.join(\"-\")\n}\n\n// str::strip_prefix requires Rust 1.45\n#[must_use]\nfn strip_prefix<'a>(s: &'a str, pat: &str) -> Option<&'a str> {\n if s.starts_with(pat) { Some(&s[pat.len()..]) } else { None }\n}\n// str::strip_suffix requires Rust 1.45\n#[must_use]\nfn strip_suffix<'a>(s: &'a str, pat: &str) -> Option<&'a str> {\n if s.ends_with(pat) { Some(&s[..s.len() - pat.len()]) } else { None }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/cfgs.rs", "sha256": "7a438600e742bca248c16c980c20c2b65be01bb786e6cde738cbd94f612344e0", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![allow(missing_docs)]\n\n#[cfg(not(all(\n portable_atomic_no_atomic_load_store,\n not(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )),\n)))]\n#[macro_use]\nmod atomic_8_16_macros {\n #[macro_export]\n macro_rules! cfg_has_atomic_8 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_8 {\n ($($tt:tt)*) => {};\n }\n #[macro_export]\n macro_rules! cfg_has_atomic_16 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_16 {\n ($($tt:tt)*) => {};\n }\n}\n#[cfg(all(\n portable_atomic_no_atomic_load_store,\n not(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )),\n))]\n#[macro_use]\nmod atomic_8_16_macros {\n #[macro_export]\n macro_rules! cfg_has_atomic_8 {\n ($($tt:tt)*) => {};\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_8 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n #[macro_export]\n macro_rules! cfg_has_atomic_16 {\n ($($tt:tt)*) => {};\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_16 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n}\n\n#[cfg(all(\n any(not(target_pointer_width = \"16\"), feature = \"fallback\"),\n not(all(\n portable_atomic_no_atomic_load_store,\n not(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )),\n )),\n))]\n#[macro_use]\nmod atomic_32_macros {\n #[macro_export]\n macro_rules! cfg_has_atomic_32 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_32 {\n ($($tt:tt)*) => {};\n }\n}\n#[cfg(not(all(\n any(not(target_pointer_width = \"16\"), feature = \"fallback\"),\n not(all(\n portable_atomic_no_atomic_load_store,\n not(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )),\n )),\n)))]\n#[macro_use]\nmod atomic_32_macros {\n #[macro_export]\n macro_rules! cfg_has_atomic_32 {\n ($($tt:tt)*) => {};\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_32 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n}\n\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(\n all(\n feature = \"fallback\",\n any(\n not(portable_atomic_no_atomic_cas),\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ),\n ),\n not(portable_atomic_no_atomic_64),\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")),\n all(\n target_arch = \"riscv32\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"),\n ),\n ))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n all(\n feature = \"fallback\",\n any(\n target_has_atomic = \"ptr\",\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ),\n ),\n target_has_atomic = \"64\",\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")),\n all(\n target_arch = \"riscv32\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"),\n ),\n ))\n)]\n#[macro_use]\nmod atomic_64_macros {\n #[macro_export]\n macro_rules! cfg_has_atomic_64 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_64 {\n ($($tt:tt)*) => {};\n }\n}\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(not(any(\n all(\n feature = \"fallback\",\n any(\n not(portable_atomic_no_atomic_cas),\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ),\n ),\n not(portable_atomic_no_atomic_64),\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")),\n all(\n target_arch = \"riscv32\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"),\n ),\n )))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(not(any(\n all(\n feature = \"fallback\",\n any(\n target_has_atomic = \"ptr\",\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ),\n ),\n target_has_atomic = \"64\",\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")),\n all(\n target_arch = \"riscv32\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"),\n ),\n )))\n)]\n#[macro_use]\nmod atomic_64_macros {\n #[macro_export]\n macro_rules! cfg_has_atomic_64 {\n ($($tt:tt)*) => {};\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_64 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n}\n\n#[cfg_attr(\n not(feature = \"fallback\"),\n cfg(any(\n all(\n target_arch = \"aarch64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n ),\n all(\n target_arch = \"arm64ec\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n ),\n all(\n target_arch = \"x86_64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n portable_atomic_no_cmpxchg16b_intrinsic,\n )),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n ),\n all(\n target_arch = \"riscv64\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"),\n ),\n all(\n target_arch = \"powerpc64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n ),\n ),\n all(\n target_arch = \"s390x\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n ),\n ))\n)]\n#[cfg_attr(\n all(feature = \"fallback\", portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n not(portable_atomic_no_atomic_cas),\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ))\n)]\n#[cfg_attr(\n all(feature = \"fallback\", not(portable_atomic_no_cfg_target_has_atomic)),\n cfg(any(\n target_has_atomic = \"ptr\",\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ))\n)]\n#[macro_use]\nmod atomic_128_macros {\n #[macro_export]\n macro_rules! cfg_has_atomic_128 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_128 {\n ($($tt:tt)*) => {};\n }\n}\n#[cfg_attr(\n not(feature = \"fallback\"),\n cfg(not(any(\n all(\n target_arch = \"aarch64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n ),\n all(\n target_arch = \"arm64ec\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n ),\n all(\n target_arch = \"x86_64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n portable_atomic_no_cmpxchg16b_intrinsic,\n )),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n ),\n all(\n target_arch = \"riscv64\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"),\n ),\n all(\n target_arch = \"powerpc64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n ),\n ),\n all(\n target_arch = \"s390x\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n ),\n )))\n)]\n#[cfg_attr(\n all(feature = \"fallback\", portable_atomic_no_cfg_target_has_atomic),\n cfg(not(any(\n not(portable_atomic_no_atomic_cas),\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )))\n)]\n#[cfg_attr(\n all(feature = \"fallback\", not(portable_atomic_no_cfg_target_has_atomic)),\n cfg(not(any(\n target_has_atomic = \"ptr\",\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )))\n)]\n#[macro_use]\nmod atomic_128_macros {\n #[macro_export]\n macro_rules! cfg_has_atomic_128 {\n ($($tt:tt)*) => {};\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_128 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n}\n\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(\n not(portable_atomic_no_atomic_cas),\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n target_has_atomic = \"ptr\",\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ))\n)]\n#[macro_use]\nmod atomic_cas_macros {\n #[macro_export]\n macro_rules! cfg_has_atomic_cas {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_cas {\n ($($tt:tt)*) => {};\n }\n // private\n macro_rules! cfg_has_atomic_cas_or_amo32 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n macro_rules! cfg_has_atomic_cas_or_amo8 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n}\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(not(any(\n not(portable_atomic_no_atomic_cas),\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(not(any(\n target_has_atomic = \"ptr\",\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )))\n)]\n#[macro_use]\nmod atomic_cas_macros {\n #[macro_export]\n macro_rules! cfg_has_atomic_cas {\n ($($tt:tt)*) => {};\n }\n #[macro_export]\n macro_rules! cfg_no_atomic_cas {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n // private\n #[cfg_attr(\n any(target_arch = \"riscv32\", target_arch = \"riscv64\"),\n cfg(not(any(target_feature = \"zaamo\", portable_atomic_target_feature = \"zaamo\")))\n )]\n macro_rules! cfg_has_atomic_cas_or_amo32 {\n ($($tt:tt)*) => {};\n }\n #[cfg_attr(\n any(target_arch = \"riscv32\", target_arch = \"riscv64\"),\n cfg(not(any(target_feature = \"zaamo\", portable_atomic_target_feature = \"zaamo\")))\n )]\n macro_rules! cfg_no_atomic_cas_or_amo32 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n #[cfg(all(\n any(target_arch = \"riscv32\", target_arch = \"riscv64\"),\n any(target_feature = \"zaamo\", portable_atomic_target_feature = \"zaamo\"),\n ))]\n macro_rules! cfg_has_atomic_cas_or_amo32 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n #[cfg(all(\n any(target_arch = \"riscv32\", target_arch = \"riscv64\"),\n any(target_feature = \"zaamo\", portable_atomic_target_feature = \"zaamo\"),\n ))]\n macro_rules! cfg_no_atomic_cas_or_amo32 {\n ($($tt:tt)*) => {};\n }\n #[cfg_attr(\n any(target_arch = \"riscv32\", target_arch = \"riscv64\"),\n cfg(not(any(target_feature = \"zabha\", portable_atomic_target_feature = \"zabha\")))\n )]\n #[allow(unused_macros)]\n macro_rules! cfg_has_atomic_cas_or_amo8 {\n ($($tt:tt)*) => {};\n }\n #[cfg_attr(\n any(target_arch = \"riscv32\", target_arch = \"riscv64\"),\n cfg(not(any(target_feature = \"zabha\", portable_atomic_target_feature = \"zabha\")))\n )]\n #[cfg_attr(target_arch = \"bpf\", allow(unused_macros))]\n macro_rules! cfg_no_atomic_cas_or_amo8 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n #[cfg(all(\n any(target_arch = \"riscv32\", target_arch = \"riscv64\"),\n any(target_feature = \"zabha\", portable_atomic_target_feature = \"zabha\"),\n ))]\n macro_rules! cfg_has_atomic_cas_or_amo8 {\n ($($tt:tt)*) => {\n $($tt)*\n };\n }\n #[cfg(all(\n any(target_arch = \"riscv32\", target_arch = \"riscv64\"),\n any(target_feature = \"zabha\", portable_atomic_target_feature = \"zabha\"),\n ))]\n macro_rules! cfg_no_atomic_cas_or_amo8 {\n ($($tt:tt)*) => {};\n }\n}\n\nmacro_rules! cfg_core_atomic {\n ($($tt:tt)*) => {\n #[cfg(not(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n all(\n portable_atomic_no_atomic_load_store,\n not(all(target_arch = \"bpf\", not(feature = \"critical-section\"))),\n ),\n )))]\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(not(all(\n any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ),\n portable_atomic_no_atomic_cas,\n )))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(not(all(\n any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ),\n not(target_has_atomic = \"ptr\"),\n )))\n )]\n items!($($tt)*);\n };\n}\n\n// Check that all cfg_ macros work.\nmod check {\n crate::cfg_has_atomic_8! { type _Atomic8 = (); }\n crate::cfg_no_atomic_8! { type _Atomic8 = (); }\n crate::cfg_has_atomic_16! { type _Atomic16 = (); }\n crate::cfg_no_atomic_16! { type _Atomic16 = (); }\n crate::cfg_has_atomic_32! { type _Atomic32 = (); }\n crate::cfg_no_atomic_32! { type _Atomic32 = (); }\n crate::cfg_has_atomic_64! { type _Atomic64 = (); }\n crate::cfg_no_atomic_64! { type _Atomic64 = (); }\n crate::cfg_has_atomic_128! { type _Atomic128 = (); }\n crate::cfg_no_atomic_128! { type _Atomic128 = (); }\n crate::cfg_has_atomic_ptr! { type _AtomicPtr = (); }\n crate::cfg_no_atomic_ptr! { type _AtomicPtr = (); }\n crate::cfg_has_atomic_cas! { type __AtomicPtr = (); }\n crate::cfg_no_atomic_cas! { type __AtomicPtr = (); }\n #[allow(unused_imports)]\n use self::{\n __AtomicPtr as _, _Atomic8 as _, _Atomic16 as _, _Atomic32 as _, _Atomic64 as _,\n _Atomic128 as _, _AtomicPtr as _,\n };\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/gen/build.rs", "sha256": "4ad2fa43f8f725631e9754338f39b80f64a2e61facdb3bd735efc215b95a6597", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n// This file is @generated by target_spec.sh.\n// It is not intended for manual editing.\n\n// Note: This is the list as of nightly-2022-02-10. We don't refer to this in\n// nightly-2022-02-11+ because feature(cfg_target_has_atomic) stabilized.\n#[rustfmt::skip]\npub(crate) static NO_ATOMIC_CAS: &[&str] = &[\n \"avr-unknown-gnu-atmega328\",\n \"bpfeb-unknown-none\",\n \"bpfel-unknown-none\",\n \"msp430-none-elf\",\n \"riscv32i-unknown-none-elf\",\n \"riscv32imc-unknown-none-elf\",\n \"thumbv4t-none-eabi\",\n \"thumbv6m-none-eabi\",\n];\n\n// Note: This is the list as of nightly-2022-02-10. We don't refer to this in\n// nightly-2022-02-11+ because feature(cfg_target_has_atomic) stabilized.\n#[rustfmt::skip]\npub(crate) static NO_ATOMIC_64: &[&str] = &[\n \"arm-linux-androideabi\",\n \"armebv7r-none-eabi\",\n \"armebv7r-none-eabihf\",\n \"armv4t-unknown-linux-gnueabi\",\n \"armv5te-unknown-linux-gnueabi\",\n \"armv5te-unknown-linux-musleabi\",\n \"armv5te-unknown-linux-uclibceabi\",\n \"armv6k-nintendo-3ds\",\n \"armv7r-none-eabi\",\n \"armv7r-none-eabihf\",\n \"avr-unknown-gnu-atmega328\",\n \"hexagon-unknown-linux-musl\",\n \"m68k-unknown-linux-gnu\",\n \"mips-unknown-linux-gnu\",\n \"mips-unknown-linux-musl\",\n \"mips-unknown-linux-uclibc\",\n \"mipsel-sony-psp\",\n \"mipsel-unknown-linux-gnu\",\n \"mipsel-unknown-linux-musl\",\n \"mipsel-unknown-linux-uclibc\",\n \"mipsel-unknown-none\",\n \"mipsisa32r6-unknown-linux-gnu\",\n \"mipsisa32r6el-unknown-linux-gnu\",\n \"msp430-none-elf\",\n \"powerpc-unknown-freebsd\",\n \"powerpc-unknown-linux-gnu\",\n \"powerpc-unknown-linux-gnuspe\",\n \"powerpc-unknown-linux-musl\",\n \"powerpc-unknown-netbsd\",\n \"powerpc-unknown-openbsd\",\n \"powerpc-wrs-vxworks\",\n \"powerpc-wrs-vxworks-spe\",\n \"riscv32gc-unknown-linux-gnu\",\n \"riscv32gc-unknown-linux-musl\",\n \"riscv32i-unknown-none-elf\",\n \"riscv32imac-unknown-none-elf\",\n \"riscv32imc-esp-espidf\",\n \"riscv32imc-unknown-none-elf\",\n \"thumbv4t-none-eabi\",\n \"thumbv6m-none-eabi\",\n \"thumbv7em-none-eabi\",\n \"thumbv7em-none-eabihf\",\n \"thumbv7m-none-eabi\",\n \"thumbv8m.base-none-eabi\",\n \"thumbv8m.main-none-eabi\",\n \"thumbv8m.main-none-eabihf\",\n];\n\n#[rustfmt::skip]\npub(crate) static NO_ATOMIC: &[&str] = &[\n \"armv4t-none-eabi\",\n \"armv5te-none-eabi\",\n \"bpfeb-unknown-none\",\n \"bpfel-unknown-none\",\n \"mipsel-sony-psx\",\n \"thumbv4t-none-eabi\",\n \"thumbv5te-none-eabi\",\n];\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/gen/utils.rs", "sha256": "c44ee71296aaa90c8104339e470688f631a2380d7f10349c7766205c415050aa", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n// This file is @generated by target_spec.sh.\n// It is not intended for manual editing.\n\n#![allow(dead_code, unused_macros)]\n\n// On AArch64, the base register of memory-related instructions must be 64-bit.\n// Passing a 32-bit value to `in(reg)` on AArch64 results in the upper bits\n// having an undefined value, but to work correctly with ILP32 ABI, the upper\n// bits must be zero, which is handled here by casting to u64. Another way to\n// handle this is to pass it as a pointer and clear the upper bits inside asm,\n// but it is easier to overlook than cast, which can catch overlooks by\n// asm_sub_register lint.\n// See also https://github.com/ARM-software/abi-aa/blob/2025Q4/aapcs64/aapcs64.rst#pointers\n//\n// Except for x86_64, which can use 32-bit registers in the destination operand\n// (on x86_64, we use the ptr_modifier macro to handle this), we need to do the\n// same for ILP32 ABI on other 64-bit architectures. (At least, as far as I can\n// see from the assembly generated by LLVM, this is also required for MIPS64 N32\n// ABI. I don't know about other similar ABIs, but in any case, this should be\n// a safe default for such ABIs).\n//\n// Known architectures that have such ABI are x86_64 (X32), AArch64 (ILP32),\n// mips64/mips64r6 (N32), riscv64 (RV64ILP32*), PowerPC64 (Lv2). (As of\n// 2026-03-22, only the former two are supported by rustc.) However, we list\n// all known 64-bit architectures because similar ABIs may exist or future added\n// for other architectures.\n#[cfg(all(\n target_pointer_width = \"32\",\n any(\n target_arch = \"aarch64\",\n target_arch = \"amdgpu\",\n target_arch = \"arm64ec\",\n target_arch = \"bpf\",\n target_arch = \"loongarch64\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_arch = \"nvptx64\",\n target_arch = \"powerpc64\",\n target_arch = \"riscv64\",\n target_arch = \"s390x\",\n target_arch = \"sparc64\",\n target_arch = \"wasm64\",\n target_arch = \"x86_64\",\n ),\n))]\n#[macro_use]\nmod imp {\n #[cfg(not(any(target_arch = \"x86\", target_arch = \"x86_64\")))]\n macro_rules! ptr_reg {\n ($ptr:ident) => {{\n let _: *const _ = $ptr; // ensure $ptr is a pointer (*mut _ or *const _)\n #[cfg(not(portable_atomic_no_asm_maybe_uninit))]\n #[allow(clippy::ptr_as_ptr)]\n {\n // If we cast to u64 here, the provenance will be lost,\n // so we convert to MaybeUninit via zero extend helper.\n crate::utils::zero_extend64_ptr($ptr as *mut ())\n }\n #[cfg(portable_atomic_no_asm_maybe_uninit)]\n {\n // Use cast on old rustc because it does not support MaybeUninit\n // registers. This is still permissive-provenance compatible and\n // is sound.\n $ptr as u64\n }\n }};\n }\n pub(crate) type RegISize = i64;\n pub(crate) type RegSize = u64;\n}\n#[cfg(not(all(\n target_pointer_width = \"32\",\n any(\n target_arch = \"aarch64\",\n target_arch = \"amdgpu\",\n target_arch = \"arm64ec\",\n target_arch = \"bpf\",\n target_arch = \"loongarch64\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_arch = \"nvptx64\",\n target_arch = \"powerpc64\",\n target_arch = \"riscv64\",\n target_arch = \"s390x\",\n target_arch = \"sparc64\",\n target_arch = \"wasm64\",\n target_arch = \"x86_64\",\n ),\n)))]\n#[macro_use]\nmod imp {\n #[cfg(not(any(target_arch = \"x86\", target_arch = \"x86_64\")))]\n macro_rules! ptr_reg {\n ($ptr:ident) => {{\n let _: *const _ = $ptr; // ensure $ptr is a pointer (*mut _ or *const _)\n $ptr // cast is unnecessary here.\n }};\n }\n pub(crate) type RegISize = isize;\n pub(crate) type RegSize = usize;\n}\npub(crate) use self::imp::{RegISize, RegSize};\n\n// Some 64-bit architectures have ABI with 32-bit pointer width (e.g., x86_64 X32 ABI,\n// AArch64 ILP32 ABI, MIPS64 N32 ABI). On those targets, AtomicU64 is available\n// and fast, so use it to implement normal sequence lock.\n//\n// See ptr_reg macro for the reason why all known 64-bit architectures are listed.\n//\n// Note: These macros are cfg_*!({ }), not cfg_*! { }.\n// An extra brace is used in input to make contents rustfmt-able.\n#[cfg(any(\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")), // i.e., 64-bit or greater\n target_arch = \"aarch64\",\n target_arch = \"amdgpu\",\n target_arch = \"arm64ec\",\n target_arch = \"bpf\",\n target_arch = \"loongarch64\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_arch = \"nvptx64\",\n target_arch = \"powerpc64\",\n target_arch = \"riscv64\",\n target_arch = \"s390x\",\n target_arch = \"sparc64\",\n target_arch = \"wasm64\",\n target_arch = \"x86_64\",\n))]\n#[macro_use]\nmod fast_atomic_64_macros {\n macro_rules! cfg_has_fast_atomic_64 {\n ({$($tt:tt)*}) => {\n $($tt)*\n };\n }\n macro_rules! cfg_no_fast_atomic_64 {\n ({$($tt:tt)*}) => {};\n }\n}\n#[cfg(not(any(\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")), // i.e., 64-bit or greater\n target_arch = \"aarch64\",\n target_arch = \"amdgpu\",\n target_arch = \"arm64ec\",\n target_arch = \"bpf\",\n target_arch = \"loongarch64\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_arch = \"nvptx64\",\n target_arch = \"powerpc64\",\n target_arch = \"riscv64\",\n target_arch = \"s390x\",\n target_arch = \"sparc64\",\n target_arch = \"wasm64\",\n target_arch = \"x86_64\",\n)))]\n#[macro_use]\nmod fast_atomic_64_macros {\n macro_rules! cfg_has_fast_atomic_64 {\n ({$($tt:tt)*}) => {};\n }\n macro_rules! cfg_no_fast_atomic_64 {\n ({$($tt:tt)*}) => {\n $($tt)*\n };\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/arm_linux.rs", "sha256": "6f5ba9542ec89340444094ee4c96503dec7476c38d308abd1b09168785d672c5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nAtomic operations implementation on pre-v6 ARM Linux/Android.\n\nThis module optimizes cases where LLVM code generation is not optimal,\nand provides workaround for core::sync::atomic bug (https://github.com/rust-lang/compiler-builtins/pull/1234).\n\nSee \"Atomic operation overview by architecture\" in atomic-maybe-uninit for a more comprehensive and\ndetailed description of the atomic and synchronize instructions in this architecture:\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#arm\n\nRefs:\n- https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst\n- https://github.com/rust-lang/compiler-builtins/blob/compiler_builtins-v0.1.124/src/arm_linux.rs\n- https://github.com/taiki-e/atomic-maybe-uninit\n\nNote: __kuser_memory_barrier is always SeqCst.\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.21/src/arch/arm.rs#L215\n\nNote: On Miri and ThreadSanitizer which do not go through compiler-builtins, we don't use\nthis module and use normal core::sync::atomic instead.\n*/\n\nuse core::{mem, sync::atomic::Ordering};\n\n// https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst\n// __kuser_helper_version >= 3 (kernel version 2.6.15+)\nconst KUSER_MEMORY_BARRIER: usize = 0xFFFF0FA0;\n#[inline]\nfn __kuser_memory_barrier() {\n // SAFETY: kernel docs specify a known address with the given signature.\n // core assumes __kuser_helper_version >= 3.\n unsafe {\n let f: extern \"C\" fn() =\n mem::transmute(crate::utils::ptr::with_exposed_provenance::<()>(KUSER_MEMORY_BARRIER));\n f();\n }\n}\n\n/// An atomic fence.\n///\n/// # Panics\n///\n/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).\n#[cfg_attr(doc, allow(dead_code))]\n#[inline]\n#[cfg_attr(any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri), track_caller)]\npub fn fence(order: Ordering) {\n // Call __kuser_memory_barrier without going through __sync_synchronize.\n match order {\n Ordering::Relaxed => panic!(\"there is no such thing as a relaxed fence\"),\n _ => __kuser_memory_barrier(),\n }\n}\n\ncfg_core_atomic!({\n use super::core_atomic::{\n AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicU32,\n AtomicUsize,\n };\n macro_rules! atomic {\n ($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn load(&self, order: Ordering) -> $value_type {\n crate::utils::assert_load_ordering(order);\n match order {\n Ordering::Relaxed => self.inner.load(Ordering::Relaxed),\n // Acquire and SeqCst loads are equivalent.\n // This matches with LLVM, but GCC emits `dmb; ldr*; dmb` for SeqCst load.\n Ordering::Acquire | Ordering::SeqCst => {\n let out = self.inner.load(Ordering::Relaxed);\n __kuser_memory_barrier();\n out\n }\n _ => unreachable!(),\n }\n }\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn store(&self, val: $value_type, order: Ordering) {\n crate::utils::assert_store_ordering(order);\n match order {\n Ordering::Relaxed => self.inner.store(val, Ordering::Relaxed),\n Ordering::Release => {\n __kuser_memory_barrier();\n self.inner.store(val, Ordering::Relaxed);\n }\n Ordering::SeqCst => {\n __kuser_memory_barrier();\n self.inner.store(val, Ordering::Relaxed);\n __kuser_memory_barrier();\n }\n _ => unreachable!(),\n }\n }\n }\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(not(portable_atomic_no_atomic_cas))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(target_has_atomic = \"ptr\")\n )]\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange(\n &self,\n current: $value_type,\n new: $value_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$value_type, $value_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)\n #[cfg(portable_atomic_no_stronger_failure_ordering)]\n let success = crate::utils::upgrade_success_ordering(success, failure);\n let res = self.inner.compare_exchange(current, new, success, failure);\n if failure != Ordering::Relaxed && res.is_err() {\n __kuser_memory_barrier();\n }\n res\n }\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange_weak(\n &self,\n current: $value_type,\n new: $value_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$value_type, $value_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)\n #[cfg(portable_atomic_no_stronger_failure_ordering)]\n let success = crate::utils::upgrade_success_ordering(success, failure);\n let res = self.inner.compare_exchange_weak(current, new, success, failure);\n if failure != Ordering::Relaxed && res.is_err() {\n __kuser_memory_barrier();\n }\n res\n }\n }\n };\n }\n\n atomic!(AtomicI8, i8);\n atomic!(AtomicU8, u8);\n atomic!(AtomicI16, i16);\n atomic!(AtomicU16, u16);\n atomic!(AtomicI32, i32);\n atomic!(AtomicU32, u32);\n atomic!(AtomicIsize, isize);\n atomic!(AtomicUsize, usize);\n atomic!([T] AtomicPtr, *mut T);\n});\n\n#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_64)))]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n target_has_atomic = \"64\",\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")),\n ))\n)]\nitems!({\n // This is not normally reached on this platform.\n // It is only reached if something strange occurs, such as using target-feature=-v6 on armv6 target.\n // Since 64-bit atomic operations differ from 32-bit or smaller ones on Arm, similar optimizations\n // or workarounds are useless here, so do the same as atomic_int! macro in core_atomic.rs.\n // NB: Sync with atomic_int! macro in core_atomic.rs\n use super::core_atomic::{AtomicI64, AtomicU64};\n macro_rules! atomic_int {\n ($atomic_type:ident, $int_type:ident) => {\n impl $atomic_type {\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn load(&self, order: Ordering) -> $int_type {\n crate::utils::assert_load_ordering(order); // for track_caller (compiler can omit double check)\n self.inner.load(order)\n }\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn store(&self, val: $int_type, order: Ordering) {\n crate::utils::assert_store_ordering(order); // for track_caller (compiler can omit double check)\n self.inner.store(val, order);\n }\n }\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(not(portable_atomic_no_atomic_cas))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(target_has_atomic = \"ptr\")\n )]\n impl $atomic_type {\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)\n #[cfg(portable_atomic_no_stronger_failure_ordering)]\n let success = crate::utils::upgrade_success_ordering(success, failure);\n self.inner.compare_exchange(current, new, success, failure)\n }\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange_weak(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)\n #[cfg(portable_atomic_no_stronger_failure_ordering)]\n let success = crate::utils::upgrade_success_ordering(success, failure);\n self.inner.compare_exchange_weak(current, new, success, failure)\n }\n }\n };\n }\n atomic_int!(AtomicI64, i64);\n atomic_int!(AtomicU64, u64);\n});\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic128/aarch64.rs", "sha256": "8fe3bebbf1a7991b35e6ffeb796ff872bc15edcdbbdcf7a5835cb2b1ce149542", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\n128-bit atomic implementation on AArch64.\n\nThis architecture provides the following 128-bit atomic instructions:\n\n- LDXP/STXP: LL/SC (Armv8.0 baseline)\n- CASP: CAS (added as Armv8.1 FEAT_LSE (optional from Armv8.0, mandatory from Armv8.1))\n- LDP/STP: load/store (if Armv8.4 FEAT_LSE2 (optional from Armv8.2, mandatory from Armv8.4) is available)\n- LDIAPP/STILP: acquire-load/release-store (added as Armv8.9 FEAT_LRCPC3 (optional from Armv8.2) (if FEAT_LSE2 is also available))\n- LDCLRP/LDSETP/SWPP: fetch-and-{clear,or},swap (added as Armv9.4 FEAT_LSE128 (optional from Armv9.3))\n\nSee \"Atomic operation overview by architecture\" in atomic-maybe-uninit for a more comprehensive and\ndetailed description of the atomic and synchronize instructions in this architecture:\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#aarch64\n\nThis module supports all of these instructions and attempts to select the best\none based on compile-time and run-time information about available CPU features\nand platforms. For example:\n\n- If outline-atomics is not enabled and FEAT_LSE is not available at\n compile-time, we use LDXP/STXP loop.\n- If outline-atomics is enabled and FEAT_LSE is not available at\n compile-time, we use CASP for CAS if FEAT_LSE is available\n at run-time, otherwise, use LDXP/STXP loop.\n- If FEAT_LSE is available at compile-time, we use CASP for load/store/CAS/RMW.\n However, when portable_atomic_ll_sc_rmw cfg is set, use LDXP/STXP loop instead of CASP\n loop for RMW (by default, it is set on Apple hardware where CASP is slow;\n see build script for details).\n- If outline-atomics is enabled and FEAT_LSE2 is not available at compile-time,\n we use LDP/STP (and also LDIAPP/STILP/SWPP if FEAT_LRCPC3/FEAT_LSE128 is\n available) for load/store if FEAT_LSE2 is available at run-time, otherwise,\n use LDXP/STXP or CASP depending on whether FEAT_LSE is available.\n- If FEAT_LSE2 is available at compile-time, we use LDP/STP for load/store.\n- If FEAT_LSE128 is available at compile-time, we use LDCLRP/LDSETP/SWPP for fetch_and/fetch_or/swap/{release,seqcst}-store.\n- If FEAT_LSE2 and FEAT_LRCPC3 are available at compile-time, we use LDIAPP/STILP for acquire-load/release-store.\n\nSee each \"Instruction selection flow for ...\" comment in this file for the exact\ninstruction selection per operation.\n\nNote: FEAT_LSE2 doesn't imply FEAT_LSE. FEAT_LSE128 implies FEAT_LSE but not FEAT_LSE2.\n\nNote that we do not separate LL and SC into separate functions, but handle\nthem within a single asm block. This is because it is theoretically possible\nfor the compiler to insert operations that might clear the reservation between\nLL and SC. Considering the type of operations we are providing and the fact\nthat [progress64](https://github.com/ARM-software/progress64) uses such code,\nthis is probably not a problem for AArch64, but it seems that AArch64 doesn't\nguarantee it and hexagon is the only architecture with hardware guarantees\nthat such code works. See also:\n\n- https://yarchive.net/comp/linux/cmpxchg_ll_sc_portability.html\n- https://lists.llvm.org/pipermail/llvm-dev/2016-May/099490.html\n- https://lists.llvm.org/pipermail/llvm-dev/2018-June/123993.html\n\nAlso, even when using a CAS loop to implement atomic RMW, include the loop itself\nin the asm block because it is more efficient for some codegen backends.\nhttps://github.com/rust-lang/compiler-builtins/issues/339#issuecomment-1191260474\n\nNote: On Miri and ThreadSanitizer which do not support inline assembly, we don't use\nthis module and use intrinsics.rs instead.\n\nRefs:\n- Arm A-profile A64 Instruction Set Architecture\n https://developer.arm.com/documentation/ddi0602/2025-06\n- C/C++ Atomics Application Binary Interface Standard for the Arm® 64-bit Architecture\n https://github.com/ARM-software/abi-aa/blob/2025Q4/atomicsabi64/atomicsabi64.rst\n- Arm Compiler armasm User Guide\n https://developer.arm.com/documentation/dui0801/latest\n- Arm Architecture Reference Manual for A-profile architecture\n https://developer.arm.com/documentation/ddi0487/latest (PDF)\n- Arm Architecture Reference Manual Supplement - Armv8, for Armv8-R AArch64 architecture profile\n https://developer.arm.com/documentation/ddi0600/latest (PDF)\n- atomic-maybe-uninit https://github.com/taiki-e/atomic-maybe-uninit\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\ninclude!(\"macros.rs\");\n\n// On musl with static linking, it seems that getauxval is not always available.\n// See detect/auxv.rs for more.\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(\n test,\n not(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\"),\n )),\n))]\n#[cfg(any(\n all(\n target_os = \"linux\",\n any(\n target_env = \"gnu\",\n all(target_env = \"musl\", any(not(target_feature = \"crt-static\"), feature = \"std\")),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n target_os = \"freebsd\",\n))]\n#[path = \"../detect/auxv.rs\"]\nmod detect;\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(\n test,\n not(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\"),\n )),\n))]\n#[cfg(any(target_os = \"netbsd\", target_os = \"openbsd\"))]\n#[path = \"../detect/aarch64_aa64reg.rs\"]\nmod detect;\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(\n test,\n not(all(\n any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\"),\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n )),\n))]\n#[cfg(target_os = \"illumos\")]\n#[path = \"../detect/aarch64_illumos.rs\"]\nmod detect;\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(test, not(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"))))]\n#[cfg(target_os = \"fuchsia\")]\n#[path = \"../detect/aarch64_fuchsia.rs\"]\nmod detect;\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(\n test,\n not(all(\n any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\"),\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n )),\n))]\n#[cfg(target_vendor = \"apple\")]\n#[path = \"../detect/aarch64_apple.rs\"]\nmod detect;\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(\n test,\n not(all(\n any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\"),\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n )),\n))]\n#[cfg(windows)]\n#[path = \"../detect/aarch64_windows.rs\"]\nmod detect;\n\n#[cfg(test)] // test-only (we use auxv.rs instead)\n#[cfg(not(valgrind))]\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(target_os = \"linux\", target_os = \"android\", target_os = \"freebsd\"))]\n#[path = \"../detect/aarch64_aa64reg.rs\"]\nmod test_detect_aa64reg;\n#[cfg(test)] // test-only (we use aarch64_aa64reg.rs instead)\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(target_os = \"openbsd\")]\n#[path = \"../detect/auxv.rs\"]\nmod test_detect_auxv;\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\nuse core::sync::atomic::Ordering;\n\n#[cfg(portable_atomic_no_strict_provenance)]\nuse crate::utils::ptr::PtrExt as _;\nuse crate::utils::{Pair, U128};\n\n#[cfg(any(\n target_feature = \"lse\",\n portable_atomic_target_feature = \"lse\",\n not(portable_atomic_no_outline_atomics),\n))]\n#[rustfmt::skip]\nmacro_rules! debug_assert_lse {\n () => {\n #[cfg(all(\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n target_env = \"gnu\",\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n target_os = \"freebsd\",\n target_os = \"netbsd\",\n target_os = \"openbsd\",\n target_os = \"illumos\",\n target_os = \"fuchsia\",\n target_vendor = \"apple\",\n windows,\n ),\n ))]\n #[cfg(not(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\")))]\n {\n debug_assert!(detect::detect().lse());\n }\n };\n}\n#[cfg(any(\n target_feature = \"lse2\",\n portable_atomic_target_feature = \"lse2\",\n not(portable_atomic_no_outline_atomics),\n))]\n#[rustfmt::skip]\nmacro_rules! debug_assert_lse2 {\n () => {\n #[cfg(all(\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n target_env = \"gnu\",\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n target_os = \"freebsd\",\n target_os = \"netbsd\",\n target_os = \"openbsd\",\n target_os = \"illumos\",\n // These don't support detection of FEAT_LSE2.\n // target_os = \"fuchsia\",\n target_vendor = \"apple\",\n windows,\n ),\n ))]\n #[cfg(not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")))]\n {\n debug_assert!(detect::detect().lse2());\n }\n };\n}\n#[cfg(any(\n target_feature = \"lse128\",\n portable_atomic_target_feature = \"lse128\",\n all(\n not(portable_atomic_no_outline_atomics),\n not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")),\n ),\n))]\n#[rustfmt::skip]\nmacro_rules! debug_assert_lse128 {\n () => {\n #[cfg(all(\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n target_env = \"gnu\",\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n target_os = \"freebsd\",\n target_os = \"netbsd\",\n target_os = \"openbsd\",\n target_os = \"illumos\",\n // These don't support detection of FEAT_LSE128.\n // target_os = \"fuchsia\",\n target_vendor = \"apple\",\n windows,\n ),\n ))]\n #[cfg(not(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\")))]\n {\n debug_assert!(detect::detect().lse128());\n }\n };\n}\n#[cfg(any(\n target_feature = \"rcpc3\",\n portable_atomic_target_feature = \"rcpc3\",\n all(\n not(portable_atomic_no_outline_atomics),\n not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")),\n ),\n))]\n#[rustfmt::skip]\nmacro_rules! debug_assert_rcpc3 {\n () => {\n #[cfg(all(\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n target_env = \"gnu\",\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n target_os = \"freebsd\",\n target_os = \"netbsd\",\n target_os = \"openbsd\",\n target_os = \"illumos\",\n // These don't support detection of FEAT_LRCPC3.\n // target_os = \"fuchsia\",\n target_vendor = \"apple\",\n windows,\n ),\n ))]\n #[cfg(not(any(target_feature = \"rcpc3\", portable_atomic_target_feature = \"rcpc3\")))]\n {\n debug_assert!(detect::detect().rcpc3());\n }\n };\n}\n\n// Refs: https://developer.arm.com/documentation/100067/0611/armclang-Integrated-Assembler/AArch32-Target-selection-directives\n//\n// This is similar to #[target_feature(enable = \"lse\")], except that there are\n// no compiler guarantees regarding (un)inlining, and the scope is within an asm\n// block rather than a function. We use this directive because #[target_feature(enable = \"lse\")]\n// is unstable on pre-1.61 rustc and incompatible with rustc_codegen_cranelift:\n// https://github.com/rust-lang/rustc_codegen_cranelift/issues/1400#issuecomment-1774599775\n//\n// The .arch_extension directive in asm! is effective until the end of the assembly block and\n// is not propagated to subsequent code, so the end_lse macro is unneeded.\n// https://godbolt.org/z/o6EPndP94\n// https://github.com/torvalds/linux/commit/e0d5896bd356cd577f9710a02d7a474cdf58426b\n// https://github.com/torvalds/linux/commit/dd1f6308b28edf0452dd5dc7877992903ec61e69\n// (It seems GCC effectively ignores this directive and always allow FEAT_LSE instructions: https://godbolt.org/z/W9W6rensG)\n// Note that the .arch_extension directive in global_asm!/naked_asm! which are\n// not used in this crate has different behavior: https://github.com/rust-lang/rust/pull/137720#discussion_r1973608259\n// Note that this directive currently cannot be used correctly with global_asm!/naked_asm!\n// due to LLVM bug: https://github.com/rust-lang/rust/pull/137720#discussion_r2014505753\n//\n// The .arch directive has a similar effect, but we don't use it due to the following issue:\n// https://github.com/torvalds/linux/commit/dd1f6308b28edf0452dd5dc7877992903ec61e69\n//\n// Note: If FEAT_LSE is not available at compile-time, we must guarantee that\n// the function that uses it is not inlined into a function where it is not\n// clear whether FEAT_LSE is available. Otherwise, (even if we checked whether\n// FEAT_LSE is available at run-time) optimizations that reorder its\n// instructions across the if condition might introduce undefined behavior.\n// (see also https://rust-lang.github.io/rfcs/2045-target-feature.html#safely-inlining-target_feature-functions-on-more-contexts)\n// However, our code uses the ifunc helper macro that works with function pointers,\n// so we don't have to worry about this unless calling without helper macro.\n#[cfg(any(\n target_feature = \"lse\",\n portable_atomic_target_feature = \"lse\",\n not(portable_atomic_no_outline_atomics),\n))]\nmacro_rules! start_lse {\n () => {\n \".arch_extension lse\"\n };\n}\n#[cfg(not(portable_atomic_pre_llvm_16))]\n#[cfg(any(\n target_feature = \"lse128\",\n portable_atomic_target_feature = \"lse128\",\n all(\n not(portable_atomic_no_outline_atomics),\n not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")),\n ),\n))]\nmacro_rules! start_lse128 {\n () => {\n \".arch_extension lse128\"\n };\n}\n#[cfg(not(portable_atomic_pre_llvm_16))]\n#[cfg(any(\n target_feature = \"rcpc3\",\n portable_atomic_target_feature = \"rcpc3\",\n all(\n not(portable_atomic_no_outline_atomics),\n not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")),\n ),\n))]\nmacro_rules! start_rcpc3 {\n () => {\n \".arch_extension rcpc3\"\n };\n}\n\n#[cfg(target_endian = \"little\")]\nmacro_rules! select_le_or_be {\n ($le:expr, $be:expr) => {\n $le\n };\n}\n#[cfg(target_endian = \"big\")]\nmacro_rules! select_le_or_be {\n ($le:expr, $be:expr) => {\n $be\n };\n}\n\nmacro_rules! atomic_rmw {\n ($op:ident, $order:ident) => {\n atomic_rmw!($op, $order, write = $order)\n };\n ($op:ident, $order:ident, write = $write:ident) => {\n match $order {\n Ordering::Relaxed => $op!(\"\", \"\", \"\"),\n Ordering::Acquire => $op!(\"a\", \"\", \"\"),\n Ordering::Release => $op!(\"\", \"l\", \"\"),\n Ordering::AcqRel => $op!(\"a\", \"l\", \"\"),\n // In MSVC environments, SeqCst stores/writes by non-LSE* instructions needs fences after writes.\n // https://reviews.llvm.org/D141748\n // https://github.com/llvm/llvm-project/commit/1ea201d73be2fdf03347e9c6be09ebed5f8e0e00\n #[cfg(target_env = \"msvc\")]\n Ordering::SeqCst if $write == Ordering::SeqCst => $op!(\"a\", \"l\", \"dmb ish\"),\n // AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.\n Ordering::SeqCst => $op!(\"a\", \"l\", \"\"),\n _ => unreachable!(),\n }\n };\n}\n#[cfg(portable_atomic_pre_llvm_16)]\n#[cfg(any(\n target_feature = \"lse128\",\n portable_atomic_target_feature = \"lse128\",\n all(\n not(portable_atomic_no_outline_atomics),\n not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")),\n ),\n))]\nmacro_rules! atomic_rmw_inst {\n ($op:ident, $order:ident) => {\n atomic_rmw_inst!($op, $order, write = $order)\n };\n ($op:ident, $order:ident, write = $write:ident) => {\n match $order {\n Ordering::Relaxed => $op!(\"2\", \"\"), // \"\"\n Ordering::Acquire => $op!(\"a\", \"\"), // \"a\"\n Ordering::Release => $op!(\"6\", \"\"), // \"l\"\n Ordering::AcqRel => $op!(\"e\", \"\"), // \"al\"\n // In MSVC environments, SeqCst stores/writes by non-LSE* instructions needs fences after writes.\n // https://reviews.llvm.org/D141748\n // https://github.com/llvm/llvm-project/commit/1ea201d73be2fdf03347e9c6be09ebed5f8e0e00\n #[cfg(target_env = \"msvc\")]\n Ordering::SeqCst if $write == Ordering::SeqCst => $op!(\"e\", \"dmb ish\"),\n // AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.\n Ordering::SeqCst => $op!(\"e\", \"\"),\n _ => unreachable!(),\n }\n };\n}\n\n// -----------------------------------------------------------------------------\n// load\n\n/*\n\nInstruction selection flow for load:\n- if compile_time(FEAT_LSE2) => ldp:\n - if compile_time(FEAT_LRCPC3) && order != relaxed => ldiapp\n - else => ldp\n- if platform_supports_detection_of(FEAT_LSE2):\n - if detect(FEAT_LSE2) && detect(FEAT_LRCPC3) && order != relaxed => lse2_rcpc3 (ldiapp)\n - if detect(FEAT_LSE2) => lse2 (ldp)\n- else => no_lse2:\n - if compile_time(FEAT_LSE) => casp\n - else => ldxp_stxp\n\nNote:\n- If FEAT_LSE2 is available at compile-time, we don't do run-time detection of\n FEAT_LRCPC3 at this time, since FEAT_LRCPC3 is not yet available for most CPUs.\n (macOS that doesn't have any FEAT_LRCPC3-enabled CPUs as of M4 is only a platform\n that currently enables FEAT_LSE2 at compile-time by default.)\n- If FEAT_LSE2 is not available at compile-time, we want to do run-time detection\n of FEAT_LSE2, so we do run-time detection of FEAT_LRCPC3 at the same time.\n- We don't do run-time detection of FEAT_LSE for load at this time, but since\n load by CAS is wait-free, it would probably make sense to do run-time detection. (TODO)\n\n*/\n\ncfg_sel!({\n // if compile_time(FEAT_LSE2) => ldp:\n // cfg guarantee that the CPU supports FEAT_LSE2.\n #[cfg(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\"))]\n {\n use self::_atomic_load_ldp as atomic_load;\n }\n // if platform_supports_detection_of(FEAT_LSE2):\n #[cfg(all(\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n target_env = \"gnu\",\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n target_os = \"freebsd\",\n target_os = \"netbsd\",\n target_os = \"openbsd\",\n target_os = \"illumos\",\n // These don't support detection of FEAT_LSE2.\n // target_os = \"fuchsia\",\n target_vendor = \"apple\",\n windows,\n ),\n ))]\n {\n #[inline]\n unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(src: *mut u128) -> u128;\n atomic_load_lse2_relaxed = _atomic_load_ldp(Ordering::Relaxed);\n atomic_load_lse2_acquire = _atomic_load_ldp(Ordering::Acquire);\n atomic_load_lse2_seqcst = _atomic_load_ldp(Ordering::SeqCst);\n atomic_load_lse2_rcpc3_acquire = _atomic_load_ldiapp(Ordering::Acquire);\n atomic_load_lse2_rcpc3_seqcst = _atomic_load_ldiapp(Ordering::SeqCst);\n }\n fn_alias! {\n unsafe fn(src: *mut u128) -> u128;\n atomic_load_no_lse2_relaxed = atomic_load_no_lse2(Ordering::Relaxed);\n atomic_load_no_lse2_acquire = atomic_load_no_lse2(Ordering::Acquire);\n atomic_load_no_lse2_seqcst = atomic_load_no_lse2(Ordering::SeqCst);\n }\n // SAFETY: the caller must uphold the safety contract.\n // and we've checked if FEAT_LSE2/FEAT_LRCPC3 is available.\n unsafe {\n match order {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(src: *mut u128) -> u128 {\n let cpuinfo = detect::detect();\n if cpuinfo.lse2() {\n // if detect(FEAT_LSE2) => lse2 (ldp)\n atomic_load_lse2_relaxed\n } else {\n // else => no_lse2:\n atomic_load_no_lse2_relaxed\n }\n })\n }\n Ordering::Acquire => {\n ifunc!(unsafe fn(src: *mut u128) -> u128 {\n let cpuinfo = detect::detect();\n if cpuinfo.lse2() {\n if cpuinfo.rcpc3() {\n // if detect(FEAT_LSE2) && detect(FEAT_LRCPC3) && order != relaxed => lse2_rcpc3 (ldiapp)\n atomic_load_lse2_rcpc3_acquire\n } else {\n // if detect(FEAT_LSE2) => lse2 (ldp)\n atomic_load_lse2_acquire\n }\n } else {\n // else => no_lse2:\n atomic_load_no_lse2_acquire\n }\n })\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(src: *mut u128) -> u128 {\n let cpuinfo = detect::detect();\n if cpuinfo.lse2() {\n if cpuinfo.rcpc3() {\n // if detect(FEAT_LSE2) && detect(FEAT_LRCPC3) && order != relaxed => lse2_rcpc3 (ldiapp)\n atomic_load_lse2_rcpc3_seqcst\n } else {\n // if detect(FEAT_LSE2) => lse2 (ldp)\n atomic_load_lse2_seqcst\n }\n } else {\n // else => no_lse2:\n atomic_load_no_lse2_seqcst\n }\n })\n }\n _ => unreachable!(),\n }\n }\n }\n }\n // else => no_lse2:\n #[cfg(else)]\n {\n use self::atomic_load_no_lse2 as atomic_load;\n }\n});\n#[cfg(not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")))]\ncfg_sel!({\n // if compile_time(FEAT_LSE) => casp\n // cfg guarantee that the CPU supports FEAT_LSE.\n #[cfg(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"))]\n {\n use self::_atomic_load_casp as atomic_load_no_lse2;\n }\n // else => ldxp_stxp\n #[cfg(else)]\n {\n use self::_atomic_load_ldxp_stxp as atomic_load_no_lse2;\n }\n});\n// If CPU supports FEAT_LSE2, LDP/LDIAPP is single-copy atomic reads,\n// otherwise it is two single-copy atomic reads.\n// Refs: B2.2.1 of the Arm Architecture Reference Manual Armv8, for Armv8-A architecture profile\n#[cfg(any(\n target_feature = \"lse2\",\n portable_atomic_target_feature = \"lse2\",\n not(portable_atomic_no_outline_atomics),\n))]\n#[inline]\nunsafe fn _atomic_load_ldp(src: *mut u128, order: Ordering) -> u128 {\n debug_assert!(src.addr() % 16 == 0);\n debug_assert_lse2!();\n\n // SAFETY: the caller must guarantee that `dst` is valid for reads,\n // 16-byte aligned, that there are no concurrent non-atomic operations.\n //\n // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDP--Load-pair-of-registers-\n unsafe {\n let (out_lo, out_hi);\n macro_rules! atomic_load_relaxed {\n ($acquire:tt) => {{\n asm!(\n \"ldp {out_lo}, {out_hi}, [{src}]\", // atomic { out_lo:out_hi = *src }\n $acquire, // fence\n src = in(reg) ptr_reg!(src),\n out_hi = lateout(reg) out_hi,\n out_lo = lateout(reg) out_lo,\n options(nostack, preserves_flags),\n );\n U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole\n }};\n }\n match order {\n // if compile_time(FEAT_LRCPC3) && order != relaxed => ldiapp\n // SAFETY: cfg guarantee that the CPU supports FEAT_LRCPC3.\n #[cfg(any(target_feature = \"rcpc3\", portable_atomic_target_feature = \"rcpc3\"))]\n Ordering::Acquire | Ordering::SeqCst => _atomic_load_ldiapp(src, order),\n\n // else => ldp\n Ordering::Relaxed => atomic_load_relaxed!(\"\"),\n #[cfg(not(any(target_feature = \"rcpc3\", portable_atomic_target_feature = \"rcpc3\")))]\n Ordering::Acquire => atomic_load_relaxed!(\"dmb ishld\"),\n #[cfg(not(any(target_feature = \"rcpc3\", portable_atomic_target_feature = \"rcpc3\")))]\n Ordering::SeqCst => {\n asm!(\n // ldar (or dmb ishld) is required to prevent reordering with preceding stlxp.\n // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108891 for details.\n \"ldar {tmp}, [{src}]\", // atomic { tmp = *src }\n \"ldp {out_lo}, {out_hi}, [{src}]\", // atomic { out_lo:out_hi = *src }\n \"dmb ishld\", // fence\n src = in(reg) ptr_reg!(src),\n out_hi = lateout(reg) out_hi,\n out_lo = lateout(reg) out_lo,\n tmp = out(reg) _,\n options(nostack, preserves_flags),\n );\n U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole\n }\n _ => unreachable!(),\n }\n }\n}\n#[cfg(any(\n target_feature = \"lse2\",\n portable_atomic_target_feature = \"lse2\",\n not(portable_atomic_no_outline_atomics),\n))]\n#[cfg(any(\n target_feature = \"rcpc3\",\n portable_atomic_target_feature = \"rcpc3\",\n all(\n not(portable_atomic_no_outline_atomics),\n not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")),\n ),\n))]\n#[inline]\nunsafe fn _atomic_load_ldiapp(src: *mut u128, order: Ordering) -> u128 {\n debug_assert!(src.addr() % 16 == 0);\n debug_assert_lse2!();\n debug_assert_rcpc3!();\n\n // SAFETY: the caller must guarantee that `dst` is valid for reads,\n // 16-byte aligned, that there are no concurrent non-atomic operations.\n //\n // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDIAPP--Load-Acquire-RCpc-ordered-pair-of-registers-\n unsafe {\n let (out_lo, out_hi);\n match order {\n Ordering::Acquire => {\n #[cfg(not(portable_atomic_pre_llvm_16))]\n asm!(\n start_rcpc3!(),\n \"ldiapp {out_lo}, {out_hi}, [{src}]\", // atomic { out_lo:out_hi = *src }\n src = in(reg) ptr_reg!(src),\n out_hi = lateout(reg) out_hi,\n out_lo = lateout(reg) out_lo,\n options(nostack, preserves_flags),\n );\n // LLVM supports FEAT_LRCPC3 instructions on LLVM 16+, so use .inst directive on old LLVM.\n // https://github.com/llvm/llvm-project/commit/a6aaa969f7caec58a994142f8d855861cf3a1463\n #[cfg(portable_atomic_pre_llvm_16)]\n asm!(\n // ldiapp x0, x1, [x0] // atomic { out_lo:out_hi = *src }\n \".inst 0xd9411800\",\n in(\"x0\") ptr_reg!(src),\n lateout(\"x1\") out_hi,\n lateout(\"x0\") out_lo,\n options(nostack, preserves_flags),\n );\n }\n Ordering::SeqCst => {\n #[cfg(not(portable_atomic_pre_llvm_16))]\n asm!(\n start_rcpc3!(),\n // ldar (or dmb ishld) is required to prevent reordering with preceding stlxp.\n // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108891 for details.\n \"ldar {tmp}, [{src}]\", // atomic { tmp = *src }\n \"ldiapp {out_lo}, {out_hi}, [{src}]\", // atomic { out_lo:out_hi = *src }\n src = in(reg) ptr_reg!(src),\n out_hi = lateout(reg) out_hi,\n out_lo = lateout(reg) out_lo,\n tmp = out(reg) _,\n options(nostack, preserves_flags),\n );\n // LLVM supports FEAT_LRCPC3 instructions on LLVM 16+, so use .inst directive on old LLVM.\n // https://github.com/llvm/llvm-project/commit/a6aaa969f7caec58a994142f8d855861cf3a1463\n #[cfg(portable_atomic_pre_llvm_16)]\n asm!(\n // ldar (or dmb ishld) is required to prevent reordering with preceding stlxp.\n // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108891 for details.\n \"ldar {tmp}, [x0]\", // atomic { out_lo:out_hi = *src }\n // ldiapp x0, x1, [x0] // atomic { tmp = *src }\n \".inst 0xd9411800\",\n tmp = out(reg) _,\n in(\"x0\") ptr_reg!(src),\n lateout(\"x1\") out_hi,\n lateout(\"x0\") out_lo,\n options(nostack, preserves_flags),\n );\n }\n _ => unreachable!(),\n }\n U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole\n }\n}\n// Do not use _atomic_compare_exchange_casp because it needs extra MOV to implement load.\n#[cfg(any(test, not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\"))))]\n#[cfg(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"))]\n#[inline]\nunsafe fn _atomic_load_casp(src: *mut u128, order: Ordering) -> u128 {\n debug_assert!(src.addr() % 16 == 0);\n debug_assert_lse!();\n\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantee that the CPU supports FEAT_LSE.\n unsafe {\n let (out_lo, out_hi);\n macro_rules! atomic_load {\n ($acquire:tt, $release:tt) => {\n asm!(\n start_lse!(),\n concat!(\"casp\", $acquire, $release, \" x2, x3, x2, x3, [{src}]\"), // atomic { if *src == x2:x3 { *dst = x2:x3 } else { x2:x3 = *dst } }\n src = in(reg) ptr_reg!(src),\n // must be allocated to even/odd register pair\n inout(\"x2\") 0_u64 => out_lo,\n inout(\"x3\") 0_u64 => out_hi,\n options(nostack, preserves_flags),\n )\n };\n }\n match order {\n Ordering::Relaxed => atomic_load!(\"\", \"\"),\n Ordering::Acquire => atomic_load!(\"a\", \"\"),\n // TODO: in atomicsabi64, seqcst load is the same as acquire load\n Ordering::SeqCst => atomic_load!(\"a\", \"l\"),\n _ => unreachable!(),\n }\n U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole\n }\n}\n#[cfg(any(\n test,\n all(\n not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")),\n not(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\")),\n ),\n))]\n#[inline]\nunsafe fn _atomic_load_ldxp_stxp(src: *mut u128, order: Ordering) -> u128 {\n debug_assert!(src.addr() % 16 == 0);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n let (mut out_lo, mut out_hi);\n macro_rules! atomic_load {\n ($acquire:tt, $release:tt) => {\n asm!(\n \"2:\", // 'retry:\n concat!(\"ld\", $acquire, \"xp {out_lo}, {out_hi}, [{src}]\"), // atomic { out_lo:out_hi = *src; EXCLUSIVE = src }\n // write back to ensure atomicity\n concat!(\"st\", $release, \"xp {r:w}, {out_lo}, {out_hi}, [{src}]\"), // atomic { if EXCLUSIVE == src { *src = out_lo:out_hi; r = 0 } else { r = 1 }; EXCLUSIVE = None }\n \"cbnz {r:w}, 2b\", // if r != 0 { jump 'retry }\n src = in(reg) ptr_reg!(src),\n out_lo = out(reg) out_lo,\n out_hi = out(reg) out_hi,\n r = out(reg) _,\n options(nostack, preserves_flags),\n )\n };\n }\n match order {\n Ordering::Relaxed => atomic_load!(\"\", \"\"),\n Ordering::Acquire => atomic_load!(\"a\", \"\"),\n // TODO: in atomicsabi64, seqcst load is the same as acquire load\n Ordering::SeqCst => atomic_load!(\"a\", \"l\"),\n _ => unreachable!(),\n }\n U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole\n }\n}\n\n// -----------------------------------------------------------------------------\n// store\n\n/*\n\nInstruction selection flow for store:\n- if compile_time(FEAT_LSE2) => stp:\n - if compile_time(FEAT_LSE128) && order == seqcst => swpp\n - if compile_time(FEAT_LRCPC3) && order != relaxed => stilp\n - if compile_time(FEAT_LSE128) && order != relaxed => swpp\n - else => stp\n- if platform_supports_detection_of(FEAT_LSE2):\n - if detect(FEAT_LSE2) && detect(FEAT_LSE128) && order == seqcst => lse128 (swpp)\n - if detect(FEAT_LSE2) && detect(FEAT_LRCPC3) && order != relaxed => lse2_rcpc3 (stilp)\n - if detect(FEAT_LSE2) && detect(FEAT_LSE128) && order != relaxed => lse128 (swpp)\n - if detect(FEAT_LSE2) => lse2 (stp)\n- else => no_lse2:\n - if compile_time(FEAT_LSE) && not(ll_sc_rmw) => casp\n - else => ldxp_stxp\n\nNote:\n- If FEAT_LSE2 is available at compile-time, we don't do run-time detection of\n FEAT_LRCPC3/FEAT_LSE128 at this time, since FEAT_LRCPC3/FEAT_LSE128 is not yet available for most CPUs.\n (macOS that doesn't have any FEAT_LRCPC3/FEAT_LSE128-enabled CPUs as of M4 is only a platform\n that currently enables FEAT_LSE2 at compile-time by default.)\n- If FEAT_LSE2 is not available at compile-time, we want to do run-time detection\n of FEAT_LSE2, so we do run-time detection of FEAT_LRCPC3/FEAT_LSE128 at the same time.\n- We don't do run-time detection of FEAT_LSE for store at this time.\n\n*/\n\ncfg_sel!({\n // if compile_time(FEAT_LSE2) => stp:\n // cfg guarantee that the CPU supports FEAT_LSE2.\n #[cfg(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\"))]\n {\n use self::_atomic_store_stp as atomic_store;\n }\n // if platform_supports_detection_of(FEAT_LSE2):\n #[cfg(all(\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n target_env = \"gnu\",\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n target_os = \"freebsd\",\n target_os = \"netbsd\",\n target_os = \"openbsd\",\n target_os = \"illumos\",\n // These don't support detection of FEAT_LSE2.\n // target_os = \"fuchsia\",\n target_vendor = \"apple\",\n windows,\n ),\n ))]\n {\n #[inline]\n unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {\n #[cfg(any(\n target_feature = \"lse128\",\n portable_atomic_target_feature = \"lse128\",\n all(\n not(portable_atomic_no_outline_atomics),\n not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")),\n ),\n ))]\n #[inline]\n unsafe fn _atomic_store_swpp(dst: *mut u128, val: u128, order: Ordering) {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n _atomic_swap_swpp(dst, val, order);\n }\n }\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(dst: *mut u128, val: u128);\n atomic_store_lse2_relaxed = _atomic_store_stp(Ordering::Relaxed);\n atomic_store_lse2_release = _atomic_store_stp(Ordering::Release);\n atomic_store_lse2_seqcst = _atomic_store_stp(Ordering::SeqCst);\n atomic_store_lse2_rcpc3_release = _atomic_store_stilp(Ordering::Release);\n atomic_store_lse2_rcpc3_seqcst = _atomic_store_stilp(Ordering::SeqCst);\n atomic_store_lse128_release = _atomic_store_swpp(Ordering::Release);\n atomic_store_lse128_seqcst = _atomic_store_swpp(Ordering::SeqCst);\n }\n fn_alias! {\n unsafe fn(dst: *mut u128, val: u128);\n atomic_store_no_lse2_relaxed = atomic_store_no_lse2(Ordering::Relaxed);\n atomic_store_no_lse2_release = atomic_store_no_lse2(Ordering::Release);\n atomic_store_no_lse2_seqcst = atomic_store_no_lse2(Ordering::SeqCst);\n }\n // SAFETY: the caller must uphold the safety contract.\n // and we've checked if FEAT_LSE2/FEAT_LRCPC3/FEAT_LSE128 is available.\n unsafe {\n match order {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(dst: *mut u128, val: u128) {\n let cpuinfo = detect::detect();\n if cpuinfo.lse2() {\n // if detect(FEAT_LSE2) => lse2 (stp)\n atomic_store_lse2_relaxed\n } else {\n // else => no_lse2:\n atomic_store_no_lse2_relaxed\n }\n });\n }\n Ordering::Release => {\n ifunc!(unsafe fn(dst: *mut u128, val: u128) {\n let cpuinfo = detect::detect();\n if cpuinfo.lse2() {\n if cpuinfo.rcpc3() {\n // if detect(FEAT_LSE2) && detect(FEAT_LRCPC3) && order != relaxed => lse2_rcpc3 (stilp)\n atomic_store_lse2_rcpc3_release\n } else if cpuinfo.lse128() {\n // if detect(FEAT_LSE2) && detect(FEAT_LSE128) && order != relaxed => lse128 (swpp)\n atomic_store_lse128_release\n } else {\n // if detect(FEAT_LSE2) => lse2 (stp)\n atomic_store_lse2_release\n }\n } else {\n // else => no_lse2:\n atomic_store_no_lse2_release\n }\n });\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(dst: *mut u128, val: u128) {\n let cpuinfo = detect::detect();\n if cpuinfo.lse2() {\n if cpuinfo.lse128() {\n // if detect(FEAT_LSE2) && detect(FEAT_LSE128) && order == seqcst => lse128 (swpp)\n atomic_store_lse128_seqcst\n } else if cpuinfo.rcpc3() {\n // if detect(FEAT_LSE2) && detect(FEAT_LRCPC3) && order != relaxed => lse2_rcpc3 (stilp)\n atomic_store_lse2_rcpc3_seqcst\n } else {\n // if detect(FEAT_LSE2) => lse2 (stp)\n atomic_store_lse2_seqcst\n }\n } else {\n // else => no_lse2:\n atomic_store_no_lse2_seqcst\n }\n });\n }\n _ => unreachable!(),\n }\n }\n }\n }\n // else => no_lse2:\n #[cfg(else)]\n {\n use self::atomic_store_no_lse2 as atomic_store;\n }\n});\n#[cfg(not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")))]\ncfg_sel!({\n // if compile_time(FEAT_LSE) && not(ll_sc_rmw) => casp\n // If FEAT_LSE is available at compile-time and portable_atomic_ll_sc_rmw cfg is not set,\n // we use CAS-based atomic RMW.\n #[cfg(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n not(portable_atomic_ll_sc_rmw),\n ))]\n {\n #[inline]\n unsafe fn atomic_store_no_lse2(dst: *mut u128, val: u128, order: Ordering) {\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantee that the CPU supports FEAT_LSE.\n unsafe {\n _atomic_swap_casp(dst, val, order);\n }\n }\n }\n // else => ldxp_stxp\n #[cfg(else)]\n {\n use self::_atomic_store_ldxp_stxp as atomic_store_no_lse2;\n }\n});\n// If CPU supports FEAT_LSE2, STP/STILP is single-copy atomic writes,\n// otherwise it is two single-copy atomic writes.\n// Refs: B2.2.1 of the Arm Architecture Reference Manual Armv8, for Armv8-A architecture profile\n#[cfg(any(\n target_feature = \"lse2\",\n portable_atomic_target_feature = \"lse2\",\n not(portable_atomic_no_outline_atomics),\n))]\n#[inline]\nunsafe fn _atomic_store_stp(dst: *mut u128, val: u128, order: Ordering) {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_lse2!();\n\n // SAFETY: the caller must guarantee that `dst` is valid for writes,\n // 16-byte aligned, that there are no concurrent non-atomic operations.\n //\n // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STP--Store-pair-of-registers-\n unsafe {\n macro_rules! atomic_store {\n ($acquire:tt, $release:tt) => {{\n let val = U128 { whole: val };\n asm!(\n $release, // fence\n \"stp {val_lo}, {val_hi}, [{dst}]\", // atomic { *dst = val_lo:val_hi }\n $acquire, // fence\n dst = in(reg) ptr_reg!(dst),\n val_lo = in(reg) val.pair.lo,\n val_hi = in(reg) val.pair.hi,\n options(nostack, preserves_flags),\n );\n }};\n }\n match order {\n // if compile_time(FEAT_LSE128) && order == seqcst => swpp\n // Prefer swpp if stp requires fences. https://reviews.llvm.org/D143506\n // SAFETY: cfg guarantee that the CPU supports FEAT_LSE128.\n #[cfg(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\"))]\n Ordering::SeqCst => {\n _atomic_swap_swpp(dst, val, order);\n }\n\n // if compile_time(FEAT_LRCPC3) && order != relaxed => stilp\n // SAFETY: cfg guarantee that the CPU supports FEAT_LRCPC3.\n #[cfg(any(target_feature = \"rcpc3\", portable_atomic_target_feature = \"rcpc3\"))]\n Ordering::Release => _atomic_store_stilp(dst, val, order),\n #[cfg(any(target_feature = \"rcpc3\", portable_atomic_target_feature = \"rcpc3\"))]\n #[cfg(not(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\")))]\n Ordering::SeqCst => _atomic_store_stilp(dst, val, order),\n\n // if compile_time(FEAT_LSE128) && order != relaxed => swpp\n // Prefer swpp if stp requires fences. https://reviews.llvm.org/D143506\n // SAFETY: cfg guarantee that the CPU supports FEAT_LSE128.\n #[cfg(not(any(target_feature = \"rcpc3\", portable_atomic_target_feature = \"rcpc3\")))]\n #[cfg(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\"))]\n Ordering::Release => {\n _atomic_swap_swpp(dst, val, order);\n }\n\n // else => stp\n Ordering::Relaxed => atomic_store!(\"\", \"\"),\n #[cfg(not(any(target_feature = \"rcpc3\", portable_atomic_target_feature = \"rcpc3\")))]\n #[cfg(not(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\")))]\n Ordering::Release => atomic_store!(\"\", \"dmb ish\"),\n #[cfg(not(any(target_feature = \"rcpc3\", portable_atomic_target_feature = \"rcpc3\")))]\n #[cfg(not(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\")))]\n Ordering::SeqCst => atomic_store!(\"dmb ish\", \"dmb ish\"),\n _ => unreachable!(),\n }\n }\n}\n#[cfg(any(\n target_feature = \"lse2\",\n portable_atomic_target_feature = \"lse2\",\n not(portable_atomic_no_outline_atomics),\n))]\n#[cfg(any(\n target_feature = \"rcpc3\",\n portable_atomic_target_feature = \"rcpc3\",\n all(\n not(portable_atomic_no_outline_atomics),\n not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")),\n ),\n))]\n#[inline]\nunsafe fn _atomic_store_stilp(dst: *mut u128, val: u128, order: Ordering) {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_lse2!();\n debug_assert_rcpc3!();\n\n // SAFETY: the caller must guarantee that `dst` is valid for writes,\n // 16-byte aligned, that there are no concurrent non-atomic operations.\n //\n // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STILP--Store-release-ordered-pair-of-registers-\n unsafe {\n macro_rules! atomic_store {\n ($acquire:tt) => {{\n let val = U128 { whole: val };\n #[cfg(not(portable_atomic_pre_llvm_16))]\n asm!(\n start_rcpc3!(),\n \"stilp {val_lo}, {val_hi}, [{dst}]\", // atomic { *dst = val_lo:val_hi }\n $acquire, // fence\n dst = in(reg) ptr_reg!(dst),\n val_lo = in(reg) val.pair.lo,\n val_hi = in(reg) val.pair.hi,\n options(nostack, preserves_flags),\n );\n // LLVM supports FEAT_LRCPC3 instructions on LLVM 16+, so use .inst directive on old LLVM.\n // https://github.com/llvm/llvm-project/commit/a6aaa969f7caec58a994142f8d855861cf3a1463\n #[cfg(portable_atomic_pre_llvm_16)]\n asm!(\n // stilp x2, x3, [x0] // atomic { *dst = x2:x3 }\n \".inst 0xd9031802\",\n $acquire, // fence\n in(\"x0\") ptr_reg!(dst),\n in(\"x2\") val.pair.lo,\n in(\"x3\") val.pair.hi,\n options(nostack, preserves_flags),\n );\n }};\n }\n match order {\n Ordering::Release => atomic_store!(\"\"),\n // TODO: in atomicsabi64, seqcst store is the same as release store\n // LLVM uses store-release (dmb ish; stp); dmb ish, GCC (libatomic) and Atomics ABI Standard\n // uses store-release (stilp) without fence for SeqCst store\n // (https://github.com/gcc-mirror/gcc/commit/7107574958e2bed11d916a1480ef1319f15e5ffe).\n // Considering https://reviews.llvm.org/D141748, LLVM's lowering seems\n // to be the safer option here (I'm not convinced that the libatomic's implementation is wrong).\n Ordering::SeqCst => atomic_store!(\"dmb ish\"),\n _ => unreachable!(),\n }\n }\n}\n// Do not use _atomic_swap_ldxp_stxp because it needs extra registers to implement store.\n#[cfg(any(\n test,\n not(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n not(portable_atomic_ll_sc_rmw),\n ))\n))]\n#[inline]\nunsafe fn _atomic_store_ldxp_stxp(dst: *mut u128, val: u128, order: Ordering) {\n debug_assert!(dst.addr() % 16 == 0);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n let val = U128 { whole: val };\n macro_rules! store {\n ($acquire:tt, $release:tt, $msvc_fence:tt) => {\n asm!(\n \"2:\", // 'retry:\n concat!(\"ld\", $acquire, \"xp xzr, {tmp}, [{dst}]\"), // atomic { xzr:tmp = *dst; EXCLUSIVE = dst }\n concat!(\"st\", $release, \"xp {tmp:w}, {val_lo}, {val_hi}, [{dst}]\"), // atomic { if EXCLUSIVE == dst { *dst = val_lo:val_hi; tmp = 0 } else { tmp = 1 }; EXCLUSIVE = None }\n \"cbnz {tmp:w}, 2b\", // if tmp != 0 { jump 'retry }\n $msvc_fence, // fence\n dst = in(reg) ptr_reg!(dst),\n val_lo = in(reg) val.pair.lo,\n val_hi = in(reg) val.pair.hi,\n tmp = out(reg) _,\n options(nostack, preserves_flags),\n )\n };\n }\n atomic_rmw!(store, order);\n }\n}\n\n// -----------------------------------------------------------------------------\n// compare_exchange\n\n/*\n\nInstruction selection flow for compare_exchange:\n- if compile_time(FEAT_LSE) => casp\n- if platform_supports_detection_of(FEAT_LSE):\n - if detect(FEAT_LSE) => casp\n- else => ldxp_stxp\n\n*/\n\n#[inline]\nunsafe fn atomic_compare_exchange(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n) -> Result {\n // if compile_time(FEAT_LSE) => casp\n #[cfg(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"))]\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantee that the CPU supports FEAT_LSE.\n let prev = unsafe { _atomic_compare_exchange_casp(dst, old, new, success, failure) };\n // if platform_supports_detection_of(FEAT_LSE):\n #[cfg(all(\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n target_env = \"gnu\",\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n target_os = \"freebsd\",\n target_os = \"netbsd\",\n target_os = \"openbsd\",\n target_os = \"illumos\",\n target_os = \"fuchsia\",\n target_vendor = \"apple\",\n windows,\n ),\n ))]\n #[cfg(not(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\")))]\n let prev = {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(dst: *mut u128, old: u128, new: u128) -> u128;\n atomic_compare_exchange_casp_relaxed\n = _atomic_compare_exchange_casp(Ordering::Relaxed, Ordering::Relaxed);\n atomic_compare_exchange_casp_acquire\n = _atomic_compare_exchange_casp(Ordering::Acquire, Ordering::Acquire);\n atomic_compare_exchange_casp_release\n = _atomic_compare_exchange_casp(Ordering::Release, Ordering::Relaxed);\n atomic_compare_exchange_casp_acqrel\n = _atomic_compare_exchange_casp(Ordering::AcqRel, Ordering::Acquire);\n }\n fn_alias! {\n unsafe fn(dst: *mut u128, old: u128, new: u128) -> u128;\n atomic_compare_exchange_ldxp_stxp_relaxed\n = _atomic_compare_exchange_ldxp_stxp(Ordering::Relaxed, Ordering::Relaxed);\n atomic_compare_exchange_ldxp_stxp_acquire\n = _atomic_compare_exchange_ldxp_stxp(Ordering::Acquire, Ordering::Acquire);\n atomic_compare_exchange_ldxp_stxp_release\n = _atomic_compare_exchange_ldxp_stxp(Ordering::Release, Ordering::Relaxed);\n atomic_compare_exchange_ldxp_stxp_acqrel\n = _atomic_compare_exchange_ldxp_stxp(Ordering::AcqRel, Ordering::Acquire);\n // AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.\n #[cfg(target_env = \"msvc\")]\n atomic_compare_exchange_ldxp_stxp_seqcst\n = _atomic_compare_exchange_ldxp_stxp(Ordering::SeqCst, Ordering::SeqCst);\n }\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, that there are no concurrent non-atomic operations,\n // and we've checked if FEAT_LSE is available.\n unsafe {\n let success = crate::utils::upgrade_success_ordering(success, failure);\n match success {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> u128 {\n if detect::detect().lse() {\n // if detect(FEAT_LSE) => casp\n atomic_compare_exchange_casp_relaxed\n } else {\n // else => ldxp_stxp\n atomic_compare_exchange_ldxp_stxp_relaxed\n }\n })\n }\n Ordering::Acquire => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> u128 {\n if detect::detect().lse() {\n // if detect(FEAT_LSE) => casp\n atomic_compare_exchange_casp_acquire\n } else {\n // else => ldxp_stxp\n atomic_compare_exchange_ldxp_stxp_acquire\n }\n })\n }\n Ordering::Release => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> u128 {\n if detect::detect().lse() {\n // if detect(FEAT_LSE) => casp\n atomic_compare_exchange_casp_release\n } else {\n // else => ldxp_stxp\n atomic_compare_exchange_ldxp_stxp_release\n }\n })\n }\n // AcqRel and SeqCst RMWs are equivalent in both implementations in non-MSVC environments\n // or LSE* instructions is used.\n #[cfg(not(target_env = \"msvc\"))]\n Ordering::AcqRel | Ordering::SeqCst => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> u128 {\n if detect::detect().lse() {\n // if detect(FEAT_LSE) => casp\n atomic_compare_exchange_casp_acqrel\n } else {\n // else => ldxp_stxp\n atomic_compare_exchange_ldxp_stxp_acqrel\n }\n })\n }\n #[cfg(target_env = \"msvc\")]\n Ordering::AcqRel => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> u128 {\n if detect::detect().lse() {\n // if detect(FEAT_LSE) => casp\n atomic_compare_exchange_casp_acqrel\n } else {\n // else => ldxp_stxp\n atomic_compare_exchange_ldxp_stxp_acqrel\n }\n })\n }\n #[cfg(target_env = \"msvc\")]\n Ordering::SeqCst => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> u128 {\n if detect::detect().lse() {\n // if detect(FEAT_LSE) => casp\n atomic_compare_exchange_casp_acqrel\n } else {\n // else => ldxp_stxp\n atomic_compare_exchange_ldxp_stxp_seqcst\n }\n })\n }\n _ => unreachable!(),\n }\n }\n };\n // else => ldxp_stxp\n #[cfg(not(all(\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n target_env = \"gnu\",\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n target_os = \"freebsd\",\n target_os = \"netbsd\",\n target_os = \"openbsd\",\n target_os = \"illumos\",\n target_os = \"fuchsia\",\n target_vendor = \"apple\",\n windows,\n ),\n )))]\n #[cfg(not(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\")))]\n // SAFETY: the caller must uphold the safety contract.\n let prev = unsafe { _atomic_compare_exchange_ldxp_stxp(dst, old, new, success, failure) };\n if prev == old { Ok(prev) } else { Err(prev) }\n}\n#[cfg(any(\n target_feature = \"lse\",\n portable_atomic_target_feature = \"lse\",\n not(portable_atomic_no_outline_atomics),\n))]\n#[inline]\nunsafe fn _atomic_compare_exchange_casp(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_lse!();\n let order = crate::utils::upgrade_success_ordering(success, failure);\n\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, that there are no concurrent non-atomic operations,\n // and the CPU supports FEAT_LSE.\n //\n // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/CASP--CASPA--CASPAL--CASPL--Compare-and-swap-pair-of-words-or-doublewords-in-memory-\n unsafe {\n let old = U128 { whole: old };\n let new = U128 { whole: new };\n let (prev_lo, prev_hi);\n macro_rules! cmpxchg {\n ($acquire:tt, $release:tt, $_msvc_fence:tt) => {\n asm!(\n start_lse!(),\n concat!(\"casp\", $acquire, $release, \" x6, x7, x4, x5, [{dst}]\"), // atomic { if *dst == x6:x7 { *dst = x4:x5 } else { x6:x7 = *dst } }\n dst = in(reg) ptr_reg!(dst),\n // must be allocated to even/odd register pair\n inout(\"x6\") old.pair.lo => prev_lo,\n inout(\"x7\") old.pair.hi => prev_hi,\n // must be allocated to even/odd register pair\n in(\"x4\") new.pair.lo,\n in(\"x5\") new.pair.hi,\n options(nostack, preserves_flags),\n )\n };\n }\n atomic_rmw!(cmpxchg, order, write = success);\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n}\n#[cfg(any(test, not(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"))))]\n#[inline]\nunsafe fn _atomic_compare_exchange_ldxp_stxp(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n let order = crate::utils::upgrade_success_ordering(success, failure);\n\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.\n //\n // Refs:\n // - LDXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDXP--Load-exclusive-pair-of-registers-\n // - LDAXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDAXP--Load-acquire-exclusive-pair-of-registers-\n // - STXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STXP--Store-exclusive-pair-of-registers-\n // - STLXP: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/STLXP--Store-release-exclusive-pair-of-registers-\n //\n // Note: Load-Exclusive pair (by itself) does not guarantee atomicity; to complete an atomic\n // operation (even load/store), a corresponding Store-Exclusive pair must succeed.\n // See Arm Architecture Reference Manual for A-profile architecture\n // Section B2.2.1 \"Requirements for single-copy atomicity\", and\n // Section B2.9 \"Synchronization and semaphores\" for more.\n unsafe {\n let old = U128 { whole: old };\n let new = U128 { whole: new };\n let (mut prev_lo, mut prev_hi);\n macro_rules! cmpxchg {\n ($acquire:tt, $release:tt, $msvc_fence:tt) => {\n asm!(\n \"2:\", // 'retry:\n concat!(\"ld\", $acquire, \"xp {prev_lo}, {prev_hi}, [{dst}]\"), // atomic { prev_lo:prev_hi = *dst; EXCLUSIVE = dst }\n \"cmp {prev_lo}, {old_lo}\", // if prev_lo == old_lo { Z = 1 } else { Z = 0 }\n \"ccmp {prev_hi}, {old_hi}, #0, eq\", // if Z == 1 { if prev_hi == old_hi { Z = 1 } else { Z = 0 } } else { Z = 0 }\n // write back to ensure atomicity\n \"csel {tmp_lo}, {new_lo}, {prev_lo}, eq\", // if Z == 1 { tmp_lo = new_lo } else { tmp_lo = prev_lo }\n \"csel {tmp_hi}, {new_hi}, {prev_hi}, eq\", // if Z == 1 { tmp_hi = new_hi } else { tmp_hi = prev_hi }\n concat!(\"st\", $release, \"xp {r:w}, {tmp_lo}, {tmp_hi}, [{dst}]\"), // atomic { if EXCLUSIVE == dst { *dst = tmp_lo:tmp_hi; r = 0 } else { r = 1 }; EXCLUSIVE = None }\n \"cbnz {r:w}, 2b\", // if r != 0 { jump 'retry }\n $msvc_fence, // fence\n dst = in(reg) ptr_reg!(dst),\n old_lo = in(reg) old.pair.lo,\n old_hi = in(reg) old.pair.hi,\n new_lo = in(reg) new.pair.lo,\n new_hi = in(reg) new.pair.hi,\n prev_lo = out(reg) prev_lo,\n prev_hi = out(reg) prev_hi,\n r = out(reg) _,\n tmp_lo = out(reg) _,\n tmp_hi = out(reg) _,\n // Do not use `preserves_flags` because CMP and CCMP modify the condition flags.\n options(nostack),\n )\n };\n }\n atomic_rmw!(cmpxchg, order, write = success);\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n}\n\n// casp is always strong, and ldxp requires a corresponding (succeed) stxp for\n// its atomicity (see code comment in _atomic_compare_exchange_ldxp_stxp).\n// (i.e., AArch64 doesn't have 128-bit weak CAS)\nuse self::atomic_compare_exchange as atomic_compare_exchange_weak;\n\n// -----------------------------------------------------------------------------\n// RMW\n\n/*\n\nInstruction selection flow for swap/fetch_and/fetch_or:\n- if compile_time(FEAT_LSE128) => swpp/ldclrp/ldsetp\n- if compile_time(FEAT_LSE) && not(ll_sc_rmw) => casp\n- else => ldxp_stxp\n\nInstruction selection flow for other RMWs:\n- if compile_time(FEAT_LSE) && not(ll_sc_rmw) => casp\n- else => ldxp_stxp\n\nNote:\n- We don't do run-time detection of FEAT_LSE128 at this time, because\n FEAT_LSE128 is not yet available for most CPUs, but since\n swpp/ldclrp/ldsetp is wait-free, it would make sense to do run-time\n detection in the future. (TODO)\n- We don't do run-time detection of FEAT_LSE for store at this time.\n\n*/\n\ncfg_sel!({\n #[cfg(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\"))]\n {\n use self::_atomic_swap_swpp as atomic_swap;\n }\n #[cfg(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n not(portable_atomic_ll_sc_rmw),\n ))]\n {\n use self::_atomic_swap_casp as atomic_swap;\n }\n #[cfg(else)]\n {\n use self::_atomic_swap_ldxp_stxp as atomic_swap;\n }\n});\n#[cfg(any(\n target_feature = \"lse128\",\n portable_atomic_target_feature = \"lse128\",\n all(\n not(portable_atomic_no_outline_atomics),\n not(any(target_feature = \"lse2\", portable_atomic_target_feature = \"lse2\")),\n ),\n))]\n#[inline]\nunsafe fn _atomic_swap_swpp(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_lse128!();\n\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, that there are no concurrent non-atomic operations,\n // and the CPU supports FEAT_LSE128.\n //\n // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/SWPP--SWPPA--SWPPAL--SWPPL--Swap-quadword-in-memory-\n unsafe {\n let val = U128 { whole: val };\n let (prev_lo, prev_hi);\n #[cfg(not(portable_atomic_pre_llvm_16))]\n macro_rules! swap {\n ($acquire:tt, $release:tt, $_msvc_fence:tt) => {\n asm!(\n start_lse128!(),\n concat!(\"swpp\", $acquire, $release, \" {val_lo}, {val_hi}, [{dst}]\"), // atomic { _x = *dst; *dst = val_lo:val_hi; val_lo:val_hi = _x }\n dst = in(reg) ptr_reg!(dst),\n val_lo = inout(reg) val.pair.lo => prev_lo,\n val_hi = inout(reg) val.pair.hi => prev_hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(not(portable_atomic_pre_llvm_16))]\n atomic_rmw!(swap, order);\n // LLVM supports FEAT_LSE128 instructions on LLVM 16+, so use .inst directive on old LLVM.\n // https://github.com/llvm/llvm-project/commit/7fea6f2e0e606e5339c3359568f680eaf64aa306\n #[cfg(portable_atomic_pre_llvm_16)]\n macro_rules! swap {\n ($order:tt, $_msvc_fence:tt) => {\n asm!(\n // swpp{,a,l,al} x2, x1, [x0] // atomic { _x = *x0; *x0 = x2:x1; x2:x1 = _x }\n concat!(\".inst 0x19\", $order, \"18002\"),\n in(\"x0\") ptr_reg!(dst),\n inout(\"x2\") val.pair.lo => prev_lo,\n inout(\"x1\") val.pair.hi => prev_hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(portable_atomic_pre_llvm_16)]\n atomic_rmw_inst!(swap, order);\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n}\n// Do not use atomic_rmw_cas_3 because it needs extra MOV to implement swap.\n#[cfg(any(test, not(portable_atomic_ll_sc_rmw)))]\n#[cfg(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"))]\n#[inline]\nunsafe fn _atomic_swap_casp(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_lse!();\n\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantee that the CPU supports FEAT_LSE.\n unsafe {\n let val = U128 { whole: val };\n let (mut prev_lo, mut prev_hi);\n macro_rules! swap {\n ($acquire:tt, $release:tt, $_msvc_fence:tt) => {\n asm!(\n start_lse!(),\n // If FEAT_LSE2 is not supported, this works like byte-wise atomic.\n // This is not single-copy atomic reads, but this is ok because subsequent\n // CAS will check for consistency.\n \"ldp x4, x5, [{dst}]\", // atomic { x4:x5 = *dst }\n \"2:\", // 'retry:\n // casp writes the current value to the first register pair,\n // so copy the `out`'s value for later comparison.\n \"mov {tmp_lo}, x4\", // tmp_lo = x4\n \"mov {tmp_hi}, x5\", // tmp_hi = x5\n concat!(\"casp\", $acquire, $release, \" x4, x5, x2, x3, [{dst}]\"), // atomic { if *dst == x4:x5 { *dst = x2:x3 } else { x4:x5 = *dst } }\n \"cmp {tmp_hi}, x5\", // if tmp_hi == x5 { Z = 1 } else { Z = 0 }\n \"ccmp {tmp_lo}, x4, #0, eq\", // if Z == 1 { if tmp_lo == x4 { Z = 1 } else { Z = 0 } } else { Z = 0 }\n \"b.ne 2b\", // if Z == 0 { jump 'retry }\n dst = in(reg) ptr_reg!(dst),\n tmp_lo = out(reg) _,\n tmp_hi = out(reg) _,\n // must be allocated to even/odd register pair\n out(\"x4\") prev_lo,\n out(\"x5\") prev_hi,\n // must be allocated to even/odd register pair\n in(\"x2\") val.pair.lo,\n in(\"x3\") val.pair.hi,\n // Do not use `preserves_flags` because CMP and CCMP modify the condition flags.\n options(nostack),\n )\n };\n }\n atomic_rmw!(swap, order);\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n}\n// Do not use atomic_rmw_ll_sc_3 because it needs extra MOV to implement swap.\n#[cfg(any(\n test,\n not(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n not(portable_atomic_ll_sc_rmw),\n ))\n))]\n#[inline]\nunsafe fn _atomic_swap_ldxp_stxp(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n let val = U128 { whole: val };\n let (mut prev_lo, mut prev_hi);\n macro_rules! swap {\n ($acquire:tt, $release:tt, $msvc_fence:tt) => {\n asm!(\n \"2:\", // 'retry:\n concat!(\"ld\", $acquire, \"xp {prev_lo}, {prev_hi}, [{dst}]\"), // atomic { prev_lo:prev_hi = *dst; EXCLUSIVE = dst }\n concat!(\"st\", $release, \"xp {r:w}, {val_lo}, {val_hi}, [{dst}]\"), // atomic { if EXCLUSIVE == dst { *dst = val_lo:val_hi; r = 0 } else { r = 1 }; EXCLUSIVE = None }\n \"cbnz {r:w}, 2b\", // if r != 0 { jump 'retry }\n $msvc_fence, // fence\n dst = in(reg) ptr_reg!(dst),\n val_lo = in(reg) val.pair.lo,\n val_hi = in(reg) val.pair.hi,\n prev_lo = out(reg) prev_lo,\n prev_hi = out(reg) prev_hi,\n r = out(reg) _,\n options(nostack, preserves_flags),\n )\n };\n }\n atomic_rmw!(swap, order);\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n}\n\n/// Atomic RMW by LL/SC loop (3 arguments)\n/// `unsafe fn(dst: *mut u128, val: u128, order: Ordering) -> u128;`\n///\n/// `$op` can use the following registers:\n/// - val_lo/val_hi pair: val argument (read-only for `$op`)\n/// - prev_lo/prev_hi pair: previous value loaded by ll (read-only for `$op`)\n/// - new_lo/new_hi pair: new value that will be stored by sc\nmacro_rules! atomic_rmw_ll_sc_3 {\n ($name:ident as $reexport_name:ident $(($preserves_flags:tt))?, $($op:tt)*) => {\n // If FEAT_LSE is available at compile-time and portable_atomic_ll_sc_rmw cfg is not set,\n // we use CAS-based atomic RMW generated by atomic_rmw_cas_3! macro instead.\n #[cfg(not(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n not(portable_atomic_ll_sc_rmw),\n )))]\n use self::$name as $reexport_name;\n #[cfg(any(\n test,\n not(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n not(portable_atomic_ll_sc_rmw),\n ))\n ))]\n #[inline]\n unsafe fn $name(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n let val = U128 { whole: val };\n let (mut prev_lo, mut prev_hi);\n macro_rules! op {\n ($acquire:tt, $release:tt, $msvc_fence:tt) => {\n asm!(\n \"2:\", // 'retry:\n concat!(\"ld\", $acquire, \"xp {prev_lo}, {prev_hi}, [{dst}]\"), // atomic { prev_lo:prev_hi = *dst; EXCLUSIVE = dst }\n $($op)*\n concat!(\"st\", $release, \"xp {r:w}, {new_lo}, {new_hi}, [{dst}]\"), // atomic { if EXCLUSIVE == dst { *dst = new_lo:new_hi; r = 0 } else { r = 1 }; EXCLUSIVE = None }\n \"cbnz {r:w}, 2b\", // if r != 0 { jump 'retry }\n $msvc_fence, // fence\n dst = in(reg) ptr_reg!(dst),\n val_lo = in(reg) val.pair.lo,\n val_hi = in(reg) val.pair.hi,\n prev_lo = out(reg) prev_lo,\n prev_hi = out(reg) prev_hi,\n new_lo = out(reg) _,\n new_hi = out(reg) _,\n r = out(reg) _,\n options(nostack $(, $preserves_flags)?),\n )\n };\n }\n atomic_rmw!(op, order);\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n }\n };\n}\n/// Atomic RMW by CAS loop (3 arguments)\n/// `unsafe fn(dst: *mut u128, val: u128, order: Ordering) -> u128;`\n///\n/// `$op` can use the following registers:\n/// - val_lo/val_hi pair: val argument (read-only for `$op`)\n/// - x6/x7 pair: previous value loaded (read-only for `$op`)\n/// - x4/x5 pair: new value that will be stored\nmacro_rules! atomic_rmw_cas_3 {\n ($name:ident as $reexport_name:ident, $($op:tt)*) => {\n // If FEAT_LSE is not available at compile-time or portable_atomic_ll_sc_rmw cfg is set,\n // we use LL/SC-based atomic RMW generated by atomic_rmw_ll_sc_3! macro instead.\n #[cfg(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n not(portable_atomic_ll_sc_rmw),\n ))]\n use self::$name as $reexport_name;\n #[cfg(any(test, not(portable_atomic_ll_sc_rmw)))]\n #[cfg(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"))]\n #[inline]\n unsafe fn $name(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_lse!();\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantee that the CPU supports FEAT_LSE.\n unsafe {\n let val = U128 { whole: val };\n let (mut prev_lo, mut prev_hi);\n macro_rules! op {\n ($acquire:tt, $release:tt, $_msvc_fence:tt) => {\n asm!(\n start_lse!(),\n // If FEAT_LSE2 is not supported, this works like byte-wise atomic.\n // This is not single-copy atomic reads, but this is ok because subsequent\n // CAS will check for consistency.\n \"ldp x6, x7, [{dst}]\", // atomic { x6:x7 = *dst }\n \"2:\", // 'retry:\n // casp writes the current value to the first register pair,\n // so copy the `out`'s value for later comparison.\n \"mov {tmp_lo}, x6\", // tmp_lo = x6\n \"mov {tmp_hi}, x7\", // tmp_hi = x7\n $($op)*\n concat!(\"casp\", $acquire, $release, \" x6, x7, x4, x5, [{dst}]\"), // atomic { if *dst == x6:x7 { *dst = x4:x5 } else { x6:x7 = *dst } }\n \"cmp {tmp_hi}, x7\", // if tmp_hi == x7 { Z = 1 } else { Z = 0 }\n \"ccmp {tmp_lo}, x6, #0, eq\", // if Z == 1 { if tmp_lo == x6 { Z = 1 } else { Z = 0 } } else { Z = 0 }\n \"b.ne 2b\", // if Z == 0 { jump 'retry }\n dst = in(reg) ptr_reg!(dst),\n val_lo = in(reg) val.pair.lo,\n val_hi = in(reg) val.pair.hi,\n tmp_lo = out(reg) _,\n tmp_hi = out(reg) _,\n // must be allocated to even/odd register pair\n out(\"x6\") prev_lo,\n out(\"x7\") prev_hi,\n // must be allocated to even/odd register pair\n out(\"x4\") _,\n out(\"x5\") _,\n // Do not use `preserves_flags` because CMP and CCMP modify the condition flags.\n options(nostack),\n )\n };\n }\n atomic_rmw!(op, order);\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n }\n };\n}\n\n/// Atomic RMW by LL/SC loop (2 arguments)\n/// `unsafe fn(dst: *mut u128, order: Ordering) -> u128;`\n///\n/// `$op` can use the following registers:\n/// - prev_lo/prev_hi pair: previous value loaded by ll (read-only for `$op`)\n/// - new_lo/new_hi pair: new value that will be stored by sc\nmacro_rules! atomic_rmw_ll_sc_2 {\n ($name:ident as $reexport_name:ident $(($preserves_flags:tt))?, $($op:tt)*) => {\n // If FEAT_LSE is available at compile-time and portable_atomic_ll_sc_rmw cfg is not set,\n // we use CAS-based atomic RMW generated by atomic_rmw_cas_2! macro instead.\n #[cfg(not(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n not(portable_atomic_ll_sc_rmw),\n )))]\n use self::$name as $reexport_name;\n #[cfg(any(\n test,\n not(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n not(portable_atomic_ll_sc_rmw),\n ))\n ))]\n #[inline]\n unsafe fn $name(dst: *mut u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n let (mut prev_lo, mut prev_hi);\n macro_rules! op {\n ($acquire:tt, $release:tt, $msvc_fence:tt) => {\n asm!(\n \"2:\", // 'retry:\n concat!(\"ld\", $acquire, \"xp {prev_lo}, {prev_hi}, [{dst}]\"), // atomic { prev_lo:prev_hi = *dst; EXCLUSIVE = dst }\n $($op)*\n concat!(\"st\", $release, \"xp {r:w}, {new_lo}, {new_hi}, [{dst}]\"), // atomic { if EXCLUSIVE == dst { *dst = new_lo:new_hi; r = 0 } else { r = 1 }; EXCLUSIVE = None }\n \"cbnz {r:w}, 2b\", // if r != 0 { jump 'retry }\n $msvc_fence, // fence\n dst = in(reg) ptr_reg!(dst),\n prev_lo = out(reg) prev_lo,\n prev_hi = out(reg) prev_hi,\n new_lo = out(reg) _,\n new_hi = out(reg) _,\n r = out(reg) _,\n options(nostack $(, $preserves_flags)?),\n )\n };\n }\n atomic_rmw!(op, order);\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n }\n };\n}\n/// Atomic RMW by CAS loop (2 arguments)\n/// `unsafe fn(dst: *mut u128, order: Ordering) -> u128;`\n///\n/// `$op` can use the following registers:\n/// - x6/x7 pair: previous value loaded (read-only for `$op`)\n/// - x4/x5 pair: new value that will be stored\nmacro_rules! atomic_rmw_cas_2 {\n ($name:ident as $reexport_name:ident, $($op:tt)*) => {\n // If FEAT_LSE is not available at compile-time or portable_atomic_ll_sc_rmw cfg is set,\n // we use LL/SC-based atomic RMW generated by atomic_rmw_ll_sc_3! macro instead.\n #[cfg(all(\n any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"),\n not(portable_atomic_ll_sc_rmw),\n ))]\n use self::$name as $reexport_name;\n #[cfg(any(test, not(portable_atomic_ll_sc_rmw)))]\n #[cfg(any(target_feature = \"lse\", portable_atomic_target_feature = \"lse\"))]\n #[inline]\n unsafe fn $name(dst: *mut u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_lse!();\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantee that the CPU supports FEAT_LSE.\n unsafe {\n let (mut prev_lo, mut prev_hi);\n macro_rules! op {\n ($acquire:tt, $release:tt, $_msvc_fence:tt) => {\n asm!(\n start_lse!(),\n // If FEAT_LSE2 is not supported, this works like byte-wise atomic.\n // This is not single-copy atomic reads, but this is ok because subsequent\n // CAS will check for consistency.\n \"ldp x6, x7, [{dst}]\", // atomic { x6:x7 = *dst }\n \"2:\", // 'retry:\n // casp writes the current value to the first register pair,\n // so copy the `out`'s value for later comparison.\n \"mov {tmp_lo}, x6\", // tmp_lo = x6\n \"mov {tmp_hi}, x7\", // tmp_hi = x7\n $($op)*\n concat!(\"casp\", $acquire, $release, \" x6, x7, x4, x5, [{dst}]\"), // atomic { if *dst == x6:x7 { *dst = x4:x5 } else { x6:x7 = *dst } }\n \"cmp {tmp_hi}, x7\", // if tmp_hi == x7 { Z = 1 } else { Z = 0 }\n \"ccmp {tmp_lo}, x6, #0, eq\", // if Z == 1 { if tmp_lo == x6 { Z = 1 } else { Z = 0 } } else { Z = 0 }\n \"b.ne 2b\", // if Z == 0 { jump 'retry }\n dst = in(reg) ptr_reg!(dst),\n tmp_lo = out(reg) _,\n tmp_hi = out(reg) _,\n // must be allocated to even/odd register pair\n out(\"x6\") prev_lo,\n out(\"x7\") prev_hi,\n // must be allocated to even/odd register pair\n out(\"x4\") _,\n out(\"x5\") _,\n // Do not use `preserves_flags` because CMP and CCMP modify the condition flags.\n options(nostack),\n )\n };\n }\n atomic_rmw!(op, order);\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n }\n };\n}\n\n// Do not use `preserves_flags` because ADDS modifies the condition flags.\natomic_rmw_ll_sc_3! {\n _atomic_add_ldxp_stxp as atomic_add,\n select_le_or_be!(\"adds {new_lo}, {prev_lo}, {val_lo}\", \"adds {new_hi}, {prev_hi}, {val_hi}\"), // if endian == little { new_lo = prev_lo + val_lo } else { new_hi = prev_hi + val_hi }; C = carry\n select_le_or_be!(\"adc {new_hi}, {prev_hi}, {val_hi}\", \"adc {new_lo}, {prev_lo}, {val_lo}\"), // if endian == little { new_hi = prev_hi + val_hi + C } else { new_lo = prev_lo + val_lo + C }\n}\natomic_rmw_cas_3! {\n _atomic_add_casp as atomic_add,\n select_le_or_be!(\"adds x4, x6, {val_lo}\", \"adds x5, x7, {val_hi}\"), // if endian == little { x4 = x6 + val_lo } else { x5 = x7 + val_hi }; C = carry\n select_le_or_be!(\"adc x5, x7, {val_hi}\", \"adc x4, x6, {val_lo}\"), // if endian == little { x5 = x7 + val_hi + C } else { x4 = x6 + val_lo + C }\n}\n\n// Do not use `preserves_flags` because SUBS modifies the condition flags.\natomic_rmw_ll_sc_3! {\n _atomic_sub_ldxp_stxp as atomic_sub,\n select_le_or_be!(\"subs {new_lo}, {prev_lo}, {val_lo}\", \"subs {new_hi}, {prev_hi}, {val_hi}\"), // if endian == little { new_lo = prev_lo - val_lo } else { new_hi = prev_hi - val_hi }; C = !borrow\n select_le_or_be!(\"sbc {new_hi}, {prev_hi}, {val_hi}\", \"sbc {new_lo}, {prev_lo}, {val_lo}\"), // if endian == little { new_hi = prev_hi - val_hi - !C } else { new_lo = prev_lo - val_lo - !C }\n}\natomic_rmw_cas_3! {\n _atomic_sub_casp as atomic_sub,\n select_le_or_be!(\"subs x4, x6, {val_lo}\", \"subs x5, x7, {val_hi}\"), // if endian == little { x4 = x6 - val_lo } else { x5 = x7 - val_hi }; C = !borrow\n select_le_or_be!(\"sbc x5, x7, {val_hi}\", \"sbc x4, x6, {val_lo}\"), // if endian == little { x5 = x7 - val_hi - !C } else { x4 = x6 - val_lo - !C }\n}\n\n#[cfg(not(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\")))]\natomic_rmw_ll_sc_3! {\n _atomic_and_ldxp_stxp as atomic_and (preserves_flags),\n \"and {new_lo}, {prev_lo}, {val_lo}\", // new_lo = prev_lo & val_lo\n \"and {new_hi}, {prev_hi}, {val_hi}\", // new_hi = prev_hi & val_hi\n}\n#[cfg(not(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\")))]\natomic_rmw_cas_3! {\n _atomic_and_casp as atomic_and,\n \"and x4, x6, {val_lo}\", // x4 = x6 & val_lo\n \"and x5, x7, {val_hi}\", // x5 = x7 & val_hi\n}\n#[cfg(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\"))]\n#[inline]\nunsafe fn atomic_and(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, that there are no concurrent non-atomic operations,\n // and the CPU supports FEAT_LSE128.\n //\n // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDCLRP--LDCLRPA--LDCLRPAL--LDCLRPL--Atomic-bit-clear-on-quadword-in-memory-\n unsafe {\n let val = U128 { whole: !val };\n let (prev_lo, prev_hi);\n #[cfg(not(portable_atomic_pre_llvm_16))]\n macro_rules! clear {\n ($acquire:tt, $release:tt, $_msvc_fence:tt) => {\n asm!(\n start_lse128!(),\n concat!(\"ldclrp\", $acquire, $release, \" {val_lo}, {val_hi}, [{dst}]\"), // atomic { _x = *dst; *dst = _x & !val; val = _x }\n dst = in(reg) ptr_reg!(dst),\n val_lo = inout(reg) val.pair.lo => prev_lo,\n val_hi = inout(reg) val.pair.hi => prev_hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(not(portable_atomic_pre_llvm_16))]\n atomic_rmw!(clear, order);\n // LLVM supports FEAT_LSE128 instructions on LLVM 16+, so use .inst directive on old LLVM.\n // https://github.com/llvm/llvm-project/commit/7fea6f2e0e606e5339c3359568f680eaf64aa306\n #[cfg(portable_atomic_pre_llvm_16)]\n macro_rules! clear {\n ($order:tt, $_msvc_fence:tt) => {\n asm!(\n // ldclrp{,a,l,al} x8, x1, [x0] // atomic { _x = *x0; *x0 = _x & !x8:x1; x8:x1 = _x }\n concat!(\".inst 0x19\", $order, \"11008\"),\n in(\"x0\") ptr_reg!(dst),\n inout(\"x8\") val.pair.lo => prev_lo,\n inout(\"x1\") val.pair.hi => prev_hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(portable_atomic_pre_llvm_16)]\n atomic_rmw_inst!(clear, order);\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n}\n\natomic_rmw_ll_sc_3! {\n _atomic_nand_ldxp_stxp as atomic_nand (preserves_flags),\n \"and {new_lo}, {prev_lo}, {val_lo}\", // new_lo = prev_lo & val_lo\n \"and {new_hi}, {prev_hi}, {val_hi}\", // new_hi = prev_hi & val_hi\n \"mvn {new_lo}, {new_lo}\", // new_lo = !new_lo\n \"mvn {new_hi}, {new_hi}\", // new_hi = !new_hi\n}\natomic_rmw_cas_3! {\n _atomic_nand_casp as atomic_nand,\n \"and x4, x6, {val_lo}\", // x4 = x6 & val_lo\n \"and x5, x7, {val_hi}\", // x5 = x7 & val_hi\n \"mvn x4, x4\", // x4 = !x4\n \"mvn x5, x5\", // x5 = !x5\n}\n\n#[cfg(not(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\")))]\natomic_rmw_ll_sc_3! {\n _atomic_or_ldxp_stxp as atomic_or (preserves_flags),\n \"orr {new_lo}, {prev_lo}, {val_lo}\", // new_lo = prev_lo | val_lo\n \"orr {new_hi}, {prev_hi}, {val_hi}\", // new_hi = prev_hi | val_hi\n}\n#[cfg(not(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\")))]\natomic_rmw_cas_3! {\n _atomic_or_casp as atomic_or,\n \"orr x4, x6, {val_lo}\", // x4 = x6 | val_lo\n \"orr x5, x7, {val_hi}\", // x5 = x7 | val_hi\n}\n#[cfg(any(target_feature = \"lse128\", portable_atomic_target_feature = \"lse128\"))]\n#[inline]\nunsafe fn atomic_or(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, that there are no concurrent non-atomic operations,\n // and the CPU supports FEAT_LSE128.\n //\n // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/LDSETP--LDSETPA--LDSETPAL--LDSETPL--Atomic-bit-set-on-quadword-in-memory-\n unsafe {\n let val = U128 { whole: val };\n let (prev_lo, prev_hi);\n #[cfg(not(portable_atomic_pre_llvm_16))]\n macro_rules! or {\n ($acquire:tt, $release:tt, $_msvc_fence:tt) => {\n asm!(\n start_lse128!(),\n concat!(\"ldsetp\", $acquire, $release, \" {val_lo}, {val_hi}, [{dst}]\"), // atomic { _x = *dst; *dst = _x | val; val = _x }\n dst = in(reg) ptr_reg!(dst),\n val_lo = inout(reg) val.pair.lo => prev_lo,\n val_hi = inout(reg) val.pair.hi => prev_hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(not(portable_atomic_pre_llvm_16))]\n atomic_rmw!(or, order);\n // LLVM supports FEAT_LSE128 instructions on LLVM 16+, so use .inst directive on old LLVM.\n // https://github.com/llvm/llvm-project/commit/7fea6f2e0e606e5339c3359568f680eaf64aa306\n #[cfg(portable_atomic_pre_llvm_16)]\n macro_rules! or {\n ($order:tt, $_msvc_fence:tt) => {\n asm!(\n // ldsetp{,a,l,al} x2, x1, [x0] // atomic { _x = *x0; *x0 = _x | x2:x1; x2:x1 = _x }\n concat!(\".inst 0x19\", $order, \"13002\"),\n in(\"x0\") ptr_reg!(dst),\n inout(\"x2\") val.pair.lo => prev_lo,\n inout(\"x1\") val.pair.hi => prev_hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(portable_atomic_pre_llvm_16)]\n atomic_rmw_inst!(or, order);\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n}\n\natomic_rmw_ll_sc_3! {\n _atomic_xor_ldxp_stxp as atomic_xor (preserves_flags),\n \"eor {new_lo}, {prev_lo}, {val_lo}\", // new_lo = prev_lo ^ val_lo\n \"eor {new_hi}, {prev_hi}, {val_hi}\", // new_hi = prev_hi ^ val_hi\n}\natomic_rmw_cas_3! {\n _atomic_xor_casp as atomic_xor,\n \"eor x4, x6, {val_lo}\", // x4 = x6 ^ val_lo\n \"eor x5, x7, {val_hi}\", // x5 = x7 ^ val_hi\n}\n\natomic_rmw_ll_sc_2! {\n _atomic_not_ldxp_stxp as atomic_not (preserves_flags),\n \"mvn {new_lo}, {prev_lo}\", // new_lo = !prev_lo\n \"mvn {new_hi}, {prev_hi}\", // new_hi = !prev_hi\n}\natomic_rmw_cas_2! {\n _atomic_not_casp as atomic_not,\n \"mvn x4, x6\", // x4 = !x6\n \"mvn x5, x7\", // x5 = !x7\n}\n\n// Do not use `preserves_flags` because NEGS modifies the condition flags.\natomic_rmw_ll_sc_2! {\n _atomic_neg_ldxp_stxp as atomic_neg,\n select_le_or_be!(\"negs {new_lo}, {prev_lo}\", \"negs {new_hi}, {prev_hi}\"), // if endian == little { new_lo = 0 - prev_lo } else { new_hi = 0 - prev_hi }; C = !borrow\n select_le_or_be!(\"ngc {new_hi}, {prev_hi}\", \"ngc {new_lo}, {prev_lo}\"), // if endian == little { new_hi = 0 - prev_hi - !C } else { new_lo = 0 - prev_lo - !C }\n}\natomic_rmw_cas_2! {\n _atomic_neg_casp as atomic_neg,\n select_le_or_be!(\"negs x4, x6\", \"negs x5, x7\"), // if endian == little { x4 = 0 - x6 } else { x5 = 0 - x7 }; C = !borrow\n select_le_or_be!(\"ngc x5, x7\", \"ngc x4, x6\"), // if endian == little { x5 = 0 - x7 - !C } else { x4 = 0 - x6 - !C }\n}\n\n// Do not use `preserves_flags` because CMP and SBCS modify the condition flags.\natomic_rmw_ll_sc_3! {\n _atomic_max_ldxp_stxp as atomic_max,\n select_le_or_be!(\"cmp {val_lo}, {prev_lo}\", \"cmp {val_hi}, {prev_hi}\"), // if endian == little { C = !(val_lo < prev_lo) } else { C = !(val_hi < prev_hi) }\n select_le_or_be!(\"sbcs xzr, {val_hi}, {prev_hi}\", \"sbcs xzr, {val_lo}, {prev_lo}\"), // if endian == little { (_, NZCV) = val_hi - prev_hi - !C } else { (_, NZCV) = val_lo - prev_lo - !C }\n \"csel {new_hi}, {prev_hi}, {val_hi}, lt\", // if N != V { new_hi = prev_hi } else { new_hi = val_hi }\n \"csel {new_lo}, {prev_lo}, {val_lo}, lt\", // if N != V { new_lo = prev_lo } else { new_lo = val_lo }\n}\natomic_rmw_cas_3! {\n _atomic_max_casp as atomic_max,\n select_le_or_be!(\"cmp {val_lo}, x6\", \"cmp {val_hi}, x7\"), // if endian == little { C = !(val_lo < x6) } else { C = !(val_hi < x7) }\n select_le_or_be!(\"sbcs xzr, {val_hi}, x7\", \"sbcs xzr, {val_lo}, x6\"), // if endian == little { (_, NZCV) = val_hi - x7 - !C } else { (_, NZCV) = val_lo - x6 - !C }\n \"csel x5, x7, {val_hi}, lt\", // if N != V { x5 = x7 } else { x5 = val_hi }\n \"csel x4, x6, {val_lo}, lt\", // if N != V { x4 = x6 } else { x4 = val_lo }\n}\n\n// Do not use `preserves_flags` because CMP and SBCS modify the condition flags.\natomic_rmw_ll_sc_3! {\n _atomic_umax_ldxp_stxp as atomic_umax,\n select_le_or_be!(\"cmp {val_lo}, {prev_lo}\", \"cmp {val_hi}, {prev_hi}\"), // if endian == little { C = !(val_lo < prev_lo) } else { C = !(val_hi < prev_hi) }\n select_le_or_be!(\"sbcs xzr, {val_hi}, {prev_hi}\", \"sbcs xzr, {val_lo}, {prev_lo}\"), // if endian == little { (_, NZCV) = val_hi - prev_hi - !C } else { (_, NZCV) = val_lo - prev_lo - !C }\n \"csel {new_hi}, {prev_hi}, {val_hi}, lo\", // if C == 0 { new_hi = prev_hi } else { new_hi = val_hi }\n \"csel {new_lo}, {prev_lo}, {val_lo}, lo\", // if C == 0 { new_hi = prev_hi } else { new_hi = val_hi }\n}\natomic_rmw_cas_3! {\n _atomic_umax_casp as atomic_umax,\n select_le_or_be!(\"cmp {val_lo}, x6\", \"cmp {val_hi}, x7\"), // if endian == little { C = !(val_lo < x6) } else { C = !(val_hi < x7) }\n select_le_or_be!(\"sbcs xzr, {val_hi}, x7\", \"sbcs xzr, {val_lo}, x6\"), // if endian == little { (_, NZCV) = val_hi - x7 - !C } else { (_, NZCV) = val_lo - x6 - !C }\n \"csel x5, x7, {val_hi}, lo\", // if C == 0 { x5 = x7 } else { x5 = val_hi }\n \"csel x4, x6, {val_lo}, lo\", // if C == 0 { x4 = x6 } else { x4 = val_lo }\n}\n\n// Do not use `preserves_flags` because CMP and SBCS modify the condition flags.\natomic_rmw_ll_sc_3! {\n _atomic_min_ldxp_stxp as atomic_min,\n select_le_or_be!(\"cmp {val_lo}, {prev_lo}\", \"cmp {val_hi}, {prev_hi}\"), // if endian == little { C = !(val_lo < prev_lo) } else { C = !(val_hi < prev_hi) }\n select_le_or_be!(\"sbcs xzr, {val_hi}, {prev_hi}\", \"sbcs xzr, {val_lo}, {prev_lo}\"), // if endian == little { (_, NZCV) = val_hi - prev_hi - !C } else { (_, NZCV) = val_lo - prev_lo - !C }\n \"csel {new_hi}, {prev_hi}, {val_hi}, ge\", // if N == V { new_hi = prev_hi } else { new_hi = val_hi }\n \"csel {new_lo}, {prev_lo}, {val_lo}, ge\", // if N == V { new_lo = prev_lo } else { new_lo = val_lo }\n}\natomic_rmw_cas_3! {\n _atomic_min_casp as atomic_min,\n select_le_or_be!(\"cmp {val_lo}, x6\", \"cmp {val_hi}, x7\"), // if endian == little { C = !(val_lo < x6) } else { C = !(val_hi < x7) }\n select_le_or_be!(\"sbcs xzr, {val_hi}, x7\", \"sbcs xzr, {val_lo}, x6\"), // if endian == little { (_, NZCV) = val_hi - x7 - !C } else { (_, NZCV) = val_lo - x6 - !C }\n \"csel x5, x7, {val_hi}, ge\", // if N == V { x5 = x7 } else { x5 = val_hi }\n \"csel x4, x6, {val_lo}, ge\", // if N == V { x4 = x6 } else { x4 = val_lo }\n}\n\n// Do not use `preserves_flags` because CMP and SBCS modify the condition flags.\natomic_rmw_ll_sc_3! {\n _atomic_umin_ldxp_stxp as atomic_umin,\n select_le_or_be!(\"cmp {val_lo}, {prev_lo}\", \"cmp {val_hi}, {prev_hi}\"), // if endian == little { C = !(val_lo < prev_lo) } else { C = !(val_hi < prev_hi) }\n select_le_or_be!(\"sbcs xzr, {val_hi}, {prev_hi}\", \"sbcs xzr, {val_lo}, {prev_lo}\"), // if endian == little { (_, NZCV) = val_hi - prev_hi - !C } else { (_, NZCV) = val_lo - prev_lo - !C }\n \"csel {new_hi}, {prev_hi}, {val_hi}, hs\", // if C == 1 { new_hi = prev_hi } else { new_hi = val_hi }\n \"csel {new_lo}, {prev_lo}, {val_lo}, hs\", // if C == 1 { new_lo = prev_lo } else { new_lo = val_lo }\n}\natomic_rmw_cas_3! {\n _atomic_umin_casp as atomic_umin,\n select_le_or_be!(\"cmp {val_lo}, x6\", \"cmp {val_hi}, x7\"), // if endian == little { C = !(val_lo < x6) } else { C = !(val_hi < x7) }\n select_le_or_be!(\"sbcs xzr, {val_hi}, x7\", \"sbcs xzr, {val_lo}, x6\"), // if endian == little { (_, NZCV) = val_hi - x7 - !C } else { (_, NZCV) = val_lo - x6 - !C }\n \"csel x5, x7, {val_hi}, hs\", // if C == 1 { x5 = x7 } else { x5 = val_hi }\n \"csel x4, x6, {val_lo}, hs\", // if C == 1 { x4 = x6 } else { x4 = val_lo }\n}\n\n#[inline]\nconst fn is_lock_free() -> bool {\n IS_ALWAYS_LOCK_FREE\n}\nconst IS_ALWAYS_LOCK_FREE: bool = true;\n\natomic128!(AtomicI128, i128, atomic_max, atomic_min);\natomic128!(AtomicU128, u128, atomic_umax, atomic_umin);\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n test_atomic_int!(i128);\n test_atomic_int!(u128);\n\n // load/store/swap implementation is not affected by signedness, so it is\n // enough to test only unsigned types.\n stress_test!(u128);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic128/intrinsics.rs", "sha256": "882b817a77ba6b90ad96585bdd5545c8526a9590473a93ca8dda96d91d1fcbcb", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\n128-bit atomic implementation without inline assembly.\n\nAdapted from https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs.\n\nNote: This module is currently only enabled on Miri and ThreadSanitizer which\ndo not support inline assembly.\n\nThis uses `core::arch::x86_64::cmpxchg16b` on x86_64 and\n`core::intrinsics::atomic_*` on aarch64, powerpc64, and s390x.\n\nSee README.md of this directory for performance comparison with the\nimplementation with inline assembly.\n\nNote:\n- This currently needs Rust 1.70 on x86_64, otherwise nightly compilers.\n- On powerpc64, this requires LLVM 15+ and quadword-atomics target feature:\n https://github.com/llvm/llvm-project/commit/549e118e93c666914a1045fde38a2cac33e1e445\n- On s390x, old LLVM (pre-18) generates libcalls for operations other than load/store/cmpxchg:\n https://github.com/llvm/llvm-project/commit/c568927f3e2e7d9804ea74ecbf11c16c014ddcbc\n- On aarch64 big-endian, LLVM (as of 17) generates broken code. (wrong result in stress test)\n (on cfg(miri)/cfg(sanitize) it may be fine though)\n- On powerpc64, LLVM (as of 17) doesn't support 128-bit atomic min/max:\n https://github.com/llvm/llvm-project/issues/68390\n- On powerpc64le, LLVM (as of 17) generates broken code. (wrong result from fetch_add)\n- On riscv64, LLVM does not automatically use 128-bit atomic instructions even if zacas feature is\n enabled, because doing it changes the ABI. (If the ability to do that is provided by LLVM in the\n future, it should probably be controlled by another ABI feature similar to forced-atomics.)\n*/\n\ninclude!(\"macros.rs\");\n\n#[allow(dead_code)] // we only use compare_exchange.\n#[cfg(target_arch = \"x86_64\")]\n#[cfg(not(target_feature = \"cmpxchg16b\"))]\n#[path = \"../fallback/outline_atomics.rs\"]\nmod fallback;\n\n#[cfg(target_arch = \"x86_64\")]\n#[cfg(not(target_feature = \"cmpxchg16b\"))]\n#[path = \"../detect/x86_64.rs\"]\nmod detect;\n\n#[cfg(not(target_arch = \"x86_64\"))]\nuse core::intrinsics;\nuse core::sync::atomic::Ordering::{self, AcqRel, Acquire, Relaxed, Release, SeqCst};\n\n#[cfg(target_arch = \"x86_64\")]\n#[inline]\nfn strongest_failure_ordering(order: Ordering) -> Ordering {\n match order {\n Release | Relaxed => Relaxed,\n SeqCst => SeqCst,\n Acquire | AcqRel => Acquire,\n _ => unreachable!(),\n }\n}\n\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {\n #[cfg(target_arch = \"x86_64\")]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n let fail_order = strongest_failure_ordering(order);\n match atomic_compare_exchange(src, 0, 0, order, fail_order) {\n Ok(v) | Err(v) => v,\n }\n }\n #[cfg(not(target_arch = \"x86_64\"))]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_load_acquire(src),\n Relaxed => intrinsics::atomic_load_relaxed(src),\n SeqCst => intrinsics::atomic_load_seqcst(src),\n _ => unreachable!(),\n }\n }\n}\n\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {\n #[cfg(target_arch = \"x86_64\")]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n atomic_swap(dst, val, order);\n }\n #[cfg(not(target_arch = \"x86_64\"))]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Release => intrinsics::atomic_store_release(dst, val),\n Relaxed => intrinsics::atomic_store_relaxed(dst, val),\n SeqCst => intrinsics::atomic_store_seqcst(dst, val),\n _ => unreachable!(),\n }\n }\n}\n\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_compare_exchange(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n) -> Result {\n #[cfg(target_arch = \"x86_64\")]\n let (val, ok) = {\n #[target_feature(enable = \"cmpxchg16b\")]\n #[cfg_attr(target_feature = \"cmpxchg16b\", inline)]\n #[cfg_attr(not(target_feature = \"cmpxchg16b\"), inline(never))]\n unsafe fn cmpxchg16b(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n ) -> (u128, bool) {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n debug_assert!(dst.addr() % 16 == 0);\n #[cfg(not(target_feature = \"cmpxchg16b\"))]\n {\n debug_assert!(detect::detect().cmpxchg16b());\n }\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned (required by CMPXCHG16B), that there are no\n // concurrent non-atomic operations, and that the CPU supports CMPXCHG16B.\n let prev = unsafe { core::arch::x86_64::cmpxchg16b(dst, old, new, success, failure) };\n (prev, prev == old)\n }\n #[cfg(target_feature = \"cmpxchg16b\")]\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, that there are no concurrent non-atomic operations,\n // and cfg guarantees that CMPXCHG16B is available at compile-time.\n unsafe {\n cmpxchg16b(dst, old, new, success, failure)\n }\n #[cfg(not(target_feature = \"cmpxchg16b\"))]\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, and that there are no different kinds of concurrent accesses.\n unsafe {\n ifunc!(unsafe fn(\n dst: *mut u128, old: u128, new: u128, success: Ordering, failure: Ordering\n ) -> (u128, bool) {\n if detect::detect().cmpxchg16b() {\n cmpxchg16b\n } else {\n fallback::atomic_compare_exchange\n }\n })\n }\n };\n #[cfg(not(target_arch = \"x86_64\"))]\n // SAFETY: the caller must uphold the safety contract.\n let (val, ok) = unsafe {\n match (success, failure) {\n (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new),\n (Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new),\n (Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new),\n (Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new),\n (Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new),\n (Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new),\n (Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new),\n (Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new),\n (Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new),\n (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new),\n (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new),\n (AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new),\n (SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new),\n (SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new),\n (SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new),\n _ => unreachable!(),\n }\n };\n if ok { Ok(val) } else { Err(val) }\n}\n\n#[cfg(target_arch = \"x86_64\")]\nuse self::atomic_compare_exchange as atomic_compare_exchange_weak;\n#[cfg(not(target_arch = \"x86_64\"))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_compare_exchange_weak(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n) -> Result {\n // SAFETY: the caller must uphold the safety contract.\n let (val, ok) = unsafe {\n match (success, failure) {\n (Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed_relaxed(dst, old, new),\n (Relaxed, Acquire) => intrinsics::atomic_cxchgweak_relaxed_acquire(dst, old, new),\n (Relaxed, SeqCst) => intrinsics::atomic_cxchgweak_relaxed_seqcst(dst, old, new),\n (Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acquire_relaxed(dst, old, new),\n (Acquire, Acquire) => intrinsics::atomic_cxchgweak_acquire_acquire(dst, old, new),\n (Acquire, SeqCst) => intrinsics::atomic_cxchgweak_acquire_seqcst(dst, old, new),\n (Release, Relaxed) => intrinsics::atomic_cxchgweak_release_relaxed(dst, old, new),\n (Release, Acquire) => intrinsics::atomic_cxchgweak_release_acquire(dst, old, new),\n (Release, SeqCst) => intrinsics::atomic_cxchgweak_release_seqcst(dst, old, new),\n (AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_relaxed(dst, old, new),\n (AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel_acquire(dst, old, new),\n (AcqRel, SeqCst) => intrinsics::atomic_cxchgweak_acqrel_seqcst(dst, old, new),\n (SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_seqcst_relaxed(dst, old, new),\n (SeqCst, Acquire) => intrinsics::atomic_cxchgweak_seqcst_acquire(dst, old, new),\n (SeqCst, SeqCst) => intrinsics::atomic_cxchgweak_seqcst_seqcst(dst, old, new),\n _ => unreachable!(),\n }\n };\n if ok { Ok(val) } else { Err(val) }\n}\n\n#[inline(always)]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_update(dst: *mut u128, order: Ordering, mut f: F) -> u128\nwhere\n F: FnMut(u128) -> u128,\n{\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n // This is a private function and all instances of `f` only operate on the value\n // loaded, so there is no need to synchronize the first load/failed CAS.\n let mut prev = atomic_load(dst, Ordering::Relaxed);\n loop {\n let next = f(prev);\n match atomic_compare_exchange_weak(dst, prev, next, order, Ordering::Relaxed) {\n Ok(x) => return x,\n Err(x) => prev = x,\n }\n }\n }\n}\n\n// On x86_64, we use core::arch::x86_64::cmpxchg16b instead of core::intrinsics.\n// - On s390x, old LLVM (pre-18) generates libcalls for operations other than load/store/cmpxchg (see also module-level comment).\n#[cfg(any(target_arch = \"x86_64\", all(target_arch = \"s390x\", portable_atomic_pre_llvm_18)))]\natomic_rmw_by_atomic_update!();\n// On powerpc64, LLVM doesn't support 128-bit atomic min/max (see also module-level comment).\n#[cfg(target_arch = \"powerpc64\")]\natomic_rmw_by_atomic_update!(cmp);\n\n#[cfg(not(any(target_arch = \"x86_64\", all(target_arch = \"s390x\", portable_atomic_pre_llvm_18))))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_swap(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_xchg_acquire(dst, val),\n Release => intrinsics::atomic_xchg_release(dst, val),\n AcqRel => intrinsics::atomic_xchg_acqrel(dst, val),\n Relaxed => intrinsics::atomic_xchg_relaxed(dst, val),\n SeqCst => intrinsics::atomic_xchg_seqcst(dst, val),\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(any(target_arch = \"x86_64\", all(target_arch = \"s390x\", portable_atomic_pre_llvm_18))))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_add(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_xadd_acquire(dst, val),\n Release => intrinsics::atomic_xadd_release(dst, val),\n AcqRel => intrinsics::atomic_xadd_acqrel(dst, val),\n Relaxed => intrinsics::atomic_xadd_relaxed(dst, val),\n SeqCst => intrinsics::atomic_xadd_seqcst(dst, val),\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(any(target_arch = \"x86_64\", all(target_arch = \"s390x\", portable_atomic_pre_llvm_18))))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_sub(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_xsub_acquire(dst, val),\n Release => intrinsics::atomic_xsub_release(dst, val),\n AcqRel => intrinsics::atomic_xsub_acqrel(dst, val),\n Relaxed => intrinsics::atomic_xsub_relaxed(dst, val),\n SeqCst => intrinsics::atomic_xsub_seqcst(dst, val),\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(any(target_arch = \"x86_64\", all(target_arch = \"s390x\", portable_atomic_pre_llvm_18))))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_and(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_and_acquire(dst, val),\n Release => intrinsics::atomic_and_release(dst, val),\n AcqRel => intrinsics::atomic_and_acqrel(dst, val),\n Relaxed => intrinsics::atomic_and_relaxed(dst, val),\n SeqCst => intrinsics::atomic_and_seqcst(dst, val),\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(any(target_arch = \"x86_64\", all(target_arch = \"s390x\", portable_atomic_pre_llvm_18))))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_nand(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_nand_acquire(dst, val),\n Release => intrinsics::atomic_nand_release(dst, val),\n AcqRel => intrinsics::atomic_nand_acqrel(dst, val),\n Relaxed => intrinsics::atomic_nand_relaxed(dst, val),\n SeqCst => intrinsics::atomic_nand_seqcst(dst, val),\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(any(target_arch = \"x86_64\", all(target_arch = \"s390x\", portable_atomic_pre_llvm_18))))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_or(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_or_acquire(dst, val),\n Release => intrinsics::atomic_or_release(dst, val),\n AcqRel => intrinsics::atomic_or_acqrel(dst, val),\n Relaxed => intrinsics::atomic_or_relaxed(dst, val),\n SeqCst => intrinsics::atomic_or_seqcst(dst, val),\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(any(target_arch = \"x86_64\", all(target_arch = \"s390x\", portable_atomic_pre_llvm_18))))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_xor(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_xor_acquire(dst, val),\n Release => intrinsics::atomic_xor_release(dst, val),\n AcqRel => intrinsics::atomic_xor_acqrel(dst, val),\n Relaxed => intrinsics::atomic_xor_relaxed(dst, val),\n SeqCst => intrinsics::atomic_xor_seqcst(dst, val),\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(any(\n target_arch = \"x86_64\",\n target_arch = \"powerpc64\",\n all(target_arch = \"s390x\", portable_atomic_pre_llvm_18),\n)))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_max(dst: *mut u128, val: u128, order: Ordering) -> i128 {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_max_acquire(dst.cast::(), val as i128),\n Release => intrinsics::atomic_max_release(dst.cast::(), val as i128),\n AcqRel => intrinsics::atomic_max_acqrel(dst.cast::(), val as i128),\n Relaxed => intrinsics::atomic_max_relaxed(dst.cast::(), val as i128),\n SeqCst => intrinsics::atomic_max_seqcst(dst.cast::(), val as i128),\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(any(\n target_arch = \"x86_64\",\n target_arch = \"powerpc64\",\n all(target_arch = \"s390x\", portable_atomic_pre_llvm_18),\n)))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_min(dst: *mut u128, val: u128, order: Ordering) -> i128 {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_min_acquire(dst.cast::(), val as i128),\n Release => intrinsics::atomic_min_release(dst.cast::(), val as i128),\n AcqRel => intrinsics::atomic_min_acqrel(dst.cast::(), val as i128),\n Relaxed => intrinsics::atomic_min_relaxed(dst.cast::(), val as i128),\n SeqCst => intrinsics::atomic_min_seqcst(dst.cast::(), val as i128),\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(any(\n target_arch = \"x86_64\",\n target_arch = \"powerpc64\",\n all(target_arch = \"s390x\", portable_atomic_pre_llvm_18),\n)))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_umax(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_umax_acquire(dst, val),\n Release => intrinsics::atomic_umax_release(dst, val),\n AcqRel => intrinsics::atomic_umax_acqrel(dst, val),\n Relaxed => intrinsics::atomic_umax_relaxed(dst, val),\n SeqCst => intrinsics::atomic_umax_seqcst(dst, val),\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(any(\n target_arch = \"x86_64\",\n target_arch = \"powerpc64\",\n all(target_arch = \"s390x\", portable_atomic_pre_llvm_18),\n)))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_umin(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match order {\n Acquire => intrinsics::atomic_umin_acquire(dst, val),\n Release => intrinsics::atomic_umin_release(dst, val),\n AcqRel => intrinsics::atomic_umin_acqrel(dst, val),\n Relaxed => intrinsics::atomic_umin_relaxed(dst, val),\n SeqCst => intrinsics::atomic_umin_seqcst(dst, val),\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(any(target_arch = \"x86_64\", all(target_arch = \"s390x\", portable_atomic_pre_llvm_18))))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_not(dst: *mut u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_xor(dst, !0, order) }\n}\n\n#[cfg(not(any(target_arch = \"x86_64\", all(target_arch = \"s390x\", portable_atomic_pre_llvm_18))))]\n#[inline]\n#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\nunsafe fn atomic_neg(dst: *mut u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, u128::wrapping_neg) }\n}\n\n#[cfg(not(target_arch = \"x86_64\"))]\n#[inline]\nconst fn is_lock_free() -> bool {\n IS_ALWAYS_LOCK_FREE\n}\n#[cfg(not(target_arch = \"x86_64\"))]\nconst IS_ALWAYS_LOCK_FREE: bool = true;\n\n#[cfg(target_arch = \"x86_64\")]\n#[inline]\nfn is_lock_free() -> bool {\n #[cfg(target_feature = \"cmpxchg16b\")]\n {\n // CMPXCHG16B is available at compile-time.\n true\n }\n #[cfg(not(target_feature = \"cmpxchg16b\"))]\n {\n detect::detect().cmpxchg16b()\n }\n}\n#[cfg(target_arch = \"x86_64\")]\nconst IS_ALWAYS_LOCK_FREE: bool = cfg!(target_feature = \"cmpxchg16b\");\n\natomic128!(AtomicI128, i128, atomic_max, atomic_min);\natomic128!(AtomicU128, u128, atomic_umax, atomic_umin);\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n test_atomic_int!(i128);\n test_atomic_int!(u128);\n\n // load/store/swap implementation is not affected by signedness, so it is\n // enough to test only unsigned types.\n stress_test!(u128);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic128/macros.rs", "sha256": "31f63dd5f6523fdc80d752afdd7b13e90394e7210b8f14bfa38a4feab0822cd5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nmacro_rules! atomic128 {\n ($atomic_type:ident, $int_type:ident, $atomic_max:ident, $atomic_min:ident) => {\n #[repr(C, align(16))]\n pub(crate) struct $atomic_type {\n v: core::cell::UnsafeCell<$int_type>,\n }\n\n // Send is implicitly implemented.\n // SAFETY: any data races are prevented by atomic intrinsics.\n unsafe impl Sync for $atomic_type {}\n\n impl_default_no_fetch_ops!($atomic_type, $int_type);\n impl_default_bit_opts!($atomic_type, $int_type);\n impl $atomic_type {\n #[inline]\n pub(crate) const fn new(v: $int_type) -> Self {\n Self { v: core::cell::UnsafeCell::new(v) }\n }\n\n #[inline]\n pub(crate) fn is_lock_free() -> bool {\n is_lock_free()\n }\n pub(crate) const IS_ALWAYS_LOCK_FREE: bool = IS_ALWAYS_LOCK_FREE;\n\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn load(&self, order: Ordering) -> $int_type {\n crate::utils::assert_load_ordering(order);\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_load(self.v.get().cast::(), order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn store(&self, val: $int_type, order: Ordering) {\n crate::utils::assert_store_ordering(order);\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_store(self.v.get().cast::(), val as u128, order)\n }\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_swap(self.v.get().cast::(), val as u128, order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure);\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n match atomic_compare_exchange(\n self.v.get().cast::(),\n current as u128,\n new as u128,\n success,\n failure,\n ) {\n Ok(v) => Ok(v as $int_type),\n Err(v) => Err(v as $int_type),\n }\n }\n }\n\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange_weak(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure);\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n match atomic_compare_exchange_weak(\n self.v.get().cast::(),\n current as u128,\n new as u128,\n success,\n failure,\n ) {\n Ok(v) => Ok(v as $int_type),\n Err(v) => Err(v as $int_type),\n }\n }\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_add(self.v.get().cast::(), val as u128, order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_sub(self.v.get().cast::(), val as u128, order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_and(self.v.get().cast::(), val as u128, order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_nand(self.v.get().cast::(), val as u128, order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_or(self.v.get().cast::(), val as u128, order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_xor(self.v.get().cast::(), val as u128, order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n $atomic_max(self.v.get().cast::(), val as u128, order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n $atomic_min(self.v.get().cast::(), val as u128, order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_not(self.v.get().cast::(), order) as $int_type\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn not(&self, order: Ordering) {\n self.fetch_not(order);\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_neg(self.v.get().cast::(), order) as $int_type\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn neg(&self, order: Ordering) {\n self.fetch_neg(order);\n }\n\n #[inline]\n pub(crate) const fn as_ptr(&self) -> *mut $int_type {\n self.v.get()\n }\n }\n };\n}\n\n#[cfg(any(target_arch = \"powerpc64\", target_arch = \"s390x\", target_arch = \"x86_64\"))]\n#[allow(unused_macros)] // also used by intrinsics.rs\nmacro_rules! atomic_rmw_by_atomic_update {\n () => {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_swap(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, |_| val) }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_add(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, |x| x.wrapping_add(val)) }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_sub(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, |x| x.wrapping_sub(val)) }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_and(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, |x| x & val) }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_nand(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, |x| !(x & val)) }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_or(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, |x| x | val) }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_xor(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, |x| x ^ val) }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_not(dst: *mut u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, |x| !x) }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_neg(dst: *mut u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, u128::wrapping_neg) }\n }\n atomic_rmw_by_atomic_update!(cmp);\n };\n (cmp) => {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_max(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n atomic_update(dst, order, |x| core::cmp::max(x as i128, val as i128) as u128)\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_umax(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, |x| core::cmp::max(x, val)) }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_min(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n atomic_update(dst, order, |x| core::cmp::min(x as i128, val as i128) as u128)\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn atomic_umin(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_update(dst, order, |x| core::cmp::min(x, val)) }\n }\n };\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic128/mod.rs", "sha256": "4d12d39ed3c0e5ab46724ac9536598ccc0028147d258b739ed27212c4e4d5e7e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\n128-bit atomic implementations on 64-bit architectures\n\nSee README.md for details.\n*/\n\n// AArch64\n#[cfg(any(\n all(\n target_arch = \"aarch64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n ),\n all(\n target_arch = \"arm64ec\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n ),\n))]\n// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.\n#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = \"intrinsics.rs\")]\npub(super) mod aarch64;\n\n// powerpc64\n#[cfg(all(\n target_arch = \"powerpc64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n all(\n feature = \"fallback\",\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n all(\n target_env = \"gnu\",\n any(target_endian = \"little\", not(target_feature = \"crt-static\")),\n ),\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n all(\n target_os = \"freebsd\",\n any(\n target_endian = \"little\",\n not(target_feature = \"crt-static\"),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"openbsd\",\n all(\n target_os = \"aix\",\n not(portable_atomic_pre_llvm_20),\n any(test, portable_atomic_outline_atomics), // TODO(aix): currently disabled by default\n ),\n ),\n not(any(miri, portable_atomic_sanitize_thread)),\n ),\n ),\n))]\n// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.\n#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = \"intrinsics.rs\")]\npub(super) mod powerpc64;\n\n// riscv64\n#[cfg(all(\n target_arch = \"riscv64\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(\n target_feature = \"zacas\",\n portable_atomic_target_feature = \"zacas\",\n all(\n feature = \"fallback\",\n not(portable_atomic_no_outline_atomics),\n any(target_os = \"linux\", target_os = \"android\"),\n ),\n ),\n))]\npub(super) mod riscv64;\n\n// s390x\n#[cfg(all(\n target_arch = \"s390x\",\n not(all(any(miri, portable_atomic_sanitize_thread), not(portable_atomic_atomic_intrinsics))),\n not(portable_atomic_no_asm),\n))]\n// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.\n#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = \"intrinsics.rs\")]\npub(super) mod s390x;\n\n// x86_64\n#[cfg(all(\n target_arch = \"x86_64\",\n not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(\n target_feature = \"cmpxchg16b\",\n portable_atomic_target_feature = \"cmpxchg16b\",\n all(\n feature = \"fallback\",\n not(portable_atomic_no_outline_atomics),\n not(any(target_env = \"sgx\", miri)),\n ),\n ),\n))]\n// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.\n#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = \"intrinsics.rs\")]\npub(super) mod x86_64;\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic128/powerpc64.rs", "sha256": "f1831b186b89049aba31be4c4cd37b5572554f2655944d5cfc04ab5654691715", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\n128-bit atomic implementation on PowerPC64.\n\nThis architecture provides the following 128-bit atomic instructions:\n\n- lq/stq: load/store (ISA 2.07 or later, included in the Linux Compliancy subset and AIX Compliancy subset)\n- lqarx/stqcx.: LL/SC (ISA 2.07 or later, included in the Linux Compliancy subset and AIX Compliancy subset)\n- plq/pstq: load/store (ISA 3.1 or later, included in the Linux Compliancy subset and AIX Compliancy subset)\n\nSee \"Atomic operation overview by architecture\" in atomic-maybe-uninit for a more comprehensive and\ndetailed description of the atomic and synchronize instructions in this architecture:\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#powerpc\n\nNote that we do not separate LL and SC into separate functions, but handle\nthem within a single asm block. This is because it is theoretically possible\nfor the compiler to insert operations that might clear the reservation between\nLL and SC. See aarch64.rs for details.\n\nNote: On Miri and ThreadSanitizer which do not support inline assembly, we don't use\nthis module and use intrinsics.rs instead.\n\nRefs:\n- Power ISA\n https://openpowerfoundation.org/specifications/isa\n- AIX Assembler language reference\n https://www.ibm.com/docs/en/aix/7.3?topic=aix-assembler-language-reference\n- atomic-maybe-uninit\n https://github.com/taiki-e/atomic-maybe-uninit\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\ninclude!(\"macros.rs\");\n\n#[cfg(not(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n)))]\n#[path = \"../fallback/outline_atomics.rs\"]\nmod fallback;\n\n// On musl with static linking, it seems that getauxval is not always available.\n// See detect/auxv.rs for more.\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(\n test,\n not(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n )),\n))]\n#[cfg(any(\n all(\n target_os = \"linux\",\n any(\n all(\n target_env = \"gnu\",\n any(target_endian = \"little\", not(target_feature = \"crt-static\")),\n ),\n all(target_env = \"musl\", any(not(target_feature = \"crt-static\"), feature = \"std\")),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n all(\n target_os = \"freebsd\",\n any(\n target_endian = \"little\",\n not(target_feature = \"crt-static\"),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"openbsd\",\n))]\n#[path = \"../detect/auxv.rs\"]\nmod detect;\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(\n test,\n not(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n )),\n))]\n#[cfg(target_os = \"aix\")]\n#[cfg(not(portable_atomic_pre_llvm_20))] // SIGTRAP on LLVM 19\n#[cfg(any(test, portable_atomic_outline_atomics))] // TODO(aix): currently disabled by default\n#[path = \"../detect/powerpc64_aix.rs\"]\nmod detect;\n\nuse core::{arch::asm, sync::atomic::Ordering};\n\n#[cfg(portable_atomic_no_strict_provenance)]\nuse crate::utils::ptr::PtrExt as _;\nuse crate::utils::{Pair, U128};\n\nmacro_rules! debug_assert_pwr8 {\n () => {\n #[cfg(not(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n )))]\n {\n debug_assert!(detect::detect().quadword_atomics());\n }\n };\n}\n\n// Refs: https://www.ibm.com/docs/en/aix/7.3?topic=ops-machine-pseudo-op\n//\n// This is similar to #[target_feature(enable = \"quadword-atomics\")], except that there are\n// no compiler guarantees regarding (un)inlining, and the scope is within an asm\n// block rather than a function. We use this directive because #[target_feature(enable = \"quadword-atomics\")]\n// is unstable and unavailable on old nightly and incompatible with rustc_codegen_cranelift:\n// https://github.com/rust-lang/rustc_codegen_cranelift/issues/1400#issuecomment-1774599775\n//\n// Note: start_pwr8 and end_pwr8 must be used in pairs.\n//\n// Note: If power8 instructions are not available at compile-time, we must guarantee that\n// the function that uses it is not inlined into a function where it is not\n// clear whether power8 instructions are available. Otherwise, (even if we checked whether\n// power8 instructions are available at run-time) optimizations that reorder its\n// instructions across the if condition might introduce undefined behavior.\n// (see also https://rust-lang.github.io/rfcs/2045-target-feature.html#safely-inlining-target_feature-functions-on-more-contexts)\n// However, our code uses the ifunc helper macro that works with function pointers,\n// so we don't have to worry about this unless calling without helper macro.\nmacro_rules! start_pwr8 {\n () => {\n \".machine push\\n.machine power8\"\n };\n}\nmacro_rules! end_pwr8 {\n () => {\n \".machine pop\"\n };\n}\n\nmacro_rules! atomic_rmw {\n ($op:ident, $order:ident) => {\n match $order {\n Ordering::Relaxed => $op!(\"\", \"\"),\n Ordering::Acquire => $op!(\"isync\", \"\"),\n Ordering::Release => $op!(\"\", \"lwsync\"),\n Ordering::AcqRel => $op!(\"isync\", \"lwsync\"),\n Ordering::SeqCst => $op!(\"isync\", \"sync\"),\n _ => unreachable!(),\n }\n };\n}\nmacro_rules! atomic_cas {\n ($op:ident, $success:ident, $failure:ident) => {\n if $failure == Ordering::Relaxed {\n match $success {\n Ordering::Relaxed => $op!(\"\", \"\", \"\"),\n Ordering::Acquire => $op!(\"\", \"isync\", \"\"),\n Ordering::Release => $op!(\"\", \"\", \"lwsync\"),\n Ordering::AcqRel => $op!(\"\", \"isync\", \"lwsync\"),\n Ordering::SeqCst => $op!(\"\", \"isync\", \"sync\"),\n _ => unreachable!(),\n }\n } else {\n let order = crate::utils::upgrade_success_ordering($success, $failure);\n match order {\n // Relaxed and Release are covered in $failure == Relaxed branch.\n Ordering::Acquire => $op!(\"isync\", \"\", \"\"),\n Ordering::AcqRel => $op!(\"isync\", \"\", \"lwsync\"),\n Ordering::SeqCst => $op!(\"isync\", \"\", \"sync\"),\n _ => unreachable!(),\n }\n }\n };\n}\n\n// Extracts and checks the EQ bit of cr0.\n#[inline]\nfn test_cr0_eq(cr: u64) -> bool {\n cr & 0x20000000 != 0\n}\n\n// If quadword-atomics is available at compile-time, we can always use pwr8_fn.\n#[cfg(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n))]\nuse self::atomic_load_pwr8 as atomic_load;\n// Otherwise, we need to do run-time detection and can use pwr8_fn only if quadword-atomics is available.\n#[cfg(not(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n)))]\n#[inline]\nunsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(src: *mut u128) -> u128;\n atomic_load_pwr8_relaxed = atomic_load_pwr8(Ordering::Relaxed);\n atomic_load_pwr8_acquire = atomic_load_pwr8(Ordering::Acquire);\n atomic_load_pwr8_seqcst = atomic_load_pwr8(Ordering::SeqCst);\n }\n // SAFETY: the caller must uphold the safety contract.\n // we only calls atomic_load_pwr8 if quadword-atomics is available.\n unsafe {\n match order {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(src: *mut u128) -> u128 {\n if detect::detect().quadword_atomics() {\n atomic_load_pwr8_relaxed\n } else {\n fallback::atomic_load_non_seqcst\n }\n })\n }\n Ordering::Acquire => {\n ifunc!(unsafe fn(src: *mut u128) -> u128 {\n if detect::detect().quadword_atomics() {\n atomic_load_pwr8_acquire\n } else {\n fallback::atomic_load_non_seqcst\n }\n })\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(src: *mut u128) -> u128 {\n if detect::detect().quadword_atomics() {\n atomic_load_pwr8_seqcst\n } else {\n fallback::atomic_load_seqcst\n }\n })\n }\n _ => unreachable!(),\n }\n }\n}\n#[inline]\nunsafe fn atomic_load_pwr8(src: *mut u128, order: Ordering) -> u128 {\n debug_assert!(src.addr() % 16 == 0);\n debug_assert_pwr8!();\n let (out_hi, out_lo);\n\n // SAFETY: the caller must uphold the safety contract.\n //\n // Refs: Section 3.3.4 \"Fixed Point Load and Store Quadword Instructions\" of Power ISA 3.1C Book I\n unsafe {\n macro_rules! atomic_load_acquire {\n ($release:tt) => {\n asm!(\n start_pwr8!(),\n $release,\n \"lq %r4, 0({src})\", // atomic { r4:r5 = *src }\n \"cmpw %r4, %r4\", // if r4 == r4 { cr0.EQ = 1 } else { cr0.EQ = 0 }\n \"bne- %cr0, 2f\", // if unlikely(cr0.EQ == 0) { jump 'never }\n \"2:\", // 'never:\n \"isync\", // fence (works in combination with a branch that depends on the loaded value)\n end_pwr8!(),\n src = in(reg_nonzero) ptr_reg!(src),\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n // We cannot use r1 (sp) and r2 (system reserved), so start with r4 or grater.\n out(\"r4\") out_hi,\n out(\"r5\") out_lo,\n out(\"cr0\") _,\n options(nostack, preserves_flags),\n )\n };\n }\n match order {\n Ordering::Relaxed => {\n asm!(\n start_pwr8!(),\n \"lq %r4, 0({src})\", // atomic { r4:r5 = *src }\n end_pwr8!(),\n src = in(reg_nonzero) ptr_reg!(src),\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n // We cannot use r1 (sp) and r2 (system reserved), so start with r4 or grater.\n out(\"r4\") out_hi,\n out(\"r5\") out_lo,\n options(nostack, preserves_flags),\n );\n }\n Ordering::Acquire => atomic_load_acquire!(\"\"),\n Ordering::SeqCst => atomic_load_acquire!(\"sync\"),\n _ => unreachable!(),\n }\n U128 { pair: Pair { hi: out_hi, lo: out_lo } }.whole\n }\n}\n\n// If quadword-atomics is available at compile-time, we can always use pwr8_fn.\n#[cfg(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n))]\nuse self::atomic_store_pwr8 as atomic_store;\n// Otherwise, we need to do run-time detection and can use pwr8_fn only if quadword-atomics is available.\n#[cfg(not(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n)))]\n#[inline]\nunsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(dst: *mut u128, val: u128);\n atomic_store_pwr8_relaxed = atomic_store_pwr8(Ordering::Relaxed);\n atomic_store_pwr8_release = atomic_store_pwr8(Ordering::Release);\n atomic_store_pwr8_seqcst = atomic_store_pwr8(Ordering::SeqCst);\n }\n // SAFETY: the caller must uphold the safety contract.\n // we only calls atomic_store_pwr8 if quadword-atomics is available.\n unsafe {\n match order {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(dst: *mut u128, val: u128) {\n if detect::detect().quadword_atomics() {\n atomic_store_pwr8_relaxed\n } else {\n fallback::atomic_store_non_seqcst\n }\n });\n }\n Ordering::Release => {\n ifunc!(unsafe fn(dst: *mut u128, val: u128) {\n if detect::detect().quadword_atomics() {\n atomic_store_pwr8_release\n } else {\n fallback::atomic_store_non_seqcst\n }\n });\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(dst: *mut u128, val: u128) {\n if detect::detect().quadword_atomics() {\n atomic_store_pwr8_seqcst\n } else {\n fallback::atomic_store_seqcst\n }\n });\n }\n _ => unreachable!(),\n }\n }\n}\n#[inline]\nunsafe fn atomic_store_pwr8(dst: *mut u128, val: u128, order: Ordering) {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_pwr8!();\n let val = U128 { whole: val };\n\n // SAFETY: the caller must uphold the safety contract.\n //\n // Refs: Section 3.3.4 \"Fixed Point Load and Store Quadword Instructions\" of Power ISA 3.1C Book I\n unsafe {\n macro_rules! atomic_store {\n ($release:tt) => {\n asm!(\n start_pwr8!(),\n $release, // fence\n \"stq %r4, 0({dst})\", // atomic { *dst = r4:r5 }\n end_pwr8!(),\n dst = in(reg_nonzero) ptr_reg!(dst),\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n // We cannot use r1 (sp) and r2 (system reserved), so start with r4 or grater.\n in(\"r4\") val.pair.hi,\n in(\"r5\") val.pair.lo,\n options(nostack, preserves_flags),\n )\n };\n }\n match order {\n Ordering::Relaxed => atomic_store!(\"\"),\n Ordering::Release => atomic_store!(\"lwsync\"),\n Ordering::SeqCst => atomic_store!(\"sync\"),\n _ => unreachable!(),\n }\n }\n}\n\n#[inline]\nunsafe fn atomic_compare_exchange(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n) -> Result {\n #[cfg(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n ))]\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantees that quadword atomics instructions are available at compile-time.\n let (prev, ok) = unsafe { atomic_compare_exchange_pwr8(dst, old, new, success, failure) };\n #[cfg(not(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n )))]\n // SAFETY: the caller must uphold the safety contract.\n let (prev, ok) = {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool);\n pwr8_relaxed_fn = atomic_compare_exchange_pwr8(Ordering::Relaxed, Ordering::Relaxed);\n pwr8_acquire_fn = atomic_compare_exchange_pwr8(Ordering::Acquire, Ordering::Acquire);\n pwr8_release_fn = atomic_compare_exchange_pwr8(Ordering::Release, Ordering::Relaxed);\n pwr8_acqrel_fn = atomic_compare_exchange_pwr8(Ordering::AcqRel, Ordering::Acquire);\n pwr8_seqcst_fn = atomic_compare_exchange_pwr8(Ordering::SeqCst, Ordering::SeqCst);\n }\n // SAFETY: the caller must uphold the safety contract.\n // we only calls pwr8_fn if quadword-atomics is available.\n unsafe {\n let success = crate::utils::upgrade_success_ordering(success, failure);\n match success {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n if detect::detect().quadword_atomics() {\n pwr8_relaxed_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::Acquire => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n if detect::detect().quadword_atomics() {\n pwr8_acquire_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::Release => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n if detect::detect().quadword_atomics() {\n pwr8_release_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::AcqRel => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n if detect::detect().quadword_atomics() {\n pwr8_acqrel_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n if detect::detect().quadword_atomics() {\n pwr8_seqcst_fn\n } else {\n fallback::atomic_compare_exchange_seqcst\n }\n })\n }\n _ => unreachable!(),\n }\n }\n };\n if ok { Ok(prev) } else { Err(prev) }\n}\n#[inline]\nunsafe fn atomic_compare_exchange_pwr8(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n) -> (u128, bool) {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_pwr8!();\n let old = U128 { whole: old };\n let new = U128 { whole: new };\n let (mut prev_hi, mut prev_lo);\n let mut r;\n\n // SAFETY: the caller must uphold the safety contract.\n //\n // Refs: Section 4.6.2.2 \"128-bit Load And Reserve and Store Conditional Instructions\" of Power ISA 3.1C Book II\n unsafe {\n macro_rules! cmpxchg {\n ($acquire_always:tt, $acquire_success:tt, $release:tt) => {\n asm!(\n start_pwr8!(),\n $release, // fence\n \"2:\", // 'retry:\n \"lqarx %r8, 0, {dst}\", // atomic { RESERVE = (dst, 16); r8:r9 = *dst }\n \"xor {tmp_lo}, %r9, {old_lo}\", // tmp_lo = r9 ^ old_lo\n \"xor {tmp_hi}, %r8, {old_hi}\", // tmp_hi = r8 ^ old_hi\n \"or. {tmp_lo}, {tmp_lo}, {tmp_hi}\", // tmp_lo |= tmp_hi; if tmp_lo == 0 { cr0.EQ = 1 } else { cr0.EQ = 0 }\n \"bne %cr0, 3f\", // if cr0.EQ == 0 { jump 'cmp-fail }\n \"stqcx. %r6, 0, {dst}\", // atomic { if RESERVE == (dst, 16) { *dst = r6:r7; cr0.EQ = 1 } else { cr0.EQ = 0 }; RESERVE = None }\n \"bne %cr0, 2b\", // if cr0.EQ == 0 { jump 'retry }\n $acquire_success, // fence\n \"3:\", // 'cmp-fail:\n $acquire_always, // fence\n \"mfcr {tmp_lo}\", // tmp_lo = zero_extend(cr)\n end_pwr8!(),\n dst = in(reg_nonzero) ptr_reg!(dst),\n old_hi = in(reg) old.pair.hi,\n old_lo = in(reg) old.pair.lo,\n tmp_hi = out(reg) _,\n tmp_lo = out(reg) r,\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n // We cannot use r1 (sp) and r2 (system reserved), so start with r4 or grater.\n in(\"r6\") new.pair.hi,\n in(\"r7\") new.pair.lo,\n out(\"r8\") prev_hi,\n out(\"r9\") prev_lo,\n out(\"cr0\") _,\n options(nostack, preserves_flags),\n )\n };\n }\n atomic_cas!(cmpxchg, success, failure);\n // if compare failed EQ bit is cleared, if store succeeds EQ bit is set.\n (U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole, test_cr0_eq(r))\n }\n}\n\n// Always use strong CAS for outline-atomics.\n#[cfg(not(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n)))]\nuse self::atomic_compare_exchange as atomic_compare_exchange_weak;\n#[cfg(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n))]\n#[inline]\nunsafe fn atomic_compare_exchange_weak(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n) -> Result {\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantees that quadword atomics instructions are available at compile-time.\n let (prev, ok) = unsafe { atomic_compare_exchange_weak_pwr8(dst, old, new, success, failure) };\n if ok { Ok(prev) } else { Err(prev) }\n}\n#[cfg(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n))]\n#[inline]\nunsafe fn atomic_compare_exchange_weak_pwr8(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n) -> (u128, bool) {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_pwr8!();\n let old = U128 { whole: old };\n let new = U128 { whole: new };\n let (mut prev_hi, mut prev_lo);\n let mut r;\n\n // SAFETY: the caller must uphold the safety contract.\n //\n // Refs: Section 4.6.2.2 \"128-bit Load And Reserve and Store Conditional Instructions\" of Power ISA 3.1C Book II\n unsafe {\n macro_rules! cmpxchg_weak {\n ($acquire_always:tt, $acquire_success:tt, $release:tt) => {\n asm!(\n start_pwr8!(),\n $release, // fence\n \"lqarx %r8, 0, {dst}\", // atomic { RESERVE = (dst, 16); r8:r9 = *dst }\n \"xor {tmp_lo}, %r9, {old_lo}\", // tmp_lo = r9 ^ old_lo\n \"xor {tmp_hi}, %r8, {old_hi}\", // tmp_hi = r8 ^ old_hi\n \"or. {tmp_lo}, {tmp_lo}, {tmp_hi}\", // tmp_lo |= tmp_hi; if tmp_lo == 0 { cr0.EQ = 1 } else { cr0.EQ = 0 }\n \"bne %cr0, 3f\", // if cr0.EQ == 0 { jump 'cmp-fail }\n \"stqcx. %r6, 0, {dst}\", // atomic { if RESERVE == (dst, 16) { *dst = r6:r7; cr0.EQ = 1 } else { cr0.EQ = 0 }; RESERVE = None }\n $acquire_success, // fence\n \"3:\", // 'cmp-fail:\n $acquire_always, // fence\n \"mfcr {tmp_lo}\", // tmp_lo = zero_extend(cr)\n end_pwr8!(),\n dst = in(reg_nonzero) ptr_reg!(dst),\n old_hi = in(reg) old.pair.hi,\n old_lo = in(reg) old.pair.lo,\n tmp_hi = out(reg) _,\n tmp_lo = out(reg) r,\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n // We cannot use r1 (sp) and r2 (system reserved), so start with r4 or grater.\n in(\"r6\") new.pair.hi,\n in(\"r7\") new.pair.lo,\n out(\"r8\") prev_hi,\n out(\"r9\") prev_lo,\n out(\"cr0\") _,\n options(nostack, preserves_flags),\n )\n };\n }\n atomic_cas!(cmpxchg_weak, success, failure);\n // if compare or store failed EQ bit is cleared, if store succeeds EQ bit is set.\n (U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole, test_cr0_eq(r))\n }\n}\n\n// Do not use atomic_rmw_ll_sc_3 because it needs extra MR to implement swap.\n#[inline]\nunsafe fn atomic_swap_pwr8(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_pwr8!();\n let val = U128 { whole: val };\n let (mut prev_hi, mut prev_lo);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n macro_rules! swap {\n ($acquire:tt, $release:tt) => {\n asm!(\n start_pwr8!(),\n $release, // fence\n \"2:\", // 'retry:\n \"lqarx %r6, 0, {dst}\", // atomic { RESERVE = (dst, 16); r6:r7 = *dst }\n \"stqcx. %r8, 0, {dst}\", // atomic { if RESERVE == (dst, 16) { *dst = r8:r9; cr0.EQ = 1 } else { cr0.EQ = 0 }; RESERVE = None }\n \"bne %cr0, 2b\", // if cr0.EQ == 0 { jump 'retry }\n $acquire, // fence\n end_pwr8!(),\n dst = in(reg_nonzero) ptr_reg!(dst),\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n // We cannot use r1 (sp) and r2 (system reserved), so start with r4 or grater.\n out(\"r6\") prev_hi,\n out(\"r7\") prev_lo,\n in(\"r8\") val.pair.hi,\n in(\"r9\") val.pair.lo,\n out(\"cr0\") _,\n options(nostack, preserves_flags),\n )\n };\n }\n atomic_rmw!(swap, order);\n U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole\n }\n}\n\n/// Atomic RMW by LL/SC loop (3 arguments)\n/// `unsafe fn(dst: *mut u128, val: u128, order: Ordering) -> u128;`\n///\n/// $op can use the following registers:\n/// - val_hi/val_lo pair: val argument (read-only for `$op`)\n/// - r6/r7 pair: previous value loaded by ll (read-only for `$op`)\n/// - r8/r9 pair: new value that will be stored by sc\nmacro_rules! atomic_rmw_ll_sc_3 {\n ($name:ident, [$($reg:tt)*], $($op:tt)*) => {\n #[inline]\n unsafe fn $name(dst: *mut u128, val: u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_pwr8!();\n let val = U128 { whole: val };\n let (mut prev_hi, mut prev_lo);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n macro_rules! op {\n ($acquire:tt, $release:tt) => {\n asm!(\n start_pwr8!(),\n $release, // fence\n \"2:\", // 'retry:\n \"lqarx %r6, 0, {dst}\", // atomic { RESERVE = (dst, 16); r6:r7 = *dst }\n $($op)*\n \"stqcx. %r8, 0, {dst}\", // atomic { if RESERVE == (dst, 16) { *dst = r8:r9; cr0.EQ = 1 } else { cr0.EQ = 0 }; RESERVE = None }\n \"bne %cr0, 2b\", // if cr0.EQ == 0 { jump 'retry }\n $acquire, // fence\n end_pwr8!(),\n dst = in(reg_nonzero) ptr_reg!(dst),\n val_hi = in(reg) val.pair.hi,\n val_lo = in(reg) val.pair.lo,\n $($reg)*\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n // We cannot use r1 (sp) and r2 (system reserved), so start with r4 or grater.\n out(\"r6\") prev_hi,\n out(\"r7\") prev_lo,\n out(\"r8\") _, // new (hi)\n out(\"r9\") _, // new (lo)\n out(\"cr0\") _,\n options(nostack, preserves_flags),\n )\n };\n }\n atomic_rmw!(op, order);\n U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole\n }\n }\n };\n}\n/// Atomic RMW by LL/SC loop (2 arguments)\n/// `unsafe fn(dst: *mut u128, order: Ordering) -> u128;`\n///\n/// $op can use the following registers:\n/// - r6/r7 pair: previous value loaded by ll (read-only for `$op`)\n/// - r8/r9 pair: new value that will be stored by sc\nmacro_rules! atomic_rmw_ll_sc_2 {\n ($name:ident, [$($reg:tt)*], $($op:tt)*) => {\n #[inline]\n unsafe fn $name(dst: *mut u128, order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_pwr8!();\n let (mut prev_hi, mut prev_lo);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n macro_rules! op {\n ($acquire:tt, $release:tt) => {\n asm!(\n start_pwr8!(),\n $release, // fence\n \"2:\", // 'retry:\n \"lqarx %r6, 0, {dst}\", // atomic { RESERVE = (dst, 16); r6:r7 = *dst }\n $($op)*\n \"stqcx. %r8, 0, {dst}\", // atomic { if RESERVE == (dst, 16) { *dst = r8:r9; cr0.EQ = 1 } else { cr0.EQ = 0 }; RESERVE = None }\n \"bne %cr0, 2b\", // if cr0.EQ == 0 { jump 'retry }\n $acquire, // fence\n end_pwr8!(),\n dst = in(reg_nonzero) ptr_reg!(dst),\n $($reg)*\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n // We cannot use r1 (sp) and r2 (system reserved), so start with r4 or grater.\n out(\"r6\") prev_hi,\n out(\"r7\") prev_lo,\n out(\"r8\") _, // new (hi)\n out(\"r9\") _, // new (lo)\n out(\"cr0\") _,\n options(nostack, preserves_flags),\n )\n };\n }\n atomic_rmw!(op, order);\n U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole\n }\n }\n };\n}\n\natomic_rmw_ll_sc_3! {\n atomic_add_pwr8, [out(\"xer\") _,],\n \"addc %r9, {val_lo}, %r7\", // r9 = val_lo + r7; xer.CA = carry\n \"adde %r8, {val_hi}, %r6\", // r8 = val_hi + r6 + xer.CA\n}\natomic_rmw_ll_sc_3! {\n atomic_sub_pwr8, [out(\"xer\") _,],\n \"subc %r9, %r7, {val_lo}\", // r9 = val_lo - r7; xer.CA = borrow\n \"subfe %r8, {val_hi}, %r6\", // r8 = val_hi - r6 - xer.CA\n}\natomic_rmw_ll_sc_3! {\n atomic_and_pwr8, [],\n \"and %r9, {val_lo}, %r7\", // r9 = val_lo & r7\n \"and %r8, {val_hi}, %r6\", // r8 = val_hi & r6\n}\natomic_rmw_ll_sc_3! {\n atomic_nand_pwr8, [],\n \"nand %r9, {val_lo}, %r7\", // r9 = !(val_lo & r7)\n \"nand %r8, {val_hi}, %r6\", // r8 = !(val_hi & r6)\n}\natomic_rmw_ll_sc_3! {\n atomic_or_pwr8, [],\n \"or %r9, {val_lo}, %r7\", // r9 = val_lo | r7\n \"or %r8, {val_hi}, %r6\", // r8 = val_hi | r6\n}\natomic_rmw_ll_sc_3! {\n atomic_xor_pwr8, [],\n \"xor %r9, {val_lo}, %r7\", // r9 = val_lo ^ r7\n \"xor %r8, {val_hi}, %r6\", // r8 = val_hi ^ r6\n}\natomic_rmw_ll_sc_3! {\n atomic_max_pwr8, [out(\"cr1\") _,],\n \"cmpld %r7, {val_lo}\", // if r7(u) < val_lo(u) { cr0 = { LT: 1, ..0 } } else if r7(u) > val_lo(u) { cr0 = { GT: 1, ..0 } } else { cr0 = { EQ: 1, ..0 } }\n \"iselgt %r9, %r7, {val_lo}\", // if cr0.GT == 1 { r9 = r7 } else { r9 = val_lo }\n \"cmpd %cr1, %r6, {val_hi}\", // if r6(i) < val_hi(i) { cr1 = { LT: 1, ..0 } } else if r6(i) > val_hi(i) { cr1 = { GT: 1, ..0 } } else { cr1 = { EQ: 1, ..0 } }\n \"isel %r8, %r7, {val_lo}, 5\", // if cr1.GT == 1 { r8 = r7 } else { r8 = val_lo }\n \"cmpld %r6, {val_hi}\", // if r6(u) < val_hi(u) { cr0 = { LT: 1, ..0 } } else if r6(u) > val_hi(u) { cr0 = { GT: 1, ..0 } } else { cr0 = { EQ: 1, ..0 } }\n \"iseleq %r9, %r9, %r8\", // if cr0.EQ == 1 { r9 = r9 } else { r9 = r8 }\n \"isel %r8, %r6, {val_hi}, 5\", // if cr1.GT == 1 { r8 = r6 } else { r8 = val_hi }\n}\natomic_rmw_ll_sc_3! {\n atomic_umax_pwr8, [],\n \"cmpld %r7, {val_lo}\", // if r7(u) < val_lo(u) { cr0 = { LT: 1, ..0 } } else if r7(u) > val_lo(u) { cr0 = { GT: 1, ..0 } } else { cr0 = { EQ: 1, ..0 } }\n \"iselgt %r9, %r7, {val_lo}\", // if cr0.GT == 1 { r9 = r7 } else { r9 = val_lo }\n \"cmpld %r6, {val_hi}\", // if r6(u) < val_hi(u) { cr0 = { LT: 1, ..0 } } else if r6(u) > val_hi(u) { cr0 = { GT: 1, ..0 } } else { cr0 = { EQ: 1, ..0 } }\n \"iselgt %r8, %r7, {val_lo}\", // if cr0.GT == 1 { r8 = r7 } else { r8 = val_lo }\n \"iseleq %r9, %r9, %r8\", // if cr0.EQ == 1 { r9 = r9 } else { r9 = r8 }\n \"iselgt %r8, %r6, {val_hi}\", // if cr0.GT == 1 { r8 = r6 } else { r8 = val_hi }\n}\natomic_rmw_ll_sc_3! {\n atomic_min_pwr8, [out(\"cr1\") _,],\n \"cmpld %r7, {val_lo}\", // if r7(u) < val_lo(u) { cr0 = { LT: 1, ..0 } } else if r7(u) > val_lo(u) { cr0 = { GT: 1, ..0 } } else { cr0 = { EQ: 1, ..0 } }\n \"isellt %r9, %r7, {val_lo}\", // if cr0.LT == 1 { r9 = r7 } else { r9 = val_lo }\n \"cmpd %cr1, %r6, {val_hi}\", // if r6(i) < val_hi(i) { cr1 = { LT: 1, ..0 } } else if r6(i) > val_hi(i) { cr1 = { GT: 1, ..0 } } else { cr1 = { EQ: 1, ..0 } }\n \"isel %r8, %r7, {val_lo}, 4\", // if cr1.LT == 1 { r8 = r7 } else { r8 = val_lo }\n \"cmpld %r6, {val_hi}\", // if r6(u) < val_hi(u) { cr0 = { LT: 1, ..0 } } else if r6(u) > val_hi(u) { cr0 = { GT: 1, ..0 } } else { cr0 = { EQ: 1, ..0 } }\n \"iseleq %r9, %r9, %r8\", // if cr0.EQ == 1 { r9 = r9 } else { r9 = r8 }\n \"isel %r8, %r6, {val_hi}, 4\", // if cr1.LT == 1 { r8 = r6 } else { r8 = val_hi }\n}\natomic_rmw_ll_sc_3! {\n atomic_umin_pwr8, [],\n \"cmpld %r7, {val_lo}\", // if r7(u) < val_lo(u) { cr0 = { LT: 1, ..0 } } else if r7(u) > val_lo(u) { cr0 = { GT: 1, ..0 } } else { cr0 = { EQ: 1, ..0 } }\n \"isellt %r9, %r7, {val_lo}\", // if cr0.LT == 1 { r9 = r7 } else { r9 = val_lo }\n \"cmpld %r6, {val_hi}\", // if r6(u) < val_hi(u) { cr0 = { LT: 1, ..0 } } else if r6(u) > val_hi(u) { cr0 = { GT: 1, ..0 } } else { cr0 = { EQ: 1, ..0 } }\n \"isellt %r8, %r7, {val_lo}\", // if cr0.LT == 1 { r8 = r7 } else { r8 = val_lo }\n \"iseleq %r9, %r9, %r8\", // if cr0.EQ == 1 { r9 = r9 } else { r9 = r8 }\n \"isellt %r8, %r6, {val_hi}\", // if cr0.LT == 1 { r8 = r6 } else { r8 = val_hi }\n}\n\n#[inline]\nunsafe fn atomic_not_pwr8(dst: *mut u128, order: Ordering) -> u128 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { atomic_xor_pwr8(dst, !0, order) }\n}\n\n#[cfg(not(portable_atomic_pre_llvm_16))]\natomic_rmw_ll_sc_2! {\n atomic_neg_pwr8, [out(\"xer\") _,],\n \"subfic %r9, %r7, 0\", // r9 = 0 - r7; xer.CA = borrow\n \"subfze %r8, %r6\", // r8 = 0 - r6 - xer.CA\n}\n// LLVM 15 miscompiles subfic.\n#[cfg(portable_atomic_pre_llvm_16)]\natomic_rmw_ll_sc_2! {\n atomic_neg_pwr8, [zero = in(reg) 0_u64, out(\"xer\") _,],\n \"subc %r9, {zero}, %r7\", // r9 = 0 - r7; xer.CA = borrow\n \"subfze %r8, %r6\", // r8 = 0 - r6 - xer.CA\n}\n\nmacro_rules! select_atomic_rmw {\n (\n unsafe fn $name:ident($($arg:tt)*) $(-> $ret_ty:ty)?;\n pwr8 = $pwr8_fn:ident;\n non_seqcst_fallback = $non_seqcst_fallback_fn:ident;\n seqcst_fallback = $seqcst_fallback_fn:ident;\n ) => {\n // If quadword-atomics is available at compile-time, we can always use pwr8_fn.\n #[cfg(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n ))]\n use self::$pwr8_fn as $name;\n // Otherwise, we need to do run-time detection and can use pwr8_fn only if quadword-atomics is available.\n #[cfg(not(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n )))]\n #[inline]\n unsafe fn $name($($arg)*, order: Ordering) $(-> $ret_ty)? {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn($($arg)*) $(-> $ret_ty)?;\n pwr8_relaxed_fn = $pwr8_fn(Ordering::Relaxed);\n pwr8_acquire_fn = $pwr8_fn(Ordering::Acquire);\n pwr8_release_fn = $pwr8_fn(Ordering::Release);\n pwr8_acqrel_fn = $pwr8_fn(Ordering::AcqRel);\n pwr8_seqcst_fn = $pwr8_fn(Ordering::SeqCst);\n }\n // SAFETY: the caller must uphold the safety contract.\n // we only calls pwr8_fn if quadword-atomics is available.\n unsafe {\n match order {\n Ordering::Relaxed => {\n ifunc!(unsafe fn($($arg)*) $(-> $ret_ty)? {\n if detect::detect().quadword_atomics() {\n pwr8_relaxed_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::Acquire => {\n ifunc!(unsafe fn($($arg)*) $(-> $ret_ty)? {\n if detect::detect().quadword_atomics() {\n pwr8_acquire_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::Release => {\n ifunc!(unsafe fn($($arg)*) $(-> $ret_ty)? {\n if detect::detect().quadword_atomics() {\n pwr8_release_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::AcqRel => {\n ifunc!(unsafe fn($($arg)*) $(-> $ret_ty)? {\n if detect::detect().quadword_atomics() {\n pwr8_acqrel_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn($($arg)*) $(-> $ret_ty)? {\n if detect::detect().quadword_atomics() {\n pwr8_seqcst_fn\n } else {\n fallback::$seqcst_fallback_fn\n }\n })\n }\n _ => unreachable!(),\n }\n }\n }\n };\n}\n\nselect_atomic_rmw! {\n unsafe fn atomic_swap(dst: *mut u128, val: u128) -> u128;\n pwr8 = atomic_swap_pwr8;\n non_seqcst_fallback = atomic_swap_non_seqcst;\n seqcst_fallback = atomic_swap_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_add(dst: *mut u128, val: u128) -> u128;\n pwr8 = atomic_add_pwr8;\n non_seqcst_fallback = atomic_add_non_seqcst;\n seqcst_fallback = atomic_add_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_sub(dst: *mut u128, val: u128) -> u128;\n pwr8 = atomic_sub_pwr8;\n non_seqcst_fallback = atomic_sub_non_seqcst;\n seqcst_fallback = atomic_sub_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_and(dst: *mut u128, val: u128) -> u128;\n pwr8 = atomic_and_pwr8;\n non_seqcst_fallback = atomic_and_non_seqcst;\n seqcst_fallback = atomic_and_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_nand(dst: *mut u128, val: u128) -> u128;\n pwr8 = atomic_nand_pwr8;\n non_seqcst_fallback = atomic_nand_non_seqcst;\n seqcst_fallback = atomic_nand_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_or(dst: *mut u128, val: u128) -> u128;\n pwr8 = atomic_or_pwr8;\n non_seqcst_fallback = atomic_or_non_seqcst;\n seqcst_fallback = atomic_or_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_xor(dst: *mut u128, val: u128) -> u128;\n pwr8 = atomic_xor_pwr8;\n non_seqcst_fallback = atomic_xor_non_seqcst;\n seqcst_fallback = atomic_xor_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_max(dst: *mut u128, val: u128) -> u128;\n pwr8 = atomic_max_pwr8;\n non_seqcst_fallback = atomic_max_non_seqcst;\n seqcst_fallback = atomic_max_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_umax(dst: *mut u128, val: u128) -> u128;\n pwr8 = atomic_umax_pwr8;\n non_seqcst_fallback = atomic_umax_non_seqcst;\n seqcst_fallback = atomic_umax_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_min(dst: *mut u128, val: u128) -> u128;\n pwr8 = atomic_min_pwr8;\n non_seqcst_fallback = atomic_min_non_seqcst;\n seqcst_fallback = atomic_min_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_umin(dst: *mut u128, val: u128) -> u128;\n pwr8 = atomic_umin_pwr8;\n non_seqcst_fallback = atomic_umin_non_seqcst;\n seqcst_fallback = atomic_umin_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_not(dst: *mut u128) -> u128;\n pwr8 = atomic_not_pwr8;\n non_seqcst_fallback = atomic_not_non_seqcst;\n seqcst_fallback = atomic_not_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_neg(dst: *mut u128) -> u128;\n pwr8 = atomic_neg_pwr8;\n non_seqcst_fallback = atomic_neg_non_seqcst;\n seqcst_fallback = atomic_neg_seqcst;\n}\n\n#[inline]\nfn is_lock_free() -> bool {\n #[cfg(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n ))]\n {\n // lqarx and stqcx. instructions are statically available.\n true\n }\n #[cfg(not(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n )))]\n {\n detect::detect().quadword_atomics()\n }\n}\nconst IS_ALWAYS_LOCK_FREE: bool = cfg!(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n));\n\natomic128!(AtomicI128, i128, atomic_max, atomic_min);\natomic128!(AtomicU128, u128, atomic_umax, atomic_umin);\n\n#[cfg(not(valgrind))] // TODO(powerpc64): Hang (as of Valgrind 3.26)\n#[cfg(test)]\nmod tests {\n use super::*;\n\n test_atomic_int!(i128);\n test_atomic_int!(u128);\n\n // load/store/swap implementation is not affected by signedness, so it is\n // enough to test only unsigned types.\n stress_test!(u128);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic128/riscv64.rs", "sha256": "115c4d8bc0fed79973bc12462f506121c7e612f3cf14cc627a0d024c3a131387", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\n128-bit atomic implementation on riscv64.\n\nThis architecture provides the following 128-bit atomic instructions:\n\n- amocas.q: CAS (Zacas extension)\n\nSee \"Atomic operation overview by architecture\" in atomic-maybe-uninit for a more comprehensive and\ndetailed description of the atomic and synchronize instructions in this architecture:\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#risc-v\n\nNote: On Miri and ThreadSanitizer which do not support inline assembly, we don't use\nthis module and use fallback implementation instead.\n\nRefs:\n- RISC-V Instruction Set Manual\n \"Zacas\" Extension for Atomic Compare-and-Swap (CAS) Instructions\n https://docs.riscv.org/reference/isa/v20260120/unpriv/zacas.html\n- RISC-V Atomics ABI Specification\n https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20250812-301374e92976e298e676e7129a6212926b2299ce/riscv-atomic.adoc\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\ninclude!(\"macros.rs\");\n\n#[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n#[path = \"../fallback/outline_atomics.rs\"]\nmod fallback;\n\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(test, not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"))))]\n#[cfg(any(target_os = \"linux\", target_os = \"android\"))]\n#[path = \"../detect/riscv_linux.rs\"]\nmod detect;\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\nuse core::sync::atomic::Ordering;\n\n#[cfg(portable_atomic_no_strict_provenance)]\nuse crate::utils::ptr::PtrExt as _;\nuse crate::utils::{Pair, U128};\n\nmacro_rules! debug_assert_zacas {\n () => {\n #[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n {\n debug_assert!(detect::detect().zacas());\n }\n };\n}\n\n// `.option arch, +zacas` directive requires LLVM 20, so we use .4byte directive for old LLVM.\n// Note that `.insn ` directive requires LLVM 19.\n// https://github.com/llvm/llvm-project/commit/2a086dce691e3cc34a2fc27f4fb255bb2cbbfac9\n// https://github.com/riscv-non-isa/riscv-asm-manual/blob/v0.0.1/src/asm-manual.adoc#arch\n#[cfg(not(portable_atomic_pre_llvm_20))]\nmacro_rules! start_zacas {\n () => {\n \".option push\\n.option arch, +zacas\"\n };\n}\n#[cfg(not(portable_atomic_pre_llvm_20))]\nmacro_rules! end_zacas {\n () => {\n \".option pop\"\n };\n}\n\n#[cfg(not(portable_atomic_pre_llvm_20))]\nmacro_rules! atomic_rmw_amocas_order {\n ($op:ident, $order:ident) => {\n atomic_rmw_amocas_order!($op, $order, failure = $order)\n };\n ($op:ident, $order:ident, failure = $failure:ident) => {\n match $order {\n Ordering::Relaxed => $op!(\"\", \"\"),\n Ordering::Acquire => $op!(\"\", \".aq\"),\n Ordering::Release => $op!(\"\", \".rl\"),\n Ordering::AcqRel => $op!(\"\", \".aqrl\"),\n Ordering::SeqCst if $failure == Ordering::SeqCst => $op!(\"fence rw,rw\", \".aqrl\"),\n Ordering::SeqCst => $op!(\"\", \".aqrl\"),\n _ => unreachable!(),\n }\n };\n}\n#[cfg(portable_atomic_pre_llvm_20)]\nmacro_rules! atomic_rmw_amocas_order_insn {\n ($op:ident, $order:ident) => {\n atomic_rmw_amocas_order_insn!($op, $order, failure = $order)\n };\n ($op:ident, $order:ident, failure = $failure:ident) => {\n match $order {\n Ordering::Relaxed => $op!(\"\", \"8\"),\n Ordering::Acquire => $op!(\"\", \"c\"),\n Ordering::Release => $op!(\"\", \"a\"),\n Ordering::AcqRel => $op!(\"\", \"e\"),\n Ordering::SeqCst if $failure == Ordering::SeqCst => $op!(\"fence rw,rw\", \"e\"),\n Ordering::SeqCst => $op!(\"\", \"e\"),\n _ => unreachable!(),\n }\n };\n}\n\n// If zacas is available at compile-time, we can always use zacas_fn.\n#[cfg(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"))]\nuse self::atomic_load_zacas as atomic_load;\n// Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.\n#[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n#[inline]\nunsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(src: *mut u128) -> u128;\n atomic_load_zacas_relaxed = atomic_load_zacas(Ordering::Relaxed);\n atomic_load_zacas_acquire = atomic_load_zacas(Ordering::Acquire);\n atomic_load_zacas_seqcst = atomic_load_zacas(Ordering::SeqCst);\n }\n // SAFETY: the caller must uphold the safety contract.\n // we only calls atomic_load_zacas if zacas is available.\n unsafe {\n match order {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(src: *mut u128) -> u128 {\n if detect::detect().zacas() {\n atomic_load_zacas_relaxed\n } else {\n fallback::atomic_load_non_seqcst\n }\n })\n }\n Ordering::Acquire => {\n ifunc!(unsafe fn(src: *mut u128) -> u128 {\n if detect::detect().zacas() {\n atomic_load_zacas_acquire\n } else {\n fallback::atomic_load_non_seqcst\n }\n })\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(src: *mut u128) -> u128 {\n if detect::detect().zacas() {\n atomic_load_zacas_seqcst\n } else {\n fallback::atomic_load_seqcst\n }\n })\n }\n _ => unreachable!(),\n }\n }\n}\n#[inline]\nunsafe fn atomic_load_zacas(src: *mut u128, order: Ordering) -> u128 {\n debug_assert!(src.addr() % 16 == 0);\n debug_assert_zacas!();\n let (out_lo, out_hi);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n #[cfg(not(portable_atomic_pre_llvm_20))]\n macro_rules! load {\n ($fence:tt, $asm_order:tt) => {\n asm!(\n start_zacas!(),\n $fence, // fence\n concat!(\"amocas.q\", $asm_order, \" a2, a2, 0({src})\"), // atomic { if *dst == a2:a3 { *dst = a2:a3 } else { a2:a3 = *dst } }\n end_zacas!(),\n src = in(reg) ptr_reg!(src),\n inout(\"a2\") 0_u64 => out_lo,\n inout(\"a3\") 0_u64 => out_hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(not(portable_atomic_pre_llvm_20))]\n atomic_rmw_amocas_order!(load, order);\n #[cfg(portable_atomic_pre_llvm_20)]\n macro_rules! load {\n ($fence:tt, $insn_order:tt) => {\n asm!(\n $fence, // fence\n // amocas.q{,.aq,.rl,.aqrl} a2, a2, (a0) // atomic { if *a0 == a2:a3 { *a0 = a2:a3 } else { a2:a3 = *a0 } }\n concat!(\".4byte 0x2\", $insn_order, \"c5462f\"),\n in(\"a0\") ptr_reg!(src),\n inout(\"a2\") 0_u64 => out_lo,\n inout(\"a3\") 0_u64 => out_hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(portable_atomic_pre_llvm_20)]\n atomic_rmw_amocas_order_insn!(load, order);\n U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole\n }\n}\n\n#[inline]\nunsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n atomic_swap(dst, val, order);\n }\n}\n\n#[inline]\nunsafe fn atomic_compare_exchange(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n) -> Result {\n #[cfg(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"))]\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantees that zacas instructions are available at compile-time.\n let (prev, ok) = unsafe { atomic_compare_exchange_zacas(dst, old, new, success, failure) };\n #[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n let (prev, ok) = {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool);\n zacas_relaxed_fn = atomic_compare_exchange_zacas(Ordering::Relaxed, Ordering::Relaxed);\n zacas_acquire_fn = atomic_compare_exchange_zacas(Ordering::Acquire, Ordering::Acquire);\n zacas_release_fn = atomic_compare_exchange_zacas(Ordering::Release, Ordering::Relaxed);\n zacas_acqrel_fn = atomic_compare_exchange_zacas(Ordering::AcqRel, Ordering::Acquire);\n zacas_seqcst_fn = atomic_compare_exchange_zacas(Ordering::SeqCst, Ordering::SeqCst);\n }\n let order = crate::utils::upgrade_success_ordering(success, failure);\n // SAFETY: the caller must uphold the safety contract.\n // we only calls atomic_compare_exchange_zacas if zacas is available.\n unsafe {\n match order {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n if detect::detect().zacas() {\n zacas_relaxed_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::Acquire => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n if detect::detect().zacas() {\n zacas_acquire_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::Release => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n if detect::detect().zacas() {\n zacas_release_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::AcqRel => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n if detect::detect().zacas() {\n zacas_acqrel_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n if detect::detect().zacas() {\n zacas_seqcst_fn\n } else {\n fallback::atomic_compare_exchange_seqcst\n }\n })\n }\n _ => unreachable!(),\n }\n }\n };\n if ok { Ok(prev) } else { Err(prev) }\n}\n#[inline]\nunsafe fn atomic_compare_exchange_zacas(\n dst: *mut u128,\n old: u128,\n new: u128,\n success: Ordering,\n failure: Ordering,\n) -> (u128, bool) {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_zacas!();\n let order = crate::utils::upgrade_success_ordering(success, failure);\n let old = U128 { whole: old };\n let new = U128 { whole: new };\n let (prev_lo, prev_hi);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n #[cfg(not(portable_atomic_pre_llvm_20))]\n macro_rules! cmpxchg {\n ($fence:tt, $asm_order:tt) => {\n asm!(\n start_zacas!(),\n $fence, // fence\n concat!(\"amocas.q\", $asm_order, \" a4, a2, 0({dst})\"), // atomic { if *dst == a4:a5 { *dst = a2:a3 } else { a4:a5 = *dst } }\n end_zacas!(),\n dst = in(reg) ptr_reg!(dst),\n // must be allocated to even/odd register pair\n inout(\"a4\") old.pair.lo => prev_lo,\n inout(\"a5\") old.pair.hi => prev_hi,\n // must be allocated to even/odd register pair\n in(\"a2\") new.pair.lo,\n in(\"a3\") new.pair.hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(not(portable_atomic_pre_llvm_20))]\n atomic_rmw_amocas_order!(cmpxchg, order, failure = failure);\n #[cfg(portable_atomic_pre_llvm_20)]\n macro_rules! cmpxchg {\n ($fence:tt, $insn_order:tt) => {\n asm!(\n $fence, // fence\n // amocas.q{,.aq,.rl,.aqrl} a4, a2, (a0) // atomic { if *a0 == a4:a5 { *a0 = a2:a3 } else { a4:a5 = *a0 } }\n concat!(\".4byte 0x2\", $insn_order, \"c5472f\"),\n in(\"a0\") ptr_reg!(dst),\n // must be allocated to even/odd register pair\n inout(\"a4\") old.pair.lo => prev_lo,\n inout(\"a5\") old.pair.hi => prev_hi,\n // must be allocated to even/odd register pair\n in(\"a2\") new.pair.lo,\n in(\"a3\") new.pair.hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(portable_atomic_pre_llvm_20)]\n atomic_rmw_amocas_order_insn!(cmpxchg, order, failure = failure);\n let prev = U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole;\n (prev, prev == old.whole)\n }\n}\n\n// amocas is always strong.\nuse self::atomic_compare_exchange as atomic_compare_exchange_weak;\n\n// 128-bit atomic load by two 64-bit atomic loads.\n#[inline]\nunsafe fn byte_wise_atomic_load(src: *const u128) -> u128 {\n let (out_lo, out_hi);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n asm!(\n \"ld {out_lo}, ({src})\", // atomic { out_lo = *src }\n \"ld {out_hi}, 8({src})\", // atomic { out_hi = *src.byte_add(8) }\n src = in(reg) ptr_reg!(src),\n out_lo = out(reg) out_lo,\n out_hi = out(reg) out_hi,\n options(pure, nostack, preserves_flags, readonly),\n );\n U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole\n }\n}\n\nmacro_rules! select_atomic_rmw {\n (\n unsafe fn $name:ident(dst: *mut u128 $(, $($arg:tt)*)?) $(-> $ret_ty:ty)? {\n |$zacas_fn_binding:ident| $($zacas_fn_body:tt)*\n }\n zacas = $zacas_fn:ident;\n non_seqcst_fallback = $non_seqcst_fallback_fn:ident;\n seqcst_fallback = $seqcst_fallback_fn:ident;\n ) => {\n #[inline]\n unsafe fn $zacas_fn(dst: *mut u128 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n // This is not single-copy atomic reads, but this is ok because subsequent\n // CAS will check for consistency.\n //\n // Note that the C++20 memory model does not allow mixed-sized atomic access,\n // so we must use inline assembly to implement byte_wise_atomic_load.\n // (i.e., byte-wise atomic based on the standard library's atomic types\n // cannot be used here).\n let mut prev = byte_wise_atomic_load(dst);\n loop {\n let next = {\n let $zacas_fn_binding = prev;\n $($zacas_fn_body)*\n };\n match atomic_compare_exchange_weak(dst, prev, next, order, Ordering::Relaxed) {\n Ok(x) => return x,\n Err(x) => prev = x,\n }\n }\n }\n }\n // If zacas is available at compile-time, we can always use zacas_fn.\n #[cfg(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"))]\n use self::$zacas_fn as $name;\n // Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.\n #[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n #[inline]\n unsafe fn $name(dst: *mut u128 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)?;\n zacas_relaxed_fn = $zacas_fn(Ordering::Relaxed);\n zacas_acquire_fn = $zacas_fn(Ordering::Acquire);\n zacas_release_fn = $zacas_fn(Ordering::Release);\n zacas_acqrel_fn = $zacas_fn(Ordering::AcqRel);\n zacas_seqcst_fn = $zacas_fn(Ordering::SeqCst);\n }\n // SAFETY: the caller must uphold the safety contract.\n // we only calls zacas_fn if zacas is available.\n unsafe {\n match order {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {\n if detect::detect().zacas() {\n zacas_relaxed_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::Acquire => {\n ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {\n if detect::detect().zacas() {\n zacas_acquire_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::Release => {\n ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {\n if detect::detect().zacas() {\n zacas_release_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::AcqRel => {\n ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {\n if detect::detect().zacas() {\n zacas_acqrel_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(dst: *mut u128 $(, $($arg)*)?) $(-> $ret_ty)? {\n if detect::detect().zacas() {\n zacas_seqcst_fn\n } else {\n fallback::$seqcst_fallback_fn\n }\n })\n }\n _ => unreachable!(),\n }\n }\n }\n };\n}\n\nselect_atomic_rmw! {\n unsafe fn atomic_swap(dst: *mut u128, val: u128) -> u128 {\n |_x| val\n }\n zacas = atomic_swap_zacas;\n non_seqcst_fallback = atomic_swap_non_seqcst;\n seqcst_fallback = atomic_swap_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_add(dst: *mut u128, val: u128) -> u128 {\n |x| x.wrapping_add(val)\n }\n zacas = atomic_add_zacas;\n non_seqcst_fallback = atomic_add_non_seqcst;\n seqcst_fallback = atomic_add_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_sub(dst: *mut u128, val: u128) -> u128 {\n |x| x.wrapping_sub(val)\n }\n zacas = atomic_sub_zacas;\n non_seqcst_fallback = atomic_sub_non_seqcst;\n seqcst_fallback = atomic_sub_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_and(dst: *mut u128, val: u128) -> u128 {\n |x| x & val\n }\n zacas = atomic_and_zacas;\n non_seqcst_fallback = atomic_and_non_seqcst;\n seqcst_fallback = atomic_and_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_nand(dst: *mut u128, val: u128) -> u128 {\n |x| !(x & val)\n }\n zacas = atomic_nand_zacas;\n non_seqcst_fallback = atomic_nand_non_seqcst;\n seqcst_fallback = atomic_nand_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_or(dst: *mut u128, val: u128) -> u128 {\n |x| x | val\n }\n zacas = atomic_or_zacas;\n non_seqcst_fallback = atomic_or_non_seqcst;\n seqcst_fallback = atomic_or_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_xor(dst: *mut u128, val: u128) -> u128 {\n |x| x ^ val\n }\n zacas = atomic_xor_zacas;\n non_seqcst_fallback = atomic_xor_non_seqcst;\n seqcst_fallback = atomic_xor_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_max(dst: *mut u128, val: u128) -> u128 {\n |x| {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n { core::cmp::max(x as i128, val as i128) as u128 }\n }\n }\n zacas = atomic_max_zacas;\n non_seqcst_fallback = atomic_max_non_seqcst;\n seqcst_fallback = atomic_max_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_umax(dst: *mut u128, val: u128) -> u128 {\n |x| core::cmp::max(x, val)\n }\n zacas = atomic_umax_zacas;\n non_seqcst_fallback = atomic_umax_non_seqcst;\n seqcst_fallback = atomic_umax_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_min(dst: *mut u128, val: u128) -> u128 {\n |x| {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n { core::cmp::min(x as i128, val as i128) as u128 }\n }\n }\n zacas = atomic_min_zacas;\n non_seqcst_fallback = atomic_min_non_seqcst;\n seqcst_fallback = atomic_min_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_umin(dst: *mut u128, val: u128) -> u128 {\n |x| core::cmp::min(x, val)\n }\n zacas = atomic_umin_zacas;\n non_seqcst_fallback = atomic_umin_non_seqcst;\n seqcst_fallback = atomic_umin_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_not(dst: *mut u128) -> u128 {\n |x| !x\n }\n zacas = atomic_not_zacas;\n non_seqcst_fallback = atomic_not_non_seqcst;\n seqcst_fallback = atomic_not_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_neg(dst: *mut u128) -> u128 {\n |x| x.wrapping_neg()\n }\n zacas = atomic_neg_zacas;\n non_seqcst_fallback = atomic_neg_non_seqcst;\n seqcst_fallback = atomic_neg_seqcst;\n}\n\n#[inline]\nfn is_lock_free() -> bool {\n #[cfg(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"))]\n {\n // zacas is available at compile-time.\n true\n }\n #[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n {\n detect::detect().zacas()\n }\n}\nconst IS_ALWAYS_LOCK_FREE: bool =\n cfg!(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"));\n\natomic128!(AtomicI128, i128, atomic_max, atomic_min);\natomic128!(AtomicU128, u128, atomic_umax, atomic_umin);\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n test_atomic_int!(i128);\n test_atomic_int!(u128);\n\n // load/store/swap implementation is not affected by signedness, so it is\n // enough to test only unsigned types.\n stress_test!(u128);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic128/s390x.rs", "sha256": "b657e6f3e48d12b79dfaf7dec5268d2558967c14288e097bb37e40d49b9b6eba", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\n128-bit atomic implementation on s390x.\n\nThis architecture provides the following 128-bit atomic instructions:\n\n- LPQ/STPQ: load/store (arch1 or later, i.e., baseline)\n- CDSG: CAS (arch1 or later, i.e., baseline)\n\nSee \"Atomic operation overview by architecture\" in atomic-maybe-uninit for a more comprehensive and\ndetailed description of the atomic and synchronize instructions in this architecture:\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#s390x\n\nLLVM's minimal supported architecture level is arch8 (z10):\nhttps://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/SystemZ/SystemZProcessors.td#L16-L17\nThis does not appear to have changed since the current s390x backend was added in LLVM 3.3:\nhttps://github.com/llvm/llvm-project/commit/5f613dfd1f7edb0ae95d521b7107b582d9df5103#diff-cbaef692b3958312e80fd5507a7e2aff071f1acb086f10e8a96bc06a7bb289db\n\nWe could use asm byte_wise_atomic_load + asm atomic_compare_exchange pattern for RMW here, but using an\ninline assembly allows omitting the comparison of results and the storing/comparing of condition flags.\n\nNote: On Miri and ThreadSanitizer which do not support inline assembly, we don't use\nthis module and use intrinsics.rs instead.\n\nRefs:\n- z/Architecture Principles of Operation, Fifteenth Edition (SA22-7832-14)\n https://www.ibm.com/docs/en/module_1678991624569/pdf/SA22-7832-14.pdf\n- atomic-maybe-uninit\n https://github.com/taiki-e/atomic-maybe-uninit\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\ninclude!(\"macros.rs\");\n\nuse core::{arch::asm, sync::atomic::Ordering};\n\n#[cfg(portable_atomic_no_strict_provenance)]\nuse crate::utils::ptr::PtrExt as _;\nuse crate::utils::{Pair, U128};\n\n// bcr 14,0 requires fast-BCR-serialization facility added in arch9 (z196).\n#[cfg(any(\n target_feature = \"fast-serialization\",\n portable_atomic_target_feature = \"fast-serialization\",\n))]\nmacro_rules! serialization {\n () => {\n \"bcr 14, 0\"\n };\n}\n#[cfg(not(any(\n target_feature = \"fast-serialization\",\n portable_atomic_target_feature = \"fast-serialization\",\n)))]\nmacro_rules! serialization {\n () => {\n \"bcr 15, 0\"\n };\n}\n\n// Use distinct operands on z196 or later, otherwise split to lgr and $op.\n#[cfg(any(target_feature = \"distinct-ops\", portable_atomic_target_feature = \"distinct-ops\"))]\nmacro_rules! distinct_op {\n ($op:tt, $a0:tt, $a1:tt, $a2:tt) => {\n concat!($op, \"k \", $a0, \", \", $a1, \", \", $a2)\n };\n}\n#[cfg(not(any(target_feature = \"distinct-ops\", portable_atomic_target_feature = \"distinct-ops\")))]\nmacro_rules! distinct_op {\n ($op:tt, $a0:tt, $a1:tt, $a2:tt) => {\n concat!(\"lgr \", $a0, \", \", $a1, \"\\n\", $op, \" \", $a0, \", \", $a2)\n };\n}\n\n// Use selgr$cond on z15 or later, otherwise split to locgr$cond and $op.\n#[cfg(any(\n target_feature = \"miscellaneous-extensions-3\",\n portable_atomic_target_feature = \"miscellaneous-extensions-3\",\n))]\n#[cfg(any(\n target_feature = \"load-store-on-cond\",\n portable_atomic_target_feature = \"load-store-on-cond\",\n))]\nmacro_rules! select_op {\n ($cond:tt, $a0:tt, $a1:tt, $a2:tt) => {\n concat!(\"selgr\", $cond, \" \", $a0, \", \", $a1, \", \", $a2)\n };\n}\n#[cfg(not(any(\n target_feature = \"miscellaneous-extensions-3\",\n portable_atomic_target_feature = \"miscellaneous-extensions-3\",\n)))]\n#[cfg(any(\n target_feature = \"load-store-on-cond\",\n portable_atomic_target_feature = \"load-store-on-cond\",\n))]\nmacro_rules! select_op {\n ($cond:tt, $a0:tt, $a1:tt, $a2:tt) => {\n concat!(\"lgr \", $a0, \", \", $a2, \"\\n\", \"locgr\", $cond, \" \", $a0, \", \", $a1)\n };\n}\n\n// Extracts and checks condition code.\n#[inline]\nfn extract_cc(r: i64) -> bool {\n r.wrapping_add(-268435456) & (1 << 31) != 0\n}\n\n#[inline]\nunsafe fn atomic_load(src: *mut u128, _order: Ordering) -> u128 {\n debug_assert!(src.addr() % 16 == 0);\n let (out_hi, out_lo);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n // atomic load is always SeqCst.\n asm!(\n \"lpq %r0, 0({src})\", // atomic { r0:r1 = *src }\n src = in(reg) ptr_reg!(src),\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n out(\"r0\") out_hi,\n out(\"r1\") out_lo,\n options(nostack, preserves_flags),\n );\n U128 { pair: Pair { hi: out_hi, lo: out_lo } }.whole\n }\n}\n\n#[inline]\nunsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {\n debug_assert!(dst.addr() % 16 == 0);\n let val = U128 { whole: val };\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n macro_rules! atomic_store {\n ($acquire:expr) => {\n asm!(\n \"stpq %r0, 0({dst})\", // atomic { *dst = r0:r1 }\n $acquire, // fence\n dst = in(reg) ptr_reg!(dst),\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n in(\"r0\") val.pair.hi,\n in(\"r1\") val.pair.lo,\n options(nostack, preserves_flags),\n )\n };\n }\n match order {\n // Relaxed and Release stores are equivalent.\n Ordering::Relaxed | Ordering::Release => atomic_store!(\"\"),\n Ordering::SeqCst => atomic_store!(serialization!()),\n _ => unreachable!(),\n }\n }\n}\n\n#[inline]\nunsafe fn atomic_compare_exchange(\n dst: *mut u128,\n old: u128,\n new: u128,\n _success: Ordering,\n _failure: Ordering,\n) -> Result {\n debug_assert!(dst.addr() % 16 == 0);\n let old = U128 { whole: old };\n let new = U128 { whole: new };\n let (prev_hi, prev_lo);\n let r;\n\n // SAFETY: the caller must uphold the safety contract.\n // CDSG has SeqCst semantics.\n let prev = unsafe {\n macro_rules! cmpxchg {\n ($new_hi:tt, $new_lo:tt) => {\n asm!(\n concat!(\"cdsg %r0, %\", $new_hi, \", 0({dst})\"), // atomic { if *dst == r0:r1 { cc = 0; *dst = new_hi:new_lo } else { cc = 1; r0:r1 = *dst } }\n \"ipm {r}\", // r[:] = cc\n dst = in(reg) ptr_reg!(dst),\n r = lateout(reg) r,\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n inout(\"r0\") old.pair.hi => prev_hi,\n inout(\"r1\") old.pair.lo => prev_lo,\n in($new_hi) new.pair.hi,\n in($new_lo) new.pair.lo,\n // Do not use `preserves_flags` because CDSG modifies the condition code.\n options(nostack),\n )\n };\n }\n // r4 is stack pointer on z/OS.\n #[cfg(not(target_os = \"zos\"))]\n cmpxchg!(\"r4\", \"r5\");\n #[cfg(target_os = \"zos\")]\n cmpxchg!(\"r12\", \"r13\");\n U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole\n };\n if extract_cc(r) { Ok(prev) } else { Err(prev) }\n}\n\n// cdsg is always strong.\nuse self::atomic_compare_exchange as atomic_compare_exchange_weak;\n\n#[inline]\nunsafe fn atomic_swap(dst: *mut u128, val: u128, _order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n let val = U128 { whole: val };\n let (mut prev_hi, mut prev_lo);\n\n // SAFETY: the caller must uphold the safety contract.\n // CDSG has SeqCst semantics.\n //\n // Do not use atomic_rmw_cas_3 because it needs extra LGR to implement swap.\n unsafe {\n macro_rules! swap {\n ($val_hi:tt, $val_lo:tt) => {\n asm!(\n \"lg %r1, 8({dst})\", // atomic { r1 = *dst.byte_add(8) }\n \"lg %r0, 0({dst})\", // atomic { r0 = *dst }\n \"2:\", // 'retry:\n concat!(\"cdsg %r0, %\", $val_hi, \", 0({dst})\"), // atomic { if *dst == r0:r1 { cc = 0; *dst = val_hi:val_lo } else { cc = 1; r0:r1 = *dst } }\n \"jl 2b\", // if cc == 1 { jump 'retry }\n dst = in(reg) ptr_reg!(dst),\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n out(\"r0\") prev_hi,\n out(\"r1\") prev_lo,\n in($val_hi) val.pair.hi,\n in($val_lo) val.pair.lo,\n // Do not use `preserves_flags` because CDSG modifies the condition code.\n options(nostack),\n )\n };\n }\n // r4 is stack pointer on z/OS.\n #[cfg(not(target_os = \"zos\"))]\n swap!(\"r4\", \"r5\");\n #[cfg(target_os = \"zos\")]\n swap!(\"r12\", \"r13\");\n U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole\n }\n}\n\n/// Atomic RMW by CAS loop (3 arguments)\n/// `unsafe fn(dst: *mut u128, val: u128, order: Ordering) -> u128;`\n///\n/// `$op` can use the following registers:\n/// - val_hi/val_lo pair: val argument (read-only for `$op`)\n/// - r0/r1 pair: previous value loaded (read-only for `$op`)\n/// - r12/r13 pair: new value that will be stored\nmacro_rules! atomic_rmw_cas_3 {\n ($name:ident, [$($reg:tt)*], $($op:tt)*) => {\n #[inline]\n unsafe fn $name(dst: *mut u128, val: u128, _order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n let val = U128 { whole: val };\n let (mut prev_hi, mut prev_lo);\n\n // SAFETY: the caller must uphold the safety contract.\n // CDSG has SeqCst semantics.\n unsafe {\n asm!(\n \"lg %r1, 8({dst})\", // atomic { r1 = *dst.byte_add(8) }\n \"lg %r0, 0({dst})\", // atomic { r0 = *dst }\n \"2:\", // 'retry:\n $($op)*\n \"cdsg %r0, %r12, 0({dst})\", // atomic { if *dst == r0:r1 { cc = 0; *dst = r12:r13 } else { cc = 1; r0:r1 = *dst } }\n \"jl 2b\", // if cc == 1 { jump 'retry }\n dst = in(reg) ptr_reg!(dst),\n val_hi = in(reg) val.pair.hi,\n val_lo = in(reg) val.pair.lo,\n $($reg)*\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n out(\"r0\") prev_hi,\n out(\"r1\") prev_lo,\n out(\"r12\") _,\n out(\"r13\") _,\n // Do not use `preserves_flags` because CDSG modifies the condition code.\n options(nostack),\n );\n U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole\n }\n }\n };\n}\n/// Atomic RMW by CAS loop (2 arguments)\n/// `unsafe fn(dst: *mut u128, order: Ordering) -> u128;`\n///\n/// `$op` can use the following registers:\n/// - r0/r1 pair: previous value loaded (read-only for `$op`)\n/// - r12/r13 pair: new value that will be stored\nmacro_rules! atomic_rmw_cas_2 {\n ($name:ident, [$($reg:tt)*], $($op:tt)*) => {\n #[inline]\n unsafe fn $name(dst: *mut u128, _order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n let (mut prev_hi, mut prev_lo);\n\n // SAFETY: the caller must uphold the safety contract.\n // CDSG has SeqCst semantics.\n unsafe {\n asm!(\n \"lg %r1, 8({dst})\", // atomic { r1 = *dst.byte_add(8) }\n \"lg %r0, 0({dst})\", // atomic { r0 = *dst }\n \"2:\", // 'retry:\n $($op)*\n \"cdsg %r0, %r12, 0({dst})\", // atomic { if *dst == r0:r1 { cc = 0; *dst = r12:r13 } else { cc = 1; r0:r1 = *dst } }\n \"jl 2b\", // if cc == 1 { jump 'retry }\n dst = in(reg) ptr_reg!(dst),\n $($reg)*\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n out(\"r0\") prev_hi,\n out(\"r1\") prev_lo,\n out(\"r12\") _,\n out(\"r13\") _,\n // Do not use `preserves_flags` because CDSG modifies the condition code.\n options(nostack),\n );\n U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole\n }\n }\n };\n}\n\natomic_rmw_cas_3! {\n atomic_add, [],\n distinct_op!(\"algr\", \"%r13\", \"%r1\", \"{val_lo}\"), // r13 = r1 + val_lo; cc = zero | carry\n \"lgr %r12, %r0\", // r12 = r0\n \"alcgr %r12, {val_hi}\", // r12 += val_hi + carry\n}\natomic_rmw_cas_3! {\n atomic_sub, [],\n distinct_op!(\"slgr\", \"%r13\", \"%r1\", \"{val_lo}\"), // r13 = r1 - val_lo; cc = zero | borrow\n \"lgr %r12, %r0\", // r12 = r0\n \"slbgr %r12, {val_hi}\", // r12 -= val_hi + borrow\n}\natomic_rmw_cas_3! {\n atomic_and, [],\n distinct_op!(\"ngr\", \"%r13\", \"%r1\", \"{val_lo}\"), // r13 = r1 & val_lo\n distinct_op!(\"ngr\", \"%r12\", \"%r0\", \"{val_hi}\"), // r12 = r0 & val_hi\n}\n\n// Use nngrk on z15 or later.\n#[cfg(any(\n target_feature = \"miscellaneous-extensions-3\",\n portable_atomic_target_feature = \"miscellaneous-extensions-3\",\n))]\natomic_rmw_cas_3! {\n atomic_nand, [],\n \"nngrk %r13, %r1, {val_lo}\", // r13 = !(r1 & val_lo)\n \"nngrk %r12, %r0, {val_hi}\", // r12 = !(r0 & val_hi)\n}\n#[cfg(not(any(\n target_feature = \"miscellaneous-extensions-3\",\n portable_atomic_target_feature = \"miscellaneous-extensions-3\",\n)))]\natomic_rmw_cas_3! {\n atomic_nand, [],\n distinct_op!(\"ngr\", \"%r13\", \"%r1\", \"{val_lo}\"), // r13 = r1 & val_lo\n distinct_op!(\"ngr\", \"%r12\", \"%r0\", \"{val_hi}\"), // r12 = r0 & val_hi\n \"lcgr %r13, %r13\", // r13 = !r13 + 1\n \"aghi %r13, -1\", // r13 -= 1\n \"lcgr %r12, %r12\", // r12 = !r12 + 1\n \"aghi %r12, -1\", // r12 -= 1\n}\n\natomic_rmw_cas_3! {\n atomic_or, [],\n distinct_op!(\"ogr\", \"%r13\", \"%r1\", \"{val_lo}\"), // r13 = r1 | val_lo\n distinct_op!(\"ogr\", \"%r12\", \"%r0\", \"{val_hi}\"), // r12 = r0 | val_hi\n}\natomic_rmw_cas_3! {\n atomic_xor, [],\n distinct_op!(\"xgr\", \"%r13\", \"%r1\", \"{val_lo}\"), // r13 = r1 ^ val_lo\n distinct_op!(\"xgr\", \"%r12\", \"%r0\", \"{val_hi}\"), // r12 = r0 ^ val_hi\n}\n\ncfg_sel!({\n #[cfg(any(\n target_feature = \"load-store-on-cond\",\n portable_atomic_target_feature = \"load-store-on-cond\",\n ))]\n {\n atomic_rmw_cas_3! {\n atomic_max, [],\n \"clgr %r1, {val_lo}\", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }\n select_op!(\"h\", \"%r12\", \"%r1\", \"{val_lo}\"), // if cc == 2 { r12 = r1 } else { r12 = val_lo }\n \"cgr %r0, {val_hi}\", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }\n select_op!(\"h\", \"%r13\", \"%r1\", \"{val_lo}\"), // if cc == 2 { r13 = r1 } else { r13 = val_lo }\n \"locgre %r13, %r12\", // if cc == 0 { r13 = r12 }\n select_op!(\"h\", \"%r12\", \"%r0\", \"{val_hi}\"), // if cc == 2 { r12 = r0 } else { r12 = val_hi }\n }\n atomic_rmw_cas_3! {\n atomic_umax, [tmp = out(reg) _,],\n \"clgr %r1, {val_lo}\", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }\n select_op!(\"h\", \"{tmp}\", \"%r1\", \"{val_lo}\"), // if cc == 2 { tmp = r1 } else { tmp = val_lo }\n \"clgr %r0, {val_hi}\", // if r0(u) < val_hi(u) { cc = 1 } else if r0(u) > val_hi(u) { cc = 2 } else { cc = 0 }\n select_op!(\"h\", \"%r12\", \"%r0\", \"{val_hi}\"), // if cc == 2 { r12 = r0 } else { r12 = val_hi }\n select_op!(\"h\", \"%r13\", \"%r1\", \"{val_lo}\"), // if cc == 2 { r13 = r1 } else { r13 = val_lo }\n \"cgr %r0, {val_hi}\", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }\n \"locgre %r13, {tmp}\", // if cc == 0 { r13 = tmp }\n }\n atomic_rmw_cas_3! {\n atomic_min, [],\n \"clgr %r1, {val_lo}\", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }\n select_op!(\"l\", \"%r12\", \"%r1\", \"{val_lo}\"), // if cc == 1 { r12 = r1 } else { r12 = val_lo }\n \"cgr %r0, {val_hi}\", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }\n select_op!(\"l\", \"%r13\", \"%r1\", \"{val_lo}\"), // if cc == 1 { r13 = r1 } else { r13 = val_lo }\n \"locgre %r13, %r12\", // if cc == 0 { r13 = r12 }\n select_op!(\"l\", \"%r12\", \"%r0\", \"{val_hi}\"), // if cc == 1 { r12 = r0 } else { r12 = val_hi }\n }\n atomic_rmw_cas_3! {\n atomic_umin, [tmp = out(reg) _,],\n \"clgr %r1, {val_lo}\", // if r1(u) < val_lo(u) { cc = 1 } else if r1(u) > val_lo(u) { cc = 2 } else { cc = 0 }\n select_op!(\"l\", \"{tmp}\", \"%r1\", \"{val_lo}\"), // if cc == 1 { tmp = r1 } else { tmp = val_lo }\n \"clgr %r0, {val_hi}\", // if r0(u) < val_hi(u) { cc = 1 } else if r0(u) > val_hi(u) { cc = 2 } else { cc = 0 }\n select_op!(\"l\", \"%r12\", \"%r0\", \"{val_hi}\"), // if cc == 1 { r12 = r0 } else { r12 = val_hi }\n select_op!(\"l\", \"%r13\", \"%r1\", \"{val_lo}\"), // if cc == 1 { r13 = r1 } else { r13 = val_lo }\n \"cgr %r0, {val_hi}\", // if r0(i) < val_hi(i) { cc = 1 } else if r0(i) > val_hi(i) { cc = 2 } else { cc = 0 }\n \"locgre %r13, {tmp}\", // if cc == 0 { r13 = tmp }\n }\n }\n #[cfg(else)]\n {\n #[rustfmt::skip]\n macro_rules! atomic_cmp {\n ($name:ident, $cmp_hi1:tt, $cmp_hi2:tt, $cmp_lo:tt,) => {\n #[inline]\n unsafe fn $name(dst: *mut u128, val: u128, _order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n let val = U128 { whole: val };\n let (mut prev_hi, mut prev_lo);\n\n // SAFETY: the caller must uphold the safety contract.\n // CDSG has SeqCst semantics.\n unsafe {\n macro_rules! cmp {\n ($val_hi:tt, $val_lo:tt) => {\n asm!(\n \"lg %r1, 8({dst})\", // atomic { r1 = *dst.byte_add(8) }\n \"lg %r0, 0({dst})\", // atomic { r0 = *dst }\n \"2:\", // 'retry:\n concat!($cmp_hi1, \" %\", $val_hi, \", %r0, 4f\"), // if $cmp_hi1(val_hi, r0) { jump 'val }\n concat!($cmp_hi2, \" %\", $val_hi, \", %r0, 3f\"), // if $cmp_hi2(val_hi, r0) { jump 'prev }\n concat!($cmp_lo, \" %\", $val_lo, \", %r1, 4f\"), // if $cmp_lo(val_lo, r1) { jump 'val }\n \"3:\", // 'prev:\n \"cdsg %r0, %r0, 0({dst})\", // atomic { if *dst == r0:r1 { cc = 0; *dst = r0:r1 } else { cc = 1; r0:r1 = *dst } }\n \"jl 2b\", // if cc == 1 { jump 'retry }\n \"j 5f\",\n \"4:\", // 'val:\n concat!(\"cdsg %r0, %\", $val_hi, \", 0({dst})\"), // atomic { if *dst == r0:r1 { cc = 0; *dst = val_hi:val_lo } else { cc = 1; r0:r1 = *dst } }\n \"jl 2b\", // if cc == 1 { jump 'retry }\n \"5:\",\n dst = in(reg) ptr_reg!(dst),\n // Quadword atomic instructions work with even/odd pair of specified register and subsequent register.\n out(\"r0\") prev_hi,\n out(\"r1\") prev_lo,\n in($val_hi) val.pair.hi,\n in($val_lo) val.pair.lo,\n // Do not use `preserves_flags` because CDSG modifies the condition code.\n options(nostack),\n )\n };\n }\n // r4 is stack pointer on z/OS.\n #[cfg(not(target_os = \"zos\"))]\n cmp!(\"r4\", \"r5\");\n #[cfg(target_os = \"zos\")]\n cmp!(\"r12\", \"r13\");\n U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole\n }\n }\n };\n }\n // loc{,g}r requires z196 or later.\n atomic_cmp! {\n atomic_max,\n \"cgrjh\", // signed >\n \"cgrjl\", // signed <\n \"clgrjhe\", // unsigned >=\n }\n atomic_cmp! {\n atomic_umax,\n \"clgrjh\", // unsigned >\n \"clgrjl\", // unsigned <\n \"clgrjhe\", // unsigned >=\n }\n atomic_cmp! {\n atomic_min,\n \"cgrjl\", // signed <\n \"cgrjh\", // signed >\n \"clgrjle\", // unsigned <=\n }\n atomic_cmp! {\n atomic_umin,\n \"clgrjl\", // unsigned <\n \"clgrjh\", // unsigned >\n \"clgrjle\", // unsigned <=\n }\n }\n});\n\natomic_rmw_cas_2! {\n atomic_not, [],\n \"lcgr %r13, %r1\", // r13 = !r1 + 1\n \"aghi %r13, -1\", // r13 -= 1\n \"lcgr %r12, %r0\", // r12 = !r0 + 1\n \"aghi %r12, -1\", // r12 -= 1\n}\n\n#[cfg(any(target_feature = \"distinct-ops\", portable_atomic_target_feature = \"distinct-ops\"))]\natomic_rmw_cas_2! {\n atomic_neg, [zero = in(reg) 0_u64,],\n \"slgrk %r13, {zero}, %r1\", // r13 = 0 - r1; cc = zero | borrow\n \"lghi %r12, 0\", // r12 = 0\n \"slbgr %r12, %r0\", // r12 -= r0 + borrow\n}\n#[cfg(not(any(target_feature = \"distinct-ops\", portable_atomic_target_feature = \"distinct-ops\")))]\natomic_rmw_cas_2! {\n atomic_neg, [],\n \"lghi %r13, 0\", // r13 = 0\n \"slgr %r13, %r1\", // r13 -= r1; cc = zero | borrow\n \"lghi %r12, 0\", // r12 = 0\n \"slbgr %r12, %r0\", // r12 -= r0 + borrow\n}\n\n#[inline]\nconst fn is_lock_free() -> bool {\n IS_ALWAYS_LOCK_FREE\n}\nconst IS_ALWAYS_LOCK_FREE: bool = true;\n\natomic128!(AtomicI128, i128, atomic_max, atomic_min);\natomic128!(AtomicU128, u128, atomic_umax, atomic_umin);\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n test_atomic_int!(i128);\n test_atomic_int!(u128);\n\n // load/store/swap implementation is not affected by signedness, so it is\n // enough to test only unsigned types.\n stress_test!(u128);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic128/x86_64.rs", "sha256": "19c67a75d8ee1392aae67f49b88a3b69e78acce5004280c7a7be33051774c8ed", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\n128-bit atomic implementation on x86_64.\n\nThis architecture provides the following 128-bit atomic instructions:\n\n- CMPXCHG16B: CAS (CMPXCHG16B)\n- VMOVDQA: load/store (AVX)\n\nWe could use asm byte_wise_atomic_load + asm atomic_compare_exchange pattern for RMW here, but using an\ninline assembly allows omitting the comparison of results and the storing/comparing of flags.\n\nNote: On Miri and ThreadSanitizer which do not support inline assembly, we don't use\nthis module and use intrinsics.rs instead.\n\nRefs:\n- x86 and amd64 instruction reference https://www.felixcloutier.com/x86\n- atomic-maybe-uninit https://github.com/taiki-e/atomic-maybe-uninit\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\n// TODO: use core::arch::x86_64::cmpxchg16b where available and efficient than asm\n\ninclude!(\"macros.rs\");\n\n#[cfg(not(any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\")))]\n#[path = \"../fallback/outline_atomics.rs\"]\nmod fallback;\n\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(not(target_env = \"sgx\"))]\n#[cfg_attr(\n not(target_feature = \"sse\"),\n cfg(not(any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\")))\n)]\n#[cfg(any(\n test,\n not(all(\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n target_feature = \"avx\",\n )),\n))]\n#[path = \"../detect/x86_64.rs\"]\nmod detect;\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\n#[cfg(not(all(\n not(target_feature = \"avx\"),\n any(portable_atomic_no_outline_atomics, target_env = \"sgx\", not(target_feature = \"sse\")),\n)))]\nuse core::arch::x86_64::__m128i;\nuse core::sync::atomic::Ordering;\n\n#[cfg(portable_atomic_no_strict_provenance)]\nuse crate::utils::ptr::PtrExt as _;\nuse crate::utils::{Pair, U128};\n\n// Asserts that the function is called in the correct context.\nmacro_rules! debug_assert_cmpxchg16b {\n () => {\n #[cfg(not(any(\n target_feature = \"cmpxchg16b\",\n portable_atomic_target_feature = \"cmpxchg16b\",\n )))]\n {\n debug_assert!(detect::detect().cmpxchg16b());\n }\n };\n}\n#[cfg(not(all(\n not(target_feature = \"avx\"),\n any(portable_atomic_no_outline_atomics, target_env = \"sgx\", not(target_feature = \"sse\")),\n)))]\nmacro_rules! debug_assert_cmpxchg16b_avx {\n () => {{\n debug_assert_cmpxchg16b!();\n #[cfg(not(target_feature = \"avx\"))]\n {\n debug_assert!(detect::detect().avx());\n }\n }};\n}\n\n#[cfg(not(all(\n not(target_feature = \"avx\"),\n any(portable_atomic_no_outline_atomics, target_env = \"sgx\", not(target_feature = \"sse\")),\n)))]\n#[cfg(target_pointer_width = \"32\")]\nmacro_rules! ptr_modifier {\n () => {\n \":e\"\n };\n}\n#[cfg(not(all(\n not(target_feature = \"avx\"),\n any(portable_atomic_no_outline_atomics, target_env = \"sgx\", not(target_feature = \"sse\")),\n)))]\n#[cfg(target_pointer_width = \"64\")]\nmacro_rules! ptr_modifier {\n () => {\n \"\"\n };\n}\n\n// Unlike AArch64 and RISC-V, x86's assembler doesn't check instruction\n// requirements for the currently enabled target features. In the first place,\n// there is no option in the x86 assembly for such case, like Arm .arch_extension,\n// RISC-V .option arch, PowerPC .machine, etc.\n// However, we set target_feature(enable) when available (Rust 1.69+) in case a\n// new codegen backend is added that checks for it in the future, or an option\n// is added to the assembler to check for it.\n#[cfg_attr(\n not(portable_atomic_no_cmpxchg16b_target_feature),\n target_feature(enable = \"cmpxchg16b\")\n)]\n#[inline]\nunsafe fn cmpxchg16b(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_cmpxchg16b!();\n\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned (required by CMPXCHG16B), that there are no\n // concurrent non-atomic operations, and that the CPU supports CMPXCHG16B.\n //\n // If the value at `dst` (destination operand) and rdx:rax are equal, the\n // 128-bit value in rcx:rbx is stored in the `dst`, otherwise the value at\n // `dst` is loaded to rdx:rax.\n //\n // The ZF flag is set if the value at `dst` and rdx:rax are equal,\n // otherwise it is cleared. Other flags are unaffected.\n //\n // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b\n unsafe {\n // cmpxchg16b is always SeqCst.\n let r: u8;\n let old = U128 { whole: old };\n let new = U128 { whole: new };\n let (prev_lo, prev_hi);\n macro_rules! cmpxchg16b {\n ($dst:tt) => {\n asm!(\n \"xchg r8, rbx\", // save rbx which is reserved by LLVM\n concat!(\"lock cmpxchg16b xmmword ptr [\", $dst, \"]\"),\n \"setne cl\",\n \"mov rbx, r8\", // restore rbx\n inout(\"r8\") new.pair.lo => _,\n in(\"rcx\") new.pair.hi,\n inout(\"rax\") old.pair.lo => prev_lo,\n inout(\"rdx\") old.pair.hi => prev_hi,\n in($dst) dst,\n lateout(\"cl\") r,\n // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.\n options(nostack),\n )\n };\n }\n // rdi and rsi are call-preserved on Windows.\n #[cfg(not(windows))]\n #[cfg(target_pointer_width = \"32\")]\n cmpxchg16b!(\"esi\");\n #[cfg(not(windows))]\n #[cfg(target_pointer_width = \"64\")]\n cmpxchg16b!(\"rsi\");\n #[cfg(windows)]\n #[cfg(target_pointer_width = \"32\")]\n cmpxchg16b!(\"r11d\");\n #[cfg(windows)]\n #[cfg(target_pointer_width = \"64\")]\n cmpxchg16b!(\"r11\");\n crate::utils::assert_unchecked(r == 0 || r == 1); // needed to remove extra test\n (U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole, r == 0)\n }\n}\n\n// VMOVDQA is atomic when AVX available.\n// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 for details.\n// Intel, AMD, and Zhaoxin officially guarantee this behavior (VIA has not responded),\n// but both LLVM and Boost always treat VMOVDQA as atomic when AVX available.\n// https://github.com/llvm/llvm-project/pull/74275\n// https://github.com/boostorg/atomic/commit/24a41db3e61627d99895f7e324b3d725d1be27c1\n//\n// Refs: https://www.felixcloutier.com/x86/movdqa:vmovdqa32:vmovdqa64\n//\n// Use cfg(target_feature = \"sse\") here -- SSE is included in the x86_64\n// baseline and is always available, but the SSE target feature is disabled for\n// use cases such as kernels and firmware that should not use vector registers.\n// So, do not use vector registers unless SSE target feature is enabled.\n// Note that we cannot use cfg(target_abi = \"softfloat\") because cfg(target_abi)\n// is unavailable on old rustc and target_abi = \"softfloat\" is not set for\n// softfloat x86_64 targets as of Rust 1.95.\n// See also https://github.com/rust-lang/rust/blob/1.84.0/src/doc/rustc/src/platform-support/x86_64-unknown-none.md.\n#[cfg(not(all(\n not(target_feature = \"avx\"),\n any(portable_atomic_no_outline_atomics, target_env = \"sgx\", not(target_feature = \"sse\")),\n)))]\n#[target_feature(enable = \"avx\")]\n#[inline]\nunsafe fn _atomic_load_vmovdqa(src: *mut u128) -> u128 {\n debug_assert!(src.addr() % 16 == 0);\n debug_assert_cmpxchg16b_avx!();\n\n // SAFETY: the caller must uphold the safety contract.\n //\n // atomic load by vmovdqa is always SeqCst.\n unsafe {\n let out: __m128i;\n asm!(\n concat!(\"vmovdqa {out}, xmmword ptr [{src\", ptr_modifier!(), \"}]\"),\n src = in(reg) src,\n out = out(xmm_reg) out,\n options(nostack, preserves_flags),\n );\n core::mem::transmute(out)\n }\n}\n#[cfg(not(all(\n not(target_feature = \"avx\"),\n any(portable_atomic_no_outline_atomics, target_env = \"sgx\", not(target_feature = \"sse\")),\n)))]\n#[target_feature(enable = \"avx\")]\n#[inline]\nunsafe fn _atomic_store_vmovdqa(dst: *mut u128, val: u128, order: Ordering) {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_cmpxchg16b_avx!();\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n let val: __m128i = core::mem::transmute(val);\n match order {\n // Relaxed and Release stores are equivalent.\n Ordering::Relaxed | Ordering::Release => {\n asm!(\n concat!(\"vmovdqa xmmword ptr [{dst\", ptr_modifier!(), \"}], {val}\"),\n dst = in(reg) dst,\n val = in(xmm_reg) val,\n options(nostack, preserves_flags),\n );\n }\n Ordering::SeqCst => {\n let p = core::cell::UnsafeCell::new(core::mem::MaybeUninit::::uninit());\n asm!(\n concat!(\"vmovdqa xmmword ptr [{dst\", ptr_modifier!(), \"}], {val}\"),\n // Equivalent to `mfence`, but is up to 3.1x faster on Coffee Lake and up to 2.4x faster on Raptor Lake-H at least in simple cases.\n // - https://github.com/taiki-e/portable-atomic/pull/156\n // - LLVM uses `lock or` https://godbolt.org/z/vv6rjzfYd\n // - Windows uses `xchg` for x86_32 for MemoryBarrier https://learn.microsoft.com/en-us/windows/win32/api/winnt/nf-winnt-memorybarrier\n // - MSVC STL uses `lock inc` https://github.com/microsoft/STL/pull/740\n // - boost uses `lock or` https://github.com/boostorg/atomic/commit/559eba81af71386cedd99f170dc6101c6ad7bf22\n concat!(\"xchg qword ptr [{p\", ptr_modifier!(), \"}], {tmp}\"),\n dst = in(reg) dst,\n val = in(xmm_reg) val,\n p = in(reg) p.get(),\n tmp = out(reg) _,\n options(nostack, preserves_flags),\n );\n }\n _ => unreachable!(),\n }\n }\n}\n\n#[cfg(not(all(\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n target_feature = \"avx\",\n)))]\n#[cfg(not(all(\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n any(portable_atomic_no_outline_atomics, target_env = \"sgx\", not(target_feature = \"sse\")),\n)))]\nmacro_rules! load_store_detect {\n (\n vmovdqa = $vmovdqa:ident\n cmpxchg16b = $cmpxchg16b:ident\n fallback = $fallback:ident\n ) => {{\n let cpuid = detect::detect();\n #[cfg(not(any(\n target_feature = \"cmpxchg16b\",\n portable_atomic_target_feature = \"cmpxchg16b\",\n )))]\n {\n // Check CMPXCHG16B first to prevent mixing atomic and non-atomic access.\n if cpuid.cmpxchg16b() {\n #[cfg(target_feature = \"avx\")]\n {\n $vmovdqa\n }\n // We only use VMOVDQA when SSE is enabled. See _atomic_load_vmovdqa() for more.\n #[cfg(not(target_feature = \"avx\"))]\n {\n #[cfg(target_feature = \"sse\")]\n {\n if cpuid.avx() { $vmovdqa } else { $cmpxchg16b }\n }\n #[cfg(not(target_feature = \"sse\"))]\n {\n $cmpxchg16b\n }\n }\n } else {\n fallback::$fallback\n }\n }\n #[cfg(any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"))]\n {\n if cpuid.avx() { $vmovdqa } else { $cmpxchg16b }\n }\n }};\n}\n\n#[inline]\nunsafe fn atomic_load(src: *mut u128, _order: Ordering) -> u128 {\n #[cfg(all(\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n target_feature = \"avx\",\n ))]\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantees that CMPXCHG16B and AVX are available at compile-time.\n unsafe {\n _atomic_load_vmovdqa(src)\n }\n #[cfg(not(all(\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n target_feature = \"avx\",\n )))]\n {\n // We only use VMOVDQA when SSE is enabled. See _atomic_load_vmovdqa() for more.\n // SGX doesn't support CPUID.\n #[cfg(all(\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n any(\n portable_atomic_no_outline_atomics,\n target_env = \"sgx\",\n not(target_feature = \"sse\")\n ),\n ))]\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantees that CMPXCHG16B is available at compile-time.\n unsafe {\n // cmpxchg16b is always SeqCst.\n _atomic_load_cmpxchg16b(src)\n }\n #[cfg(not(all(\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n any(\n portable_atomic_no_outline_atomics,\n target_env = \"sgx\",\n not(target_feature = \"sse\")\n ),\n )))]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n ifunc!(unsafe fn(src: *mut u128) -> u128 {\n load_store_detect! {\n vmovdqa = _atomic_load_vmovdqa\n cmpxchg16b = _atomic_load_cmpxchg16b\n // Use SeqCst because cmpxchg16b and atomic load by vmovdqa is always SeqCst.\n fallback = atomic_load_seqcst\n }\n })\n }\n }\n}\n// See cmpxchg16b() for target_feature(enable).\n#[cfg_attr(\n not(portable_atomic_no_cmpxchg16b_target_feature),\n target_feature(enable = \"cmpxchg16b\")\n)]\n#[inline]\nunsafe fn _atomic_load_cmpxchg16b(src: *mut u128) -> u128 {\n debug_assert!(src.addr() % 16 == 0);\n debug_assert_cmpxchg16b!();\n\n // SAFETY: the caller must guarantee that `src` is valid for both writes and\n // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.\n // cfg guarantees that the CPU supports CMPXCHG16B.\n //\n // See cmpxchg16b function for more.\n //\n // We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows\n // omitting the storing of condition flags and avoid use of xchg to handle rbx.\n unsafe {\n // cmpxchg16b is always SeqCst.\n let (out_lo, out_hi);\n macro_rules! cmpxchg16b {\n ($dst:tt, $save:tt) => {\n asm!(\n concat!(\"mov \", $save, \", rbx\"), // save rbx which is reserved by LLVM\n \"xor rbx, rbx\", // zeroed rbx\n concat!(\"lock cmpxchg16b xmmword ptr [\", $dst, \"]\"),\n concat!(\"mov rbx, \", $save), // restore rbx\n // set old/new args of cmpxchg16b to 0 (rbx is zeroed after saved to rbx_tmp, to avoid xchg)\n out($save) _,\n in(\"rcx\") 0_u64,\n inout(\"rax\") 0_u64 => out_lo,\n inout(\"rdx\") 0_u64 => out_hi,\n in($dst) src,\n // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.\n options(nostack),\n )\n };\n }\n // rdi and rsi are call-preserved on Windows.\n #[cfg(not(windows))]\n #[cfg(target_pointer_width = \"32\")]\n cmpxchg16b!(\"edi\", \"rsi\");\n #[cfg(not(windows))]\n #[cfg(target_pointer_width = \"64\")]\n cmpxchg16b!(\"rdi\", \"rsi\");\n #[cfg(windows)]\n #[cfg(target_pointer_width = \"32\")]\n cmpxchg16b!(\"r9d\", \"r8\");\n #[cfg(windows)]\n #[cfg(target_pointer_width = \"64\")]\n cmpxchg16b!(\"r9\", \"r8\");\n U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole\n }\n}\n\n#[inline]\nunsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {\n #[cfg(all(\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n target_feature = \"avx\",\n ))]\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantees that CMPXCHG16B and AVX are available at compile-time.\n unsafe {\n _atomic_store_vmovdqa(dst, val, order);\n }\n #[cfg(not(all(\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n target_feature = \"avx\",\n )))]\n {\n // We only use VMOVDQA when SSE is enabled. See _atomic_load_vmovdqa() for more.\n // SGX doesn't support CPUID.\n #[cfg(all(\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n any(\n portable_atomic_no_outline_atomics,\n target_env = \"sgx\",\n not(target_feature = \"sse\")\n ),\n ))]\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantees that CMPXCHG16B is available at compile-time.\n unsafe {\n // cmpxchg16b is always SeqCst.\n let _ = order;\n _atomic_store_cmpxchg16b(dst, val);\n }\n #[cfg(not(all(\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n any(\n portable_atomic_no_outline_atomics,\n target_env = \"sgx\",\n not(target_feature = \"sse\")\n ),\n )))]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n #[cfg(target_feature = \"sse\")]\n fn_alias! {\n #[target_feature(enable = \"avx\")]\n unsafe fn(dst: *mut u128, val: u128);\n // atomic store by vmovdqa has at least release semantics.\n _atomic_store_vmovdqa_non_seqcst = _atomic_store_vmovdqa(Ordering::Release);\n _atomic_store_vmovdqa_seqcst = _atomic_store_vmovdqa(Ordering::SeqCst);\n }\n match order {\n // Relaxed and Release stores are equivalent in all implementations\n // that may be called here (vmovdqa, asm-based cmpxchg16b, and fallback).\n // core::arch's cmpxchg16b will never called here.\n Ordering::Relaxed | Ordering::Release => {\n ifunc!(unsafe fn(dst: *mut u128, val: u128) {\n load_store_detect! {\n vmovdqa = _atomic_store_vmovdqa_non_seqcst\n cmpxchg16b = _atomic_store_cmpxchg16b\n fallback = atomic_store_non_seqcst\n }\n });\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(dst: *mut u128, val: u128) {\n load_store_detect! {\n vmovdqa = _atomic_store_vmovdqa_seqcst\n cmpxchg16b = _atomic_store_cmpxchg16b\n fallback = atomic_store_seqcst\n }\n });\n }\n _ => unreachable!(),\n }\n }\n }\n}\n// See cmpxchg16b() for target_feature(enable).\n#[cfg_attr(\n not(portable_atomic_no_cmpxchg16b_target_feature),\n target_feature(enable = \"cmpxchg16b\")\n)]\n#[inline]\nunsafe fn _atomic_store_cmpxchg16b(dst: *mut u128, val: u128) {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n // cmpxchg16b is always SeqCst.\n atomic_swap_cmpxchg16b(dst, val, Ordering::SeqCst);\n }\n}\n\n#[inline]\nunsafe fn atomic_compare_exchange(\n dst: *mut u128,\n old: u128,\n new: u128,\n _success: Ordering,\n _failure: Ordering,\n) -> Result {\n #[cfg(any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"))]\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, that there are no concurrent non-atomic operations,\n // and cfg guarantees that CMPXCHG16B is available at compile-time.\n let (prev, ok) = unsafe { cmpxchg16b(dst, old, new) };\n #[cfg(not(any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\")))]\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, and that there are no different kinds of concurrent accesses.\n let (prev, ok) = unsafe {\n ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {\n if detect::detect().cmpxchg16b() {\n cmpxchg16b\n } else {\n // Use SeqCst because cmpxchg16b is always SeqCst.\n fallback::atomic_compare_exchange_seqcst\n }\n })\n };\n if ok { Ok(prev) } else { Err(prev) }\n}\n\n// cmpxchg16b is always strong.\nuse self::atomic_compare_exchange as atomic_compare_exchange_weak;\n\n// See cmpxchg16b() for target_feature(enable).\n#[cfg_attr(\n not(portable_atomic_no_cmpxchg16b_target_feature),\n target_feature(enable = \"cmpxchg16b\")\n)]\n#[inline]\nunsafe fn atomic_swap_cmpxchg16b(dst: *mut u128, val: u128, _order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_cmpxchg16b!();\n\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.\n // cfg guarantees that the CPU supports CMPXCHG16B.\n //\n // See cmpxchg16b function for more.\n //\n // We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows\n // omitting the storing/comparing of condition flags and reducing uses of xchg/mov to handle rbx.\n //\n // Do not use atomic_rmw_cas_3 because it needs extra MOV to implement swap.\n unsafe {\n // cmpxchg16b is always SeqCst.\n let val = U128 { whole: val };\n let (mut prev_lo, mut prev_hi);\n macro_rules! cmpxchg16b {\n ($dst:tt, $save:tt) => {\n asm!(\n concat!(\"xchg \", $save, \", rbx\"), // save rbx which is reserved by LLVM\n // This is not single-copy atomic reads, but this is ok because subsequent\n // CAS will check for consistency.\n //\n // This is based on the code generated for the first load in DW RMWs by LLVM.\n //\n // Note that the C++20 memory model does not allow mixed-sized atomic access,\n // so we must use inline assembly to implement this.\n // (i.e., byte-wise atomic based on the standard library's atomic types\n // cannot be used here).\n concat!(\"mov rax, qword ptr [\", $dst, \"]\"),\n concat!(\"mov rdx, qword ptr [\", $dst, \" + 8]\"),\n \"2:\",\n concat!(\"lock cmpxchg16b xmmword ptr [\", $dst, \"]\"),\n \"jne 2b\",\n concat!(\"mov rbx, \", $save), // restore rbx\n inout($save) val.pair.lo => _,\n in(\"rcx\") val.pair.hi,\n out(\"rax\") prev_lo,\n out(\"rdx\") prev_hi,\n in($dst) dst,\n // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.\n options(nostack),\n )\n };\n }\n // rdi and rsi are call-preserved on Windows.\n #[cfg(not(windows))]\n #[cfg(target_pointer_width = \"32\")]\n cmpxchg16b!(\"edi\", \"rsi\");\n #[cfg(not(windows))]\n #[cfg(target_pointer_width = \"64\")]\n cmpxchg16b!(\"rdi\", \"rsi\");\n #[cfg(windows)]\n #[cfg(target_pointer_width = \"32\")]\n cmpxchg16b!(\"r9d\", \"r8\");\n #[cfg(windows)]\n #[cfg(target_pointer_width = \"64\")]\n cmpxchg16b!(\"r9\", \"r8\");\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n}\n\n/// Atomic RMW by CAS loop (3 arguments)\n/// `unsafe fn(dst: *mut u128, val: u128, order: Ordering) -> u128;`\n///\n/// `$op` can use the following registers:\n/// - r8/r9 pair: val argument (read-only for `$op`)\n/// - rax/rdx pair: previous value loaded (read-only for `$op`)\n/// - rbx/rcx pair: new value that will be stored\n// We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows\n// omitting the storing/comparing of condition flags and reducing uses of xchg/mov to handle rbx.\nmacro_rules! atomic_rmw_cas_3 {\n ($name:ident, $($op:tt)*) => {\n // See cmpxchg16b() for target_feature(enable).\n #[cfg_attr(\n not(portable_atomic_no_cmpxchg16b_target_feature),\n target_feature(enable = \"cmpxchg16b\")\n )]\n #[inline]\n unsafe fn $name(dst: *mut u128, val: u128, _order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_cmpxchg16b!();\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.\n // cfg guarantees that the CPU supports CMPXCHG16B.\n //\n // See cmpxchg16b function for more.\n unsafe {\n // cmpxchg16b is always SeqCst.\n let val = U128 { whole: val };\n let (mut prev_lo, mut prev_hi);\n macro_rules! cmpxchg16b {\n ($dst:tt, $save:tt) => {\n asm!(\n concat!(\"mov \", $save, \", rbx\"), // save rbx which is reserved by LLVM\n // This is not single-copy atomic reads, but this is ok because subsequent\n // CAS will check for consistency.\n //\n // This is based on the code generated for the first load in DW RMWs by LLVM.\n //\n // Note that the C++20 memory model does not allow mixed-sized atomic access,\n // so we must use inline assembly to implement this.\n // (i.e., byte-wise atomic based on the standard library's atomic types\n // cannot be used here).\n concat!(\"mov rax, qword ptr [\", $dst, \"]\"),\n concat!(\"mov rdx, qword ptr [\", $dst, \" + 8]\"),\n \"2:\",\n $($op)*\n concat!(\"lock cmpxchg16b xmmword ptr [\", $dst, \"]\"),\n \"jne 2b\",\n concat!(\"mov rbx, \", $save), // restore rbx\n out($save) _,\n out(\"rcx\") _,\n out(\"rax\") prev_lo,\n out(\"rdx\") prev_hi,\n in($dst) dst,\n in(\"r8\") val.pair.lo,\n in(\"r9\") val.pair.hi,\n // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.\n options(nostack),\n )\n };\n }\n // rdi and rsi are call-preserved on Windows.\n #[cfg(not(windows))]\n #[cfg(target_pointer_width = \"32\")]\n cmpxchg16b!(\"edi\", \"r10\");\n #[cfg(not(windows))]\n #[cfg(target_pointer_width = \"64\")]\n cmpxchg16b!(\"rdi\", \"r10\");\n #[cfg(windows)]\n #[cfg(target_pointer_width = \"32\")]\n cmpxchg16b!(\"r10d\", \"r11\");\n #[cfg(windows)]\n #[cfg(target_pointer_width = \"64\")]\n cmpxchg16b!(\"r10\", \"r11\");\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n }\n };\n}\n/// Atomic RMW by CAS loop (2 arguments)\n/// `unsafe fn(dst: *mut u128, order: Ordering) -> u128;`\n///\n/// `$op` can use the following registers:\n/// - rax/rdx pair: previous value loaded (read-only for `$op`)\n/// - rbx/rcx pair: new value that will be stored\n// We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows\n// omitting the storing of condition flags and avoid use of xchg to handle rbx.\nmacro_rules! atomic_rmw_cas_2 {\n ($name:ident, $($op:tt)*) => {\n // See cmpxchg16b() for target_feature(enable).\n #[cfg_attr(\n not(portable_atomic_no_cmpxchg16b_target_feature),\n target_feature(enable = \"cmpxchg16b\")\n )]\n #[inline]\n unsafe fn $name(dst: *mut u128, _order: Ordering) -> u128 {\n debug_assert!(dst.addr() % 16 == 0);\n debug_assert_cmpxchg16b!();\n // SAFETY: the caller must guarantee that `dst` is valid for both writes and\n // reads, 16-byte aligned, and that there are no concurrent non-atomic operations.\n // cfg guarantees that the CPU supports CMPXCHG16B.\n //\n // See cmpxchg16b function for more.\n unsafe {\n // cmpxchg16b is always SeqCst.\n let (mut prev_lo, mut prev_hi);\n macro_rules! cmpxchg16b {\n ($dst:tt, $save:tt) => {\n asm!(\n concat!(\"mov \", $save, \", rbx\"), // save rbx which is reserved by LLVM\n // This is not single-copy atomic reads, but this is ok because subsequent\n // CAS will check for consistency.\n //\n // This is based on the code generated for the first load in DW RMWs by LLVM.\n //\n // Note that the C++20 memory model does not allow mixed-sized atomic access,\n // so we must use inline assembly to implement this.\n // (i.e., byte-wise atomic based on the standard library's atomic types\n // cannot be used here).\n concat!(\"mov rax, qword ptr [\", $dst, \"]\"),\n concat!(\"mov rdx, qword ptr [\", $dst, \" + 8]\"),\n \"2:\",\n $($op)*\n concat!(\"lock cmpxchg16b xmmword ptr [\", $dst, \"]\"),\n \"jne 2b\",\n concat!(\"mov rbx, \", $save), // restore rbx\n out($save) _,\n out(\"rcx\") _,\n out(\"rax\") prev_lo,\n out(\"rdx\") prev_hi,\n in($dst) dst,\n // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag.\n options(nostack),\n )\n };\n }\n // rdi and rsi are call-preserved on Windows.\n #[cfg(not(windows))]\n #[cfg(target_pointer_width = \"32\")]\n cmpxchg16b!(\"edi\", \"rsi\");\n #[cfg(not(windows))]\n #[cfg(target_pointer_width = \"64\")]\n cmpxchg16b!(\"rdi\", \"rsi\");\n #[cfg(windows)]\n #[cfg(target_pointer_width = \"32\")]\n cmpxchg16b!(\"r9d\", \"r8\");\n #[cfg(windows)]\n #[cfg(target_pointer_width = \"64\")]\n cmpxchg16b!(\"r9\", \"r8\");\n U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole\n }\n }\n };\n}\n\natomic_rmw_cas_3! {\n atomic_add_cmpxchg16b,\n \"mov rbx, rax\",\n \"add rbx, r8\",\n \"mov rcx, rdx\",\n \"adc rcx, r9\",\n}\natomic_rmw_cas_3! {\n atomic_sub_cmpxchg16b,\n \"mov rbx, rax\",\n \"sub rbx, r8\",\n \"mov rcx, rdx\",\n \"sbb rcx, r9\",\n}\natomic_rmw_cas_3! {\n atomic_and_cmpxchg16b,\n \"mov rbx, rax\",\n \"and rbx, r8\",\n \"mov rcx, rdx\",\n \"and rcx, r9\",\n}\natomic_rmw_cas_3! {\n atomic_nand_cmpxchg16b,\n \"mov rbx, rax\",\n \"and rbx, r8\",\n \"not rbx\",\n \"mov rcx, rdx\",\n \"and rcx, r9\",\n \"not rcx\",\n}\natomic_rmw_cas_3! {\n atomic_or_cmpxchg16b,\n \"mov rbx, rax\",\n \"or rbx, r8\",\n \"mov rcx, rdx\",\n \"or rcx, r9\",\n}\natomic_rmw_cas_3! {\n atomic_xor_cmpxchg16b,\n \"mov rbx, rax\",\n \"xor rbx, r8\",\n \"mov rcx, rdx\",\n \"xor rcx, r9\",\n}\n\natomic_rmw_cas_2! {\n atomic_not_cmpxchg16b,\n \"mov rbx, rax\",\n \"not rbx\",\n \"mov rcx, rdx\",\n \"not rcx\",\n}\natomic_rmw_cas_2! {\n atomic_neg_cmpxchg16b,\n \"mov rbx, rax\",\n \"neg rbx\",\n \"mov rcx, 0\",\n \"sbb rcx, rdx\",\n}\n\natomic_rmw_cas_3! {\n atomic_max_cmpxchg16b,\n \"cmp r8, rax\",\n \"mov rcx, r9\",\n \"sbb rcx, rdx\",\n \"mov rcx, r9\",\n \"cmovl rcx, rdx\",\n \"mov rbx, r8\",\n \"cmovl rbx, rax\",\n}\natomic_rmw_cas_3! {\n atomic_umax_cmpxchg16b,\n \"cmp r8, rax\",\n \"mov rcx, r9\",\n \"sbb rcx, rdx\",\n \"mov rcx, r9\",\n \"cmovb rcx, rdx\",\n \"mov rbx, r8\",\n \"cmovb rbx, rax\",\n}\natomic_rmw_cas_3! {\n atomic_min_cmpxchg16b,\n \"cmp r8, rax\",\n \"mov rcx, r9\",\n \"sbb rcx, rdx\",\n \"mov rcx, r9\",\n \"cmovge rcx, rdx\",\n \"mov rbx, r8\",\n \"cmovge rbx, rax\",\n}\natomic_rmw_cas_3! {\n atomic_umin_cmpxchg16b,\n \"cmp r8, rax\",\n \"mov rcx, r9\",\n \"sbb rcx, rdx\",\n \"mov rcx, r9\",\n \"cmovae rcx, rdx\",\n \"mov rbx, r8\",\n \"cmovae rbx, rax\",\n}\n\nmacro_rules! select_atomic_rmw {\n (\n unsafe fn $name:ident($($arg:tt)*) $(-> $ret_ty:ty)?;\n cmpxchg16b = $cmpxchg16b_fn:ident;\n fallback = $seqcst_fallback_fn:ident;\n ) => {\n // If cmpxchg16b is available at compile-time, we can always use cmpxchg16b_fn.\n #[cfg(any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"))]\n use self::$cmpxchg16b_fn as $name;\n // Otherwise, we need to do run-time detection and can use cmpxchg16b_fn only if cmpxchg16b is available.\n #[cfg(not(any(\n target_feature = \"cmpxchg16b\",\n portable_atomic_target_feature = \"cmpxchg16b\",\n )))]\n #[inline]\n unsafe fn $name($($arg)*, _order: Ordering) $(-> $ret_ty)? {\n fn_alias! {\n // See cmpxchg16b() for target_feature(enable).\n #[cfg_attr(\n not(portable_atomic_no_cmpxchg16b_target_feature),\n target_feature(enable = \"cmpxchg16b\")\n )]\n unsafe fn($($arg)*) $(-> $ret_ty)?;\n // cmpxchg16b is always SeqCst.\n cmpxchg16b_seqcst_fn = $cmpxchg16b_fn(Ordering::SeqCst);\n }\n // SAFETY: the caller must uphold the safety contract.\n // we only calls cmpxchg16b_fn if cmpxchg16b is available.\n unsafe {\n ifunc!(unsafe fn($($arg)*) $(-> $ret_ty)? {\n if detect::detect().cmpxchg16b() {\n cmpxchg16b_seqcst_fn\n } else {\n // Use SeqCst because cmpxchg16b is always SeqCst.\n fallback::$seqcst_fallback_fn\n }\n })\n }\n }\n };\n}\n\nselect_atomic_rmw! {\n unsafe fn atomic_swap(dst: *mut u128, val: u128) -> u128;\n cmpxchg16b = atomic_swap_cmpxchg16b;\n fallback = atomic_swap_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_add(dst: *mut u128, val: u128) -> u128;\n cmpxchg16b = atomic_add_cmpxchg16b;\n fallback = atomic_add_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_sub(dst: *mut u128, val: u128) -> u128;\n cmpxchg16b = atomic_sub_cmpxchg16b;\n fallback = atomic_sub_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_and(dst: *mut u128, val: u128) -> u128;\n cmpxchg16b = atomic_and_cmpxchg16b;\n fallback = atomic_and_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_nand(dst: *mut u128, val: u128) -> u128;\n cmpxchg16b = atomic_nand_cmpxchg16b;\n fallback = atomic_nand_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_or(dst: *mut u128, val: u128) -> u128;\n cmpxchg16b = atomic_or_cmpxchg16b;\n fallback = atomic_or_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_xor(dst: *mut u128, val: u128) -> u128;\n cmpxchg16b = atomic_xor_cmpxchg16b;\n fallback = atomic_xor_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_max(dst: *mut u128, val: u128) -> u128;\n cmpxchg16b = atomic_max_cmpxchg16b;\n fallback = atomic_max_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_umax(dst: *mut u128, val: u128) -> u128;\n cmpxchg16b = atomic_umax_cmpxchg16b;\n fallback = atomic_umax_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_min(dst: *mut u128, val: u128) -> u128;\n cmpxchg16b = atomic_min_cmpxchg16b;\n fallback = atomic_min_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_umin(dst: *mut u128, val: u128) -> u128;\n cmpxchg16b = atomic_umin_cmpxchg16b;\n fallback = atomic_umin_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_not(dst: *mut u128) -> u128;\n cmpxchg16b = atomic_not_cmpxchg16b;\n fallback = atomic_not_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_neg(dst: *mut u128) -> u128;\n cmpxchg16b = atomic_neg_cmpxchg16b;\n fallback = atomic_neg_seqcst;\n}\n\n#[inline]\nfn is_lock_free() -> bool {\n #[cfg(any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"))]\n {\n // CMPXCHG16B is available at compile-time.\n true\n }\n #[cfg(not(any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\")))]\n {\n detect::detect().cmpxchg16b()\n }\n}\nconst IS_ALWAYS_LOCK_FREE: bool =\n cfg!(any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"));\n\natomic128!(AtomicI128, i128, atomic_max, atomic_min);\natomic128!(AtomicU128, u128, atomic_umax, atomic_umin);\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n test_atomic_int!(i128);\n test_atomic_int!(u128);\n\n // load/store/swap implementation is not affected by signedness, so it is\n // enough to test only unsigned types.\n stress_test!(u128);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic64/arm_linux.rs", "sha256": "e4857c2c7240b73a3e3c9bb092b1251d9b71a44617cc43345ffd14bf11ee5f65", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\n64-bit atomic implementation using kuser_cmpxchg64 on pre-v6 Arm Linux/Android.\n\nSee \"Atomic operation overview by architecture\" in atomic-maybe-uninit for a more comprehensive and\ndetailed description of the atomic and synchronize instructions in this architecture:\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#arm\n\nRefs:\n- https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst\n- https://github.com/rust-lang/compiler-builtins/blob/compiler_builtins-v0.1.124/src/arm_linux.rs\n- https://github.com/taiki-e/atomic-maybe-uninit\n\nNote: __kuser_cmpxchg64 is always SeqCst.\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.21/src/arch/arm.rs#L230\n\nNote: On Miri and ThreadSanitizer which do not support inline assembly, we don't use\nthis module and use fallback implementation instead.\n*/\n\n// TODO: Since Rust 1.64, the Linux kernel requirement for Rust when using std is 3.2+, so it should\n// be possible to omit the dynamic kernel version check if the std feature is enabled on Rust 1.64+.\n// https://blog.rust-lang.org/2022/08/01/Increasing-glibc-kernel-requirements\n\ninclude!(\"macros.rs\");\n\n#[path = \"../fallback/outline_atomics.rs\"]\nmod fallback;\n\n#[cfg(test)] // test-only (unused)\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(\n all(\n target_os = \"linux\",\n any(\n target_env = \"gnu\",\n target_env = \"musl\",\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n ),\n ),\n target_os = \"android\",\n target_os = \"freebsd\",\n target_os = \"openbsd\",\n))]\n#[path = \"../detect/auxv.rs\"]\nmod test_detect_auxv;\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\nuse core::{mem, sync::atomic::Ordering};\n\n#[cfg(portable_atomic_no_strict_provenance)]\nuse crate::utils::ptr::PtrExt as _;\nuse crate::utils::{Pair, U64};\n\n// https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm/kernel_user_helpers.rst\nconst KUSER_HELPER_VERSION: usize = 0xFFFF0FFC;\n// __kuser_helper_version >= 5 (kernel version 3.1+)\nconst KUSER_CMPXCHG64: usize = 0xFFFF0F60;\n#[inline]\nfn __kuser_helper_version() -> i32 {\n // SAFETY: core assumes that at least __kuser_memory_barrier (__kuser_helper_version >= 3) is\n // available on this platform. __kuser_helper_version is always available on such a platform.\n unsafe { crate::utils::ptr::with_exposed_provenance::(KUSER_HELPER_VERSION).read() }\n}\n#[inline]\nfn has_kuser_cmpxchg64() -> bool {\n // Note: detect_false cfg is intended to make it easy for developers to test\n // cases where features usually available is not available, and is not a public API.\n if cfg!(portable_atomic_test_detect_false) {\n return false;\n }\n __kuser_helper_version() >= 5\n}\n#[inline]\nunsafe fn __kuser_cmpxchg64(old_val: *const u64, new_val: *const u64, ptr: *mut u64) -> bool {\n // SAFETY: kernel docs specify a known address with the given signature.\n // And the caller must guarantee that the pointer is valid for read and write,\n // aligned to the element size, and __kuser_helper_version >= 5.\n unsafe {\n let f: extern \"C\" fn(*const u64, *const u64, *mut u64) -> u32 =\n mem::transmute(crate::utils::ptr::with_exposed_provenance::<()>(KUSER_CMPXCHG64));\n f(old_val, new_val, ptr) == 0\n }\n}\n\n// 64-bit atomic load by two 32-bit atomic loads.\n#[inline]\nunsafe fn byte_wise_atomic_load(src: *const u64) -> u64 {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n let (out_lo, out_hi);\n asm!(\n \"ldr {out_lo}, [{src}]\", // atomic { out_lo = *src }\n \"ldr {out_hi}, [{src}, #4]\", // atomic { out_hi = *src.byte_add(4) }\n src = in(reg) src,\n out_lo = out(reg) out_lo,\n out_hi = out(reg) out_hi,\n options(pure, nostack, preserves_flags, readonly),\n );\n U64 { pair: Pair { lo: out_lo, hi: out_hi } }.whole\n }\n}\n\nmacro_rules! select_atomic {\n (\n unsafe fn $name:ident($dst:ident: *mut u64 $(, $($arg:tt)*)?) $(-> $ret_ty:ty)? {\n |$kuser_cmpxchg64_fn_binding:ident| $($kuser_cmpxchg64_fn_body:tt)*\n }\n fallback = $seqcst_fallback_fn:ident\n ) => {\n #[inline]\n unsafe fn $name($dst: *mut u64 $(, $($arg)*)?, _: Ordering) $(-> $ret_ty)? {\n unsafe fn kuser_cmpxchg64_fn($dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {\n debug_assert!($dst.addr() % 8 == 0);\n debug_assert!(has_kuser_cmpxchg64());\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n loop {\n // This is not single-copy atomic reads, but this is ok because subsequent\n // CAS will check for consistency.\n //\n // Arm's memory model allow mixed-sized atomic access.\n // https://github.com/rust-lang/unsafe-code-guidelines/issues/345#issuecomment-1172891466\n //\n // Note that the C++20 memory model does not allow mixed-sized atomic access,\n // so we must use inline assembly to implement byte_wise_atomic_load.\n // (i.e., byte-wise atomic based on the standard library's atomic types\n // cannot be used here).\n let prev = byte_wise_atomic_load($dst);\n let next = {\n let $kuser_cmpxchg64_fn_binding = prev;\n $($kuser_cmpxchg64_fn_body)*\n };\n if __kuser_cmpxchg64(&prev, &next, $dst) {\n return prev;\n }\n }\n }\n }\n // SAFETY: the caller must uphold the safety contract.\n // we only calls __kuser_cmpxchg64 if it is available.\n unsafe {\n ifunc!(unsafe fn($dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {\n if has_kuser_cmpxchg64() {\n kuser_cmpxchg64_fn\n } else {\n // Use SeqCst because __kuser_cmpxchg64 is always SeqCst.\n fallback::$seqcst_fallback_fn\n }\n })\n }\n }\n };\n}\n\nselect_atomic! {\n unsafe fn atomic_load(src: *mut u64) -> u64 {\n |old| old\n }\n fallback = atomic_load_seqcst\n}\n#[inline]\nunsafe fn atomic_store(dst: *mut u64, val: u64, order: Ordering) {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n atomic_swap(dst, val, order);\n }\n}\nselect_atomic! {\n unsafe fn atomic_swap(dst: *mut u64, val: u64) -> u64 {\n |_x| val\n }\n fallback = atomic_swap_seqcst\n}\n#[inline]\nunsafe fn atomic_compare_exchange(\n dst: *mut u64,\n old: u64,\n new: u64,\n _: Ordering,\n _: Ordering,\n) -> Result {\n unsafe fn kuser_cmpxchg64_fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {\n debug_assert!(dst.addr() % 8 == 0);\n debug_assert!(has_kuser_cmpxchg64());\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n loop {\n // See select_atomic! for more.\n let prev = byte_wise_atomic_load(dst);\n let next = if prev == old { new } else { prev };\n if __kuser_cmpxchg64(&prev, &next, dst) {\n return (prev, prev == old);\n }\n }\n }\n }\n // SAFETY: the caller must uphold the safety contract.\n // we only calls __kuser_cmpxchg64 if it is available.\n let (prev, ok) = unsafe {\n ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {\n if has_kuser_cmpxchg64() {\n kuser_cmpxchg64_fn\n } else {\n // Use SeqCst because __kuser_cmpxchg64 is always SeqCst.\n fallback::atomic_compare_exchange_seqcst\n }\n })\n };\n if ok { Ok(prev) } else { Err(prev) }\n}\nuse self::atomic_compare_exchange as atomic_compare_exchange_weak;\nselect_atomic! {\n unsafe fn atomic_add(dst: *mut u64, val: u64) -> u64 {\n |x| x.wrapping_add(val)\n }\n fallback = atomic_add_seqcst\n}\nselect_atomic! {\n unsafe fn atomic_sub(dst: *mut u64, val: u64) -> u64 {\n |x| x.wrapping_sub(val)\n }\n fallback = atomic_sub_seqcst\n}\nselect_atomic! {\n unsafe fn atomic_and(dst: *mut u64, val: u64) -> u64 {\n |x| x & val\n }\n fallback = atomic_and_seqcst\n}\nselect_atomic! {\n unsafe fn atomic_nand(dst: *mut u64, val: u64) -> u64 {\n |x| !(x & val)\n }\n fallback = atomic_nand_seqcst\n}\nselect_atomic! {\n unsafe fn atomic_or(dst: *mut u64, val: u64) -> u64 {\n |x| x | val\n }\n fallback = atomic_or_seqcst\n}\nselect_atomic! {\n unsafe fn atomic_xor(dst: *mut u64, val: u64) -> u64 {\n |x| x ^ val\n }\n fallback = atomic_xor_seqcst\n}\nselect_atomic! {\n unsafe fn atomic_max(dst: *mut u64, val: u64) -> u64 {\n |x| {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n { core::cmp::max(x as i64, val as i64) as u64 }\n }\n }\n fallback = atomic_max_seqcst\n}\nselect_atomic! {\n unsafe fn atomic_umax(dst: *mut u64, val: u64) -> u64 {\n |x| core::cmp::max(x, val)\n }\n fallback = atomic_umax_seqcst\n}\nselect_atomic! {\n unsafe fn atomic_min(dst: *mut u64, val: u64) -> u64 {\n |x| {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n { core::cmp::min(x as i64, val as i64) as u64 }\n }\n }\n fallback = atomic_min_seqcst\n}\nselect_atomic! {\n unsafe fn atomic_umin(dst: *mut u64, val: u64) -> u64 {\n |x| core::cmp::min(x, val)\n }\n fallback = atomic_umin_seqcst\n}\nselect_atomic! {\n unsafe fn atomic_not(dst: *mut u64) -> u64 {\n |x| !x\n }\n fallback = atomic_not_seqcst\n}\nselect_atomic! {\n unsafe fn atomic_neg(dst: *mut u64) -> u64 {\n |x| x.wrapping_neg()\n }\n fallback = atomic_neg_seqcst\n}\n\n#[inline]\nfn is_lock_free() -> bool {\n has_kuser_cmpxchg64()\n}\nconst IS_ALWAYS_LOCK_FREE: bool = false;\n\natomic64!(AtomicI64, i64, atomic_max, atomic_min);\natomic64!(AtomicU64, u64, atomic_umax, atomic_umin);\n\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks,\n clippy::wildcard_imports\n)]\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn kuser_helper_version() {\n let version = __kuser_helper_version();\n assert!(version >= 5, \"{:?}\", version);\n assert_eq!(version, unsafe {\n crate::utils::ptr::with_exposed_provenance::(KUSER_HELPER_VERSION).read()\n });\n }\n\n test_atomic_int!(i64);\n test_atomic_int!(u64);\n\n // load/store/swap implementation is not affected by signedness, so it is\n // enough to test only unsigned types.\n stress_test!(u64);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic64/macros.rs", "sha256": "5b6f0bfe40064885926cfd57a336f15da92f6964864c50d339d1bb9903802a9d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nmacro_rules! atomic64 {\n ($atomic_type:ident, $int_type:ident, $atomic_max:ident, $atomic_min:ident) => {\n #[repr(C, align(8))]\n pub(crate) struct $atomic_type {\n v: core::cell::UnsafeCell<$int_type>,\n }\n\n // Send is implicitly implemented.\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock.\n unsafe impl Sync for $atomic_type {}\n\n impl_default_no_fetch_ops!($atomic_type, $int_type);\n impl_default_bit_opts!($atomic_type, $int_type);\n impl $atomic_type {\n #[inline]\n pub(crate) const fn new(v: $int_type) -> Self {\n Self { v: core::cell::UnsafeCell::new(v) }\n }\n\n #[inline]\n pub(crate) fn is_lock_free() -> bool {\n is_lock_free()\n }\n pub(crate) const IS_ALWAYS_LOCK_FREE: bool = IS_ALWAYS_LOCK_FREE;\n\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n pub(crate) fn load(&self, order: Ordering) -> $int_type {\n crate::utils::assert_load_ordering(order);\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_load(self.v.get().cast::(), order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n pub(crate) fn store(&self, val: $int_type, order: Ordering) {\n crate::utils::assert_store_ordering(order);\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_store(self.v.get().cast::(), val as u64, order)\n }\n }\n\n #[inline]\n pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_swap(self.v.get().cast::(), val as u64, order) as $int_type\n }\n }\n\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n pub(crate) fn compare_exchange(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure);\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n match atomic_compare_exchange(\n self.v.get().cast::(),\n current as u64,\n new as u64,\n success,\n failure,\n ) {\n Ok(v) => Ok(v as $int_type),\n Err(v) => Err(v as $int_type),\n }\n }\n }\n\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n pub(crate) fn compare_exchange_weak(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure);\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n match atomic_compare_exchange_weak(\n self.v.get().cast::(),\n current as u64,\n new as u64,\n success,\n failure,\n ) {\n Ok(v) => Ok(v as $int_type),\n Err(v) => Err(v as $int_type),\n }\n }\n }\n\n #[inline]\n pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_add(self.v.get().cast::(), val as u64, order) as $int_type\n }\n }\n\n #[inline]\n pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_sub(self.v.get().cast::(), val as u64, order) as $int_type\n }\n }\n\n #[inline]\n pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_and(self.v.get().cast::(), val as u64, order) as $int_type\n }\n }\n\n #[inline]\n pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_nand(self.v.get().cast::(), val as u64, order) as $int_type\n }\n }\n\n #[inline]\n pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_or(self.v.get().cast::(), val as u64, order) as $int_type\n }\n }\n\n #[inline]\n pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_xor(self.v.get().cast::(), val as u64, order) as $int_type\n }\n }\n\n #[inline]\n pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n $atomic_max(self.v.get().cast::(), val as u64, order) as $int_type\n }\n }\n\n #[inline]\n pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n $atomic_min(self.v.get().cast::(), val as u64, order) as $int_type\n }\n }\n\n #[inline]\n pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_not(self.v.get().cast::(), order) as $int_type\n }\n }\n #[inline]\n pub(crate) fn not(&self, order: Ordering) {\n self.fetch_not(order);\n }\n\n #[inline]\n pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n // SAFETY: any data races are prevented by atomic intrinsics, the kernel user helper, or the lock\n // and the raw pointer passed in is valid because we got it from a reference.\n unsafe {\n atomic_neg(self.v.get().cast::(), order) as $int_type\n }\n }\n #[inline]\n pub(crate) fn neg(&self, order: Ordering) {\n self.fetch_neg(order);\n }\n\n #[inline]\n pub(crate) const fn as_ptr(&self) -> *mut $int_type {\n self.v.get()\n }\n }\n };\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic64/mod.rs", "sha256": "2977468d184134a068179eb8361d73500865b8d1476b87590099d6b503db30e5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\n64-bit atomic implementations on 32-bit architectures\n\nSee README.md for details.\n*/\n\n// pre-v6 Arm Linux\n// Miri and Sanitizer do not support inline assembly.\n#[cfg(all(\n feature = \"fallback\",\n target_arch = \"arm\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_os = \"linux\", target_os = \"android\"),\n any(test, not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\"))),\n not(portable_atomic_no_outline_atomics),\n))]\n#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_64)))]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(test, not(target_has_atomic = \"64\")))\n)]\npub(super) mod arm_linux;\n\n// riscv32\n// Miri and Sanitizer do not support inline assembly.\n#[cfg(all(\n target_arch = \"riscv32\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(\n target_feature = \"zacas\",\n portable_atomic_target_feature = \"zacas\",\n all(\n feature = \"fallback\",\n not(portable_atomic_no_outline_atomics),\n any(target_os = \"linux\", target_os = \"android\"),\n ),\n ),\n))]\npub(super) mod riscv32;\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/atomic64/riscv32.rs", "sha256": "c8e00a15c0524fc94b0df7946fcd84b6779eb01bfe48f06bf6f4f84039c00337", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\n64-bit atomic implementation on riscv32.\n\nThis architecture provides the following 64-bit atomic instructions:\n\n- amocas.d: CAS (Zacas extension)\n\nSee \"Atomic operation overview by architecture\" in atomic-maybe-uninit for a more comprehensive and\ndetailed description of the atomic and synchronize instructions in this architecture:\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#risc-v\n\nNote: On Miri and ThreadSanitizer which do not support inline assembly, we don't use\nthis module and use fallback implementation instead.\n\nRefs:\n- RISC-V Instruction Set Manual\n \"Zacas\" Extension for Atomic Compare-and-Swap (CAS) Instructions\n https://docs.riscv.org/reference/isa/v20260120/unpriv/zacas.html\n- RISC-V Atomics ABI Specification\n https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20250812-301374e92976e298e676e7129a6212926b2299ce/riscv-atomic.adoc\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\n// TODO: merge duplicated code with atomic128/riscv64.rs\n\ninclude!(\"macros.rs\");\n\n#[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n#[path = \"../fallback/outline_atomics.rs\"]\nmod fallback;\n\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(test, not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"))))]\n#[cfg(any(target_os = \"linux\", target_os = \"android\"))]\n#[path = \"../detect/riscv_linux.rs\"]\nmod detect;\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\nuse core::sync::atomic::Ordering;\n\n#[cfg(portable_atomic_no_strict_provenance)]\nuse crate::utils::ptr::PtrExt as _;\nuse crate::utils::{Pair, U64};\n\nmacro_rules! debug_assert_zacas {\n () => {\n #[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n {\n debug_assert!(detect::detect().zacas());\n }\n };\n}\n\n// `.option arch, +zacas` directive requires LLVM 20, so we use .4byte directive for old LLVM.\n// Note that `.insn ` directive requires LLVM 19.\n// https://github.com/llvm/llvm-project/commit/2a086dce691e3cc34a2fc27f4fb255bb2cbbfac9\n// https://github.com/riscv-non-isa/riscv-asm-manual/blob/v0.0.1/src/asm-manual.adoc#arch\n#[cfg(not(portable_atomic_pre_llvm_20))]\nmacro_rules! start_zacas {\n () => {\n \".option push\\n.option arch, +zacas\"\n };\n}\n#[cfg(not(portable_atomic_pre_llvm_20))]\nmacro_rules! end_zacas {\n () => {\n \".option pop\"\n };\n}\n\n#[cfg(not(portable_atomic_pre_llvm_20))]\nmacro_rules! atomic_rmw_amocas_order {\n ($op:ident, $order:ident) => {\n atomic_rmw_amocas_order!($op, $order, failure = $order)\n };\n ($op:ident, $order:ident, failure = $failure:ident) => {\n match $order {\n Ordering::Relaxed => $op!(\"\", \"\"),\n Ordering::Acquire => $op!(\"\", \".aq\"),\n Ordering::Release => $op!(\"\", \".rl\"),\n Ordering::AcqRel => $op!(\"\", \".aqrl\"),\n Ordering::SeqCst if $failure == Ordering::SeqCst => $op!(\"fence rw,rw\", \".aqrl\"),\n Ordering::SeqCst => $op!(\"\", \".aqrl\"),\n _ => unreachable!(),\n }\n };\n}\n#[cfg(portable_atomic_pre_llvm_20)]\nmacro_rules! atomic_rmw_amocas_order_insn {\n ($op:ident, $order:ident) => {\n atomic_rmw_amocas_order_insn!($op, $order, failure = $order)\n };\n ($op:ident, $order:ident, failure = $failure:ident) => {\n match $order {\n Ordering::Relaxed => $op!(\"\", \"8\"),\n Ordering::Acquire => $op!(\"\", \"c\"),\n Ordering::Release => $op!(\"\", \"a\"),\n Ordering::AcqRel => $op!(\"\", \"e\"),\n Ordering::SeqCst if $failure == Ordering::SeqCst => $op!(\"fence rw,rw\", \"e\"),\n Ordering::SeqCst => $op!(\"\", \"e\"),\n _ => unreachable!(),\n }\n };\n}\n\n// If zacas is available at compile-time, we can always use zacas_fn.\n#[cfg(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"))]\nuse self::atomic_load_zacas as atomic_load;\n// Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.\n#[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n#[inline]\nunsafe fn atomic_load(src: *mut u64, order: Ordering) -> u64 {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(src: *mut u64) -> u64;\n atomic_load_zacas_relaxed = atomic_load_zacas(Ordering::Relaxed);\n atomic_load_zacas_acquire = atomic_load_zacas(Ordering::Acquire);\n atomic_load_zacas_seqcst = atomic_load_zacas(Ordering::SeqCst);\n }\n // SAFETY: the caller must uphold the safety contract.\n // we only calls atomic_load_zacas if zacas is available.\n unsafe {\n match order {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(src: *mut u64) -> u64 {\n if detect::detect().zacas() {\n atomic_load_zacas_relaxed\n } else {\n fallback::atomic_load_non_seqcst\n }\n })\n }\n Ordering::Acquire => {\n ifunc!(unsafe fn(src: *mut u64) -> u64 {\n if detect::detect().zacas() {\n atomic_load_zacas_acquire\n } else {\n fallback::atomic_load_non_seqcst\n }\n })\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(src: *mut u64) -> u64 {\n if detect::detect().zacas() {\n atomic_load_zacas_seqcst\n } else {\n fallback::atomic_load_seqcst\n }\n })\n }\n _ => unreachable!(),\n }\n }\n}\n#[inline]\nunsafe fn atomic_load_zacas(src: *mut u64, order: Ordering) -> u64 {\n debug_assert!(src.addr() % 8 == 0);\n debug_assert_zacas!();\n let (out_lo, out_hi);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n #[cfg(not(portable_atomic_pre_llvm_20))]\n macro_rules! load {\n ($fence:tt, $asm_order:tt) => {\n asm!(\n start_zacas!(),\n $fence, // fence\n concat!(\"amocas.d\", $asm_order, \" a2, a2, 0({src})\"), // atomic { if *dst == a2:a3 { *dst = a2:a3 } else { a2:a3 = *dst } }\n end_zacas!(),\n src = in(reg) ptr_reg!(src),\n inout(\"a2\") 0_u32 => out_lo,\n inout(\"a3\") 0_u32 => out_hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(not(portable_atomic_pre_llvm_20))]\n atomic_rmw_amocas_order!(load, order);\n #[cfg(portable_atomic_pre_llvm_20)]\n macro_rules! load {\n ($fence:tt, $insn_order:tt) => {\n asm!(\n $fence, // fence\n // amocas.d{,.aq,.rl,.aqrl} a2, a2, (a0) // atomic { if *a0 == a2:a3 { *a0 = a2:a3 } else { a2:a3 = *a0 } }\n concat!(\".4byte 0x2\", $insn_order, \"c5362f\"),\n in(\"a0\") ptr_reg!(src),\n inout(\"a2\") 0_u32 => out_lo,\n inout(\"a3\") 0_u32 => out_hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(portable_atomic_pre_llvm_20)]\n atomic_rmw_amocas_order_insn!(load, order);\n U64 { pair: Pair { lo: out_lo, hi: out_hi } }.whole\n }\n}\n\n#[inline]\nunsafe fn atomic_store(dst: *mut u64, val: u64, order: Ordering) {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n atomic_swap(dst, val, order);\n }\n}\n\n#[inline]\nunsafe fn atomic_compare_exchange(\n dst: *mut u64,\n old: u64,\n new: u64,\n success: Ordering,\n failure: Ordering,\n) -> Result {\n #[cfg(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"))]\n // SAFETY: the caller must uphold the safety contract.\n // cfg guarantees that zacas instructions are available at compile-time.\n let (prev, ok) = unsafe { atomic_compare_exchange_zacas(dst, old, new, success, failure) };\n #[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n let (prev, ok) = {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool);\n zacas_relaxed_fn = atomic_compare_exchange_zacas(Ordering::Relaxed, Ordering::Relaxed);\n zacas_acquire_fn = atomic_compare_exchange_zacas(Ordering::Acquire, Ordering::Acquire);\n zacas_release_fn = atomic_compare_exchange_zacas(Ordering::Release, Ordering::Relaxed);\n zacas_acqrel_fn = atomic_compare_exchange_zacas(Ordering::AcqRel, Ordering::Acquire);\n zacas_seqcst_fn = atomic_compare_exchange_zacas(Ordering::SeqCst, Ordering::SeqCst);\n }\n let order = crate::utils::upgrade_success_ordering(success, failure);\n // SAFETY: the caller must uphold the safety contract.\n // we only calls atomic_compare_exchange_zacas if zacas is available.\n unsafe {\n match order {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {\n if detect::detect().zacas() {\n zacas_relaxed_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::Acquire => {\n ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {\n if detect::detect().zacas() {\n zacas_acquire_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::Release => {\n ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {\n if detect::detect().zacas() {\n zacas_release_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::AcqRel => {\n ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {\n if detect::detect().zacas() {\n zacas_acqrel_fn\n } else {\n fallback::atomic_compare_exchange_non_seqcst\n }\n })\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(dst: *mut u64, old: u64, new: u64) -> (u64, bool) {\n if detect::detect().zacas() {\n zacas_seqcst_fn\n } else {\n fallback::atomic_compare_exchange_seqcst\n }\n })\n }\n _ => unreachable!(),\n }\n }\n };\n if ok { Ok(prev) } else { Err(prev) }\n}\n#[inline]\nunsafe fn atomic_compare_exchange_zacas(\n dst: *mut u64,\n old: u64,\n new: u64,\n success: Ordering,\n failure: Ordering,\n) -> (u64, bool) {\n debug_assert!(dst.addr() % 8 == 0);\n debug_assert_zacas!();\n let order = crate::utils::upgrade_success_ordering(success, failure);\n let old = U64 { whole: old };\n let new = U64 { whole: new };\n let (prev_lo, prev_hi);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n #[cfg(not(portable_atomic_pre_llvm_20))]\n macro_rules! cmpxchg {\n ($fence:tt, $asm_order:tt) => {\n asm!(\n start_zacas!(),\n $fence, // fence\n concat!(\"amocas.d\", $asm_order, \" a4, a2, 0({dst})\"), // atomic { if *dst == a4:a5 { *dst = a2:a3 } else { a4:a5 = *dst } }\n end_zacas!(),\n dst = in(reg) ptr_reg!(dst),\n // must be allocated to even/odd register pair\n inout(\"a4\") old.pair.lo => prev_lo,\n inout(\"a5\") old.pair.hi => prev_hi,\n // must be allocated to even/odd register pair\n in(\"a2\") new.pair.lo,\n in(\"a3\") new.pair.hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(not(portable_atomic_pre_llvm_20))]\n atomic_rmw_amocas_order!(cmpxchg, order, failure = failure);\n #[cfg(portable_atomic_pre_llvm_20)]\n macro_rules! cmpxchg {\n ($fence:tt, $insn_order:tt) => {\n asm!(\n $fence, // fence\n // amocas.d{,.aq,.rl,.aqrl} a4, a2, (a0) // atomic { if *a0 == a4:a5 { *a0 = a2:a3 } else { a4:a5 = *a0 } }\n concat!(\".4byte 0x2\", $insn_order, \"c5372f\"),\n in(\"a0\") ptr_reg!(dst),\n // must be allocated to even/odd register pair\n inout(\"a4\") old.pair.lo => prev_lo,\n inout(\"a5\") old.pair.hi => prev_hi,\n // must be allocated to even/odd register pair\n in(\"a2\") new.pair.lo,\n in(\"a3\") new.pair.hi,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(portable_atomic_pre_llvm_20)]\n atomic_rmw_amocas_order_insn!(cmpxchg, order, failure = failure);\n let prev = U64 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole;\n (prev, prev == old.whole)\n }\n}\n\n// amocas is always strong.\nuse self::atomic_compare_exchange as atomic_compare_exchange_weak;\n\n// 64-bit atomic load by two 32-bit atomic loads.\n#[inline]\nunsafe fn byte_wise_atomic_load(src: *const u64) -> u64 {\n let (out_lo, out_hi);\n\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n asm!(\n \"lw {out_lo}, ({src})\", // atomic { out_lo = *src }\n \"lw {out_hi}, 4({src})\", // atomic { out_hi = *src.byte_add(4) }\n src = in(reg) ptr_reg!(src),\n out_lo = out(reg) out_lo,\n out_hi = out(reg) out_hi,\n options(pure, nostack, preserves_flags, readonly),\n );\n U64 { pair: Pair { lo: out_lo, hi: out_hi } }.whole\n }\n}\n\nmacro_rules! select_atomic_rmw {\n (\n unsafe fn $name:ident(dst: *mut u64 $(, $($arg:tt)*)?) $(-> $ret_ty:ty)? {\n |$zacas_fn_binding:ident| $($zacas_fn_body:tt)*\n }\n zacas = $zacas_fn:ident;\n non_seqcst_fallback = $non_seqcst_fallback_fn:ident;\n seqcst_fallback = $seqcst_fallback_fn:ident;\n ) => {\n #[inline]\n unsafe fn $zacas_fn(dst: *mut u64 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n // This is not single-copy atomic reads, but this is ok because subsequent\n // CAS will check for consistency.\n //\n // Note that the C++20 memory model does not allow mixed-sized atomic access,\n // so we must use inline assembly to implement byte_wise_atomic_load.\n // (i.e., byte-wise atomic based on the standard library's atomic types\n // cannot be used here).\n let mut prev = byte_wise_atomic_load(dst);\n loop {\n let next = {\n let $zacas_fn_binding = prev;\n $($zacas_fn_body)*\n };\n match atomic_compare_exchange_weak(dst, prev, next, order, Ordering::Relaxed) {\n Ok(x) => return x,\n Err(x) => prev = x,\n }\n }\n }\n }\n // If zacas is available at compile-time, we can always use zacas_fn.\n #[cfg(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"))]\n use self::$zacas_fn as $name;\n // Otherwise, we need to do run-time detection and can use zacas_fn only if zacas is available.\n #[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n #[inline]\n unsafe fn $name(dst: *mut u64 $(, $($arg)*)?, order: Ordering) $(-> $ret_ty)? {\n fn_alias! {\n // inline(never) is just a hint and also not strictly necessary\n // because we use ifunc helper macro, but used for clarity.\n #[inline(never)]\n unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)?;\n zacas_relaxed_fn = $zacas_fn(Ordering::Relaxed);\n zacas_acquire_fn = $zacas_fn(Ordering::Acquire);\n zacas_release_fn = $zacas_fn(Ordering::Release);\n zacas_acqrel_fn = $zacas_fn(Ordering::AcqRel);\n zacas_seqcst_fn = $zacas_fn(Ordering::SeqCst);\n }\n // SAFETY: the caller must uphold the safety contract.\n // we only calls zacas_fn if zacas is available.\n unsafe {\n match order {\n Ordering::Relaxed => {\n ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {\n if detect::detect().zacas() {\n zacas_relaxed_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::Acquire => {\n ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {\n if detect::detect().zacas() {\n zacas_acquire_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::Release => {\n ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {\n if detect::detect().zacas() {\n zacas_release_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::AcqRel => {\n ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {\n if detect::detect().zacas() {\n zacas_acqrel_fn\n } else {\n fallback::$non_seqcst_fallback_fn\n }\n })\n }\n Ordering::SeqCst => {\n ifunc!(unsafe fn(dst: *mut u64 $(, $($arg)*)?) $(-> $ret_ty)? {\n if detect::detect().zacas() {\n zacas_seqcst_fn\n } else {\n fallback::$seqcst_fallback_fn\n }\n })\n }\n _ => unreachable!(),\n }\n }\n }\n };\n}\n\nselect_atomic_rmw! {\n unsafe fn atomic_swap(dst: *mut u64, val: u64) -> u64 {\n |_x| val\n }\n zacas = atomic_swap_zacas;\n non_seqcst_fallback = atomic_swap_non_seqcst;\n seqcst_fallback = atomic_swap_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_add(dst: *mut u64, val: u64) -> u64 {\n |x| x.wrapping_add(val)\n }\n zacas = atomic_add_zacas;\n non_seqcst_fallback = atomic_add_non_seqcst;\n seqcst_fallback = atomic_add_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_sub(dst: *mut u64, val: u64) -> u64 {\n |x| x.wrapping_sub(val)\n }\n zacas = atomic_sub_zacas;\n non_seqcst_fallback = atomic_sub_non_seqcst;\n seqcst_fallback = atomic_sub_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_and(dst: *mut u64, val: u64) -> u64 {\n |x| x & val\n }\n zacas = atomic_and_zacas;\n non_seqcst_fallback = atomic_and_non_seqcst;\n seqcst_fallback = atomic_and_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_nand(dst: *mut u64, val: u64) -> u64 {\n |x| !(x & val)\n }\n zacas = atomic_nand_zacas;\n non_seqcst_fallback = atomic_nand_non_seqcst;\n seqcst_fallback = atomic_nand_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_or(dst: *mut u64, val: u64) -> u64 {\n |x| x | val\n }\n zacas = atomic_or_zacas;\n non_seqcst_fallback = atomic_or_non_seqcst;\n seqcst_fallback = atomic_or_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_xor(dst: *mut u64, val: u64) -> u64 {\n |x| x ^ val\n }\n zacas = atomic_xor_zacas;\n non_seqcst_fallback = atomic_xor_non_seqcst;\n seqcst_fallback = atomic_xor_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_max(dst: *mut u64, val: u64) -> u64 {\n |x| {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n { core::cmp::max(x as i64, val as i64) as u64 }\n }\n }\n zacas = atomic_max_zacas;\n non_seqcst_fallback = atomic_max_non_seqcst;\n seqcst_fallback = atomic_max_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_umax(dst: *mut u64, val: u64) -> u64 {\n |x| core::cmp::max(x, val)\n }\n zacas = atomic_umax_zacas;\n non_seqcst_fallback = atomic_umax_non_seqcst;\n seqcst_fallback = atomic_umax_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_min(dst: *mut u64, val: u64) -> u64 {\n |x| {\n #[allow(clippy::cast_possible_wrap, clippy::cast_sign_loss)]\n { core::cmp::min(x as i64, val as i64) as u64 }\n }\n }\n zacas = atomic_min_zacas;\n non_seqcst_fallback = atomic_min_non_seqcst;\n seqcst_fallback = atomic_min_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_umin(dst: *mut u64, val: u64) -> u64 {\n |x| core::cmp::min(x, val)\n }\n zacas = atomic_umin_zacas;\n non_seqcst_fallback = atomic_umin_non_seqcst;\n seqcst_fallback = atomic_umin_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_not(dst: *mut u64) -> u64 {\n |x| !x\n }\n zacas = atomic_not_zacas;\n non_seqcst_fallback = atomic_not_non_seqcst;\n seqcst_fallback = atomic_not_seqcst;\n}\nselect_atomic_rmw! {\n unsafe fn atomic_neg(dst: *mut u64) -> u64 {\n |x| x.wrapping_neg()\n }\n zacas = atomic_neg_zacas;\n non_seqcst_fallback = atomic_neg_non_seqcst;\n seqcst_fallback = atomic_neg_seqcst;\n}\n\n#[inline]\nfn is_lock_free() -> bool {\n #[cfg(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"))]\n {\n // zacas is available at compile-time.\n true\n }\n #[cfg(not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")))]\n {\n detect::detect().zacas()\n }\n}\nconst IS_ALWAYS_LOCK_FREE: bool =\n cfg!(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"));\n\natomic64!(AtomicI64, i64, atomic_max, atomic_min);\natomic64!(AtomicU64, u64, atomic_umax, atomic_umin);\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n test_atomic_int!(i64);\n test_atomic_int!(u64);\n\n // load/store/swap implementation is not affected by signedness, so it is\n // enough to test only unsigned types.\n stress_test!(u64);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/avr.rs", "sha256": "085cdcc03d7ffb85c491f3939e0bcc3ae1e78738acadf09472f547a0b9d7c8fc", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nAtomic load/store implementation on AVR.\n\nOperations not supported here are provided by disabling interrupts.\nSee also src/imp/interrupt/avr.rs.\n\nSee \"Atomic operation overview by architecture\" in atomic-maybe-uninit for a more comprehensive and\ndetailed description of the atomic and synchronize instructions in this architecture:\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#avr\n\nNote: Ordering is always SeqCst.\n\nRefs:\n- AVR® Instruction Set Manual, Rev. DS40002198B\n https://ww1.microchip.com/downloads/en/DeviceDoc/AVR-InstructionSet-Manual-DS40002198.pdf\n- atomic-maybe-uninit\n https://github.com/taiki-e/atomic-maybe-uninit\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\nuse core::{arch::asm, cell::UnsafeCell, sync::atomic::Ordering};\n\nmacro_rules! atomic8 {\n ($atomic_type:ident, $value_type:ty) => {\n #[repr(transparent)]\n pub(crate) struct $atomic_type {\n v: UnsafeCell<$value_type>,\n }\n\n // Send is implicitly implemented for atomic integers, but not for atomic pointers.\n // SAFETY: any data races are prevented by atomic operations.\n unsafe impl Send for $atomic_type {}\n // SAFETY: any data races are prevented by atomic operations.\n unsafe impl Sync for $atomic_type {}\n\n impl $atomic_type {\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n pub(crate) fn load(&self, order: Ordering) -> $value_type {\n crate::utils::assert_load_ordering(order);\n let src = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n let out;\n asm!(\n \"ld {out}, Z\", // atomic { out = *Z }\n out = out(reg) out,\n in(\"Z\") src,\n options(nostack, preserves_flags),\n );\n out\n }\n }\n\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n pub(crate) fn store(&self, val: $value_type, order: Ordering) {\n crate::utils::assert_store_ordering(order);\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n asm!(\n \"st Z, {val}\", // atomic { *Z = val }\n val = in(reg) val,\n in(\"Z\") dst,\n options(nostack, preserves_flags),\n );\n }\n }\n }\n\n #[cfg(any(target_feature = \"rmw\", portable_atomic_target_feature = \"rmw\"))]\n impl $atomic_type {\n #[inline]\n pub(crate) fn swap(&self, val: $value_type, _order: Ordering) -> $value_type {\n let dst = self.v.get();\n let out;\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n // cfg guarantee that the CPU supports RMW instructions.\n unsafe {\n asm!(\n \"xch Z, {val}\", // atomic { _x = *Z; *Z = val; val = _x }\n val = inout(reg) val => out,\n in(\"Z\") dst,\n options(nostack, preserves_flags),\n );\n }\n out\n }\n\n #[inline]\n pub(crate) fn fetch_and(&self, val: $value_type, _order: Ordering) -> $value_type {\n let dst = self.v.get();\n let out;\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n // cfg guarantee that the CPU supports RMW instructions.\n unsafe {\n asm!(\n \"lac Z, {val}\", // atomic { _x = *Z; *Z &= !val; val = _x }\n val = inout(reg) !val => out,\n in(\"Z\") dst,\n options(nostack, preserves_flags),\n );\n }\n out\n }\n\n #[inline]\n pub(crate) fn fetch_or(&self, val: $value_type, _order: Ordering) -> $value_type {\n let dst = self.v.get();\n let out;\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n // cfg guarantee that the CPU supports RMW instructions.\n unsafe {\n asm!(\n \"las Z, {val}\", // atomic { _x = *Z; *Z |= val; val = _x }\n val = inout(reg) val => out,\n in(\"Z\") dst,\n options(nostack, preserves_flags),\n );\n }\n out\n }\n\n #[inline]\n pub(crate) fn fetch_xor(&self, val: $value_type, _order: Ordering) -> $value_type {\n let dst = self.v.get();\n let out;\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n // cfg guarantee that the CPU supports RMW instructions.\n unsafe {\n asm!(\n \"lat Z, {val}\", // atomic { _x = *Z; *Z ^= val; val = _x }\n val = inout(reg) val => out,\n in(\"Z\") dst,\n options(nostack, preserves_flags),\n );\n }\n out\n }\n\n #[inline]\n pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type {\n self.fetch_xor(!0, order)\n }\n }\n };\n}\n\natomic8!(AtomicI8, i8);\natomic8!(AtomicU8, u8);\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/core_atomic.rs", "sha256": "0ce4a6ca4fb50fd8e3c6fb2adf96535ab9b8593b951d25f16b5762ddf867b7bd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nWrap the standard library's atomic types in newtype.\n\nThis is not a reexport, because we want to backport changes like\nhttps://github.com/rust-lang/rust/pull/98383 to old compilers.\n*/\n\nuse core::{cell::UnsafeCell, marker::PhantomData, sync::atomic::Ordering};\n\n// core::panic::RefUnwindSafe is only available on Rust 1.56+, so on pre-1.56\n// Rust, we implement RefUnwindSafe when \"std\" feature is enabled.\n// However, on pre-1.56 Rust, the standard library's atomic types implement\n// RefUnwindSafe when \"linked to std\", and that's behavior that our other atomic\n// implementations can't emulate, so use PhantomData to match\n// conditions where our other atomic implementations implement RefUnwindSafe.\n//\n// If we do not do this, for example, downstream that is only tested on x86_64\n// may incorrectly assume that AtomicU64 always implements RefUnwindSafe even on\n// older rustc, and may be broken on platforms where std AtomicU64 is not available.\nstruct NotRefUnwindSafe(UnsafeCell<()>);\n// SAFETY: this is a marker type and we'll never access the value.\nunsafe impl Sync for NotRefUnwindSafe {}\n\n#[repr(transparent)]\npub(crate) struct AtomicPtr {\n pub(super) inner: core::sync::atomic::AtomicPtr,\n // Prevent RefUnwindSafe from being propagated from the std atomic type. See NotRefUnwindSafe for more.\n _not_ref_unwind_safe: PhantomData,\n}\nimpl AtomicPtr {\n #[inline]\n pub(crate) const fn new(v: *mut T) -> Self {\n Self { inner: core::sync::atomic::AtomicPtr::new(v), _not_ref_unwind_safe: PhantomData }\n }\n #[inline]\n pub(crate) fn is_lock_free() -> bool {\n Self::IS_ALWAYS_LOCK_FREE\n }\n pub(crate) const IS_ALWAYS_LOCK_FREE: bool = true;\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];\n #[inline]\n pub(crate) const fn as_ptr(&self) -> *mut *mut T {\n // SAFETY: Self is #[repr(C)] and internally UnsafeCell<*mut T>.\n // See also https://github.com/rust-lang/rust/pull/66705 and\n // https://github.com/rust-lang/rust/issues/66136#issuecomment-557867116.\n unsafe { (*(self as *const Self as *const UnsafeCell<*mut T>)).get() }\n }\n }\n}\n#[cfg(not(all(\n target_arch = \"arm\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(target_os = \"linux\", target_os = \"android\"),\n not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\")),\n)))]\nimpl AtomicPtr {\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn load(&self, order: Ordering) -> *mut T {\n crate::utils::assert_load_ordering(order); // for track_caller (compiler can omit double check)\n self.inner.load(order)\n }\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn store(&self, ptr: *mut T, order: Ordering) {\n crate::utils::assert_store_ordering(order); // for track_caller (compiler can omit double check)\n self.inner.store(ptr, order);\n }\n}\n#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]\n#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = \"ptr\"))]\nitems!({\n #[cfg(not(all(\n target_arch = \"arm\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(target_os = \"linux\", target_os = \"android\"),\n not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\")),\n )))]\n impl AtomicPtr {\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange(\n &self,\n current: *mut T,\n new: *mut T,\n success: Ordering,\n failure: Ordering,\n ) -> Result<*mut T, *mut T> {\n crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)\n #[cfg(portable_atomic_no_stronger_failure_ordering)]\n let success = crate::utils::upgrade_success_ordering(success, failure);\n self.inner.compare_exchange(current, new, success, failure)\n }\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange_weak(\n &self,\n current: *mut T,\n new: *mut T,\n success: Ordering,\n failure: Ordering,\n ) -> Result<*mut T, *mut T> {\n crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)\n #[cfg(portable_atomic_no_stronger_failure_ordering)]\n let success = crate::utils::upgrade_success_ordering(success, failure);\n self.inner.compare_exchange_weak(current, new, success, failure)\n }\n }\n // Ideally, we would always use AtomicPtr::fetch_* since it is strict-provenance\n // compatible, but it requires 1.91+. So, for now emulate it only on cfg(miri).\n // Code using AtomicUsize::fetch_* via casts is still permissive-provenance\n // compatible and is sound.\n #[cfg(portable_atomic_no_strict_provenance_atomic_ptr)]\n impl AtomicPtr {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_byte_add(&self, val: usize, order: Ordering) -> *mut T {\n #[cfg(miri)]\n {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n self.fetch_update_(order, |x| x.with_addr(x.addr().wrapping_add(val)))\n }\n #[cfg(not(miri))]\n {\n crate::utils::ptr::with_exposed_provenance_mut(\n self.as_atomic_usize().fetch_add(val, order),\n )\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_byte_sub(&self, val: usize, order: Ordering) -> *mut T {\n #[cfg(miri)]\n {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n self.fetch_update_(order, |x| x.with_addr(x.addr().wrapping_sub(val)))\n }\n #[cfg(not(miri))]\n {\n crate::utils::ptr::with_exposed_provenance_mut(\n self.as_atomic_usize().fetch_sub(val, order),\n )\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_or(&self, val: usize, order: Ordering) -> *mut T {\n #[cfg(miri)]\n {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n self.fetch_update_(order, |x| x.with_addr(x.addr() | val))\n }\n #[cfg(not(miri))]\n {\n crate::utils::ptr::with_exposed_provenance_mut(\n self.as_atomic_usize().fetch_or(val, order),\n )\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_and(&self, val: usize, order: Ordering) -> *mut T {\n #[cfg(miri)]\n {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n self.fetch_update_(order, |x| x.with_addr(x.addr() & val))\n }\n #[cfg(not(miri))]\n {\n crate::utils::ptr::with_exposed_provenance_mut(\n self.as_atomic_usize().fetch_and(val, order),\n )\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_xor(&self, val: usize, order: Ordering) -> *mut T {\n #[cfg(miri)]\n {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n self.fetch_update_(order, |x| x.with_addr(x.addr() ^ val))\n }\n #[cfg(not(miri))]\n {\n crate::utils::ptr::with_exposed_provenance_mut(\n self.as_atomic_usize().fetch_xor(val, order),\n )\n }\n }\n #[cfg(miri)]\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n fn fetch_update_(&self, order: Ordering, mut f: F) -> *mut T\n where\n F: FnMut(*mut T) -> *mut T,\n {\n // This is a private function and all instances of `f` only operate on the value\n // loaded, so there is no need to synchronize the first load/failed CAS.\n let mut prev = self.load(Ordering::Relaxed);\n loop {\n let next = f(prev);\n match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {\n Ok(x) => return x,\n Err(next_prev) => prev = next_prev,\n }\n }\n }\n #[cfg(not(miri))]\n #[inline(always)]\n fn as_atomic_usize(&self) -> &AtomicUsize {\n static_assert!(\n core::mem::size_of::>() == core::mem::size_of::()\n );\n static_assert!(\n core::mem::align_of::>() == core::mem::align_of::()\n );\n // SAFETY: AtomicPtr and AtomicUsize have the same layout,\n // and both access data in the same way.\n unsafe { &*(self as *const Self as *const AtomicUsize) }\n }\n }\n #[cfg(not(all(\n any(target_arch = \"x86\", target_arch = \"x86_64\"),\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n )))]\n impl_default_bit_opts!(AtomicPtr, usize);\n});\nimpl core::ops::Deref for AtomicPtr {\n type Target = core::sync::atomic::AtomicPtr;\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n fn deref(&self) -> &Self::Target {\n &self.inner\n }\n}\n\n// NB: Sync with atomic_int! macro in arm_linux.rs\nmacro_rules! atomic_int {\n ($atomic_type:ident, $int_type:ident) => {\n #[repr(transparent)]\n pub(crate) struct $atomic_type {\n pub(super) inner: core::sync::atomic::$atomic_type,\n // Prevent RefUnwindSafe from being propagated from the std atomic type. See NotRefUnwindSafe for more.\n _not_ref_unwind_safe: PhantomData,\n }\n impl $atomic_type {\n #[inline]\n pub(crate) const fn new(v: $int_type) -> Self {\n Self {\n inner: core::sync::atomic::$atomic_type::new(v),\n _not_ref_unwind_safe: PhantomData,\n }\n }\n #[inline]\n pub(crate) fn is_lock_free() -> bool {\n Self::IS_ALWAYS_LOCK_FREE\n }\n // ESP-IDF targets' 64-bit atomics are not lock-free.\n // https://github.com/rust-lang/rust/pull/115577#issuecomment-1732259297\n pub(crate) const IS_ALWAYS_LOCK_FREE: bool = cfg!(not(all(\n any(target_arch = \"riscv32\", target_arch = \"xtensa\"),\n target_os = \"espidf\",\n ))) | (core::mem::size_of::<$int_type>()\n < 8);\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];\n #[inline]\n pub(crate) const fn as_ptr(&self) -> *mut $int_type {\n // SAFETY: Self is #[repr(C)] and internally UnsafeCell<$int_type>.\n // See also https://github.com/rust-lang/rust/pull/66705 and\n // https://github.com/rust-lang/rust/issues/66136#issuecomment-557867116.\n unsafe {\n (*(self as *const Self as *const UnsafeCell<$int_type>)).get()\n }\n }\n }\n }\n #[cfg(not(all(\n target_arch = \"arm\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(target_os = \"linux\", target_os = \"android\"),\n not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\")),\n )))]\n impl $atomic_type {\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn load(&self, order: Ordering) -> $int_type {\n crate::utils::assert_load_ordering(order); // for track_caller (compiler can omit double check)\n self.inner.load(order)\n }\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn store(&self, val: $int_type, order: Ordering) {\n crate::utils::assert_store_ordering(order); // for track_caller (compiler can omit double check)\n self.inner.store(val, order);\n }\n }\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(not(portable_atomic_no_atomic_cas))\n )]\n #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = \"ptr\"))]\n items!({\n impl_default_no_fetch_ops!($atomic_type, $int_type);\n #[cfg(not(all(\n target_arch = \"arm\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(target_os = \"linux\", target_os = \"android\"),\n not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\")),\n )))]\n impl $atomic_type {\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)\n #[cfg(portable_atomic_no_stronger_failure_ordering)]\n let success = crate::utils::upgrade_success_ordering(success, failure);\n self.inner.compare_exchange(current, new, success, failure)\n }\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange_weak(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure); // for track_caller (compiler can omit double check)\n #[cfg(portable_atomic_no_stronger_failure_ordering)]\n let success = crate::utils::upgrade_success_ordering(success, failure);\n self.inner.compare_exchange_weak(current, new, success, failure)\n }\n }\n impl $atomic_type {\n #[allow(dead_code)]\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n fn fetch_update_(&self, order: Ordering, mut f: F) -> $int_type\n where\n F: FnMut($int_type) -> $int_type,\n {\n // This is a private function and all instances of `f` only operate on the value\n // loaded, so there is no need to synchronize the first load/failed CAS.\n let mut prev = self.load(Ordering::Relaxed);\n loop {\n let next = f(prev);\n match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {\n Ok(x) => return x,\n Err(next_prev) => prev = next_prev,\n }\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {\n #[cfg(not(portable_atomic_no_atomic_min_max))]\n {\n #[cfg(any(\n all(\n any(target_arch = \"aarch64\", target_arch = \"arm64ec\"),\n any(\n target_feature = \"lse\",\n portable_atomic_target_feature = \"lse\",\n ),\n ),\n all(\n target_arch = \"arm\",\n not(any(\n target_feature = \"v6\",\n portable_atomic_target_feature = \"v6\",\n )),\n ),\n target_arch = \"mips\",\n target_arch = \"mips32r6\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_arch = \"powerpc\",\n target_arch = \"powerpc64\",\n ))]\n {\n // HACK: the following operations are currently broken (at least on qemu-user):\n // - aarch64's `AtomicI{8,16}::fetch_{max,min}` (release mode + lse)\n // - armv5te's `Atomic{I,U}{8,16}::fetch_{max,min}`\n // - mips's `AtomicI8::fetch_{max,min}` (release mode)\n // - mipsel's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)\n // - mips64's `AtomicI8::fetch_{max,min}` (release mode)\n // - mips64el's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)\n // - powerpc's `AtomicI{8,16}::fetch_{max,min}`\n // - powerpc64's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)\n // - powerpc64le's `AtomicU{8,16}::fetch_{max,min}` (release mode + fat LTO)\n // See also:\n // https://github.com/llvm/llvm-project/issues/61880\n // https://github.com/llvm/llvm-project/issues/61881\n // https://github.com/llvm/llvm-project/issues/61882\n // https://github.com/taiki-e/portable-atomic/issues/2\n // https://github.com/rust-lang/rust/issues/100650\n if core::mem::size_of::<$int_type>() <= 2 {\n return self.fetch_update_(order, |x| core::cmp::max(x, val));\n }\n }\n self.inner.fetch_max(val, order)\n }\n #[cfg(portable_atomic_no_atomic_min_max)]\n {\n self.fetch_update_(order, |x| core::cmp::max(x, val))\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {\n #[cfg(not(portable_atomic_no_atomic_min_max))]\n {\n #[cfg(any(\n all(\n any(target_arch = \"aarch64\", target_arch = \"arm64ec\"),\n any(\n target_feature = \"lse\",\n portable_atomic_target_feature = \"lse\",\n ),\n ),\n all(\n target_arch = \"arm\",\n not(any(\n target_feature = \"v6\",\n portable_atomic_target_feature = \"v6\",\n )),\n ),\n target_arch = \"mips\",\n target_arch = \"mips32r6\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_arch = \"powerpc\",\n target_arch = \"powerpc64\",\n ))]\n {\n // HACK: the following operations are currently broken (at least on qemu-user):\n // - aarch64's `AtomicI{8,16}::fetch_{max,min}` (release mode + lse)\n // - armv5te's `Atomic{I,U}{8,16}::fetch_{max,min}`\n // - mips's `AtomicI8::fetch_{max,min}` (release mode)\n // - mipsel's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)\n // - mips64's `AtomicI8::fetch_{max,min}` (release mode)\n // - mips64el's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)\n // - powerpc's `AtomicI{8,16}::fetch_{max,min}`\n // - powerpc64's `AtomicI{8,16}::fetch_{max,min}` (debug mode, at least)\n // - powerpc64le's `AtomicU{8,16}::fetch_{max,min}` (release mode + fat LTO)\n // See also:\n // https://github.com/llvm/llvm-project/issues/61880\n // https://github.com/llvm/llvm-project/issues/61881\n // https://github.com/llvm/llvm-project/issues/61882\n // https://github.com/taiki-e/portable-atomic/issues/2\n // https://github.com/rust-lang/rust/issues/100650\n if core::mem::size_of::<$int_type>() <= 2 {\n return self.fetch_update_(order, |x| core::cmp::min(x, val));\n }\n }\n self.inner.fetch_min(val, order)\n }\n #[cfg(portable_atomic_no_atomic_min_max)]\n {\n self.fetch_update_(order, |x| core::cmp::min(x, val))\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {\n self.fetch_xor(!0, order)\n }\n // TODO: provide asm-based implementation on AArch64 without FEAT_LSE, Armv7, RISC-V, etc.\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {\n self.fetch_update_(order, $int_type::wrapping_neg)\n }\n }\n #[cfg(not(all(\n any(target_arch = \"x86\", target_arch = \"x86_64\"),\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n )))]\n items!({\n impl_default_bit_opts!($atomic_type, $int_type);\n impl $atomic_type {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn not(&self, order: Ordering) {\n self.fetch_not(order);\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn neg(&self, order: Ordering) {\n self.fetch_neg(order);\n }\n }\n });\n });\n impl core::ops::Deref for $atomic_type {\n type Target = core::sync::atomic::$atomic_type;\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n fn deref(&self) -> &Self::Target {\n &self.inner\n }\n }\n };\n}\n\natomic_int!(AtomicIsize, isize);\natomic_int!(AtomicUsize, usize);\n#[cfg(not(portable_atomic_no_atomic_load_store))]\natomic_int!(AtomicI8, i8);\n#[cfg(not(portable_atomic_no_atomic_load_store))]\natomic_int!(AtomicU8, u8);\n#[cfg(not(portable_atomic_no_atomic_load_store))]\natomic_int!(AtomicI16, i16);\n#[cfg(not(portable_atomic_no_atomic_load_store))]\natomic_int!(AtomicU16, u16);\n#[cfg(not(portable_atomic_no_atomic_load_store))]\n#[cfg(not(target_pointer_width = \"16\"))]\natomic_int!(AtomicI32, i32);\n#[cfg(not(portable_atomic_no_atomic_load_store))]\n#[cfg(not(target_pointer_width = \"16\"))]\natomic_int!(AtomicU32, u32);\n#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_64)))]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n target_has_atomic = \"64\",\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")),\n ))\n)]\nitems!({\n atomic_int!(AtomicI64, i64);\n atomic_int!(AtomicU64, u64);\n});\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/detect/aarch64_aa64reg.rs", "sha256": "77d6b3a2fafbdeb881c671379d0be0a330b46d78c8252d074c97ed44b1138123", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nRun-time CPU feature detection on AArch64 Linux/Android/FreeBSD/NetBSD/OpenBSD by parsing system registers.\n\nAs of Rust 1.94, is_aarch64_feature_detected doesn't support run-time detection on NetBSD.\nhttps://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs\nRun-time detection on OpenBSD by is_aarch64_feature_detected is supported on Rust 1.70+.\nhttps://github.com/rust-lang/stdarch/pull/1374\n\nRefs:\n- https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers\n- https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm64/cpu-feature-registers.rst\n- https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/aarch64.rs\n\nSupported platforms:\n- Linux 4.11+ (emulate mrs instruction)\n https://github.com/torvalds/linux/commit/77c97b4ee21290f5f083173d957843b615abbff2\n- FreeBSD 12.0+ (emulate mrs instruction)\n https://github.com/freebsd/freebsd-src/commit/398810619cb32abf349f8de23f29510b2ee0839b\n- NetBSD 9.0+ (through sysctl/sysctlbyname)\n https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb\n sysctl/sysctlbyname returns an unsupported error if operation is not supported,\n so we can safely use this on older versions.\n- OpenBSD 7.1+ (through sysctl)\n https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8\n sysctl returns an unsupported error if operation is not supported,\n so we can safely use this on older versions.\n\nFor now, this module is only used on NetBSD/OpenBSD.\n\nOn Linux/Android/FreeBSD, we use auxv.rs and this module is test-only because:\n- On Linux/Android, this approach requires a higher kernel version than Rust supports,\n and also does not work with qemu-user (as of QEMU 7.2) and Valgrind (as of Valgrind 3.24).\n (Looking into HWCAP_CPUID in auxvec, it appears that Valgrind is setting it\n to false correctly, but qemu-user is setting it to true.)\n - qemu-user issue seem to be fixed as of QEMU 9.2.\n- On FreeBSD, this approach does not work on FreeBSD 12 on QEMU (confirmed on\n FreeBSD 12.{2,3,4}), and we got SIGILL (worked on FreeBSD 13 and 14).\n*/\n\ninclude!(\"common.rs\");\n\n#[cfg_attr(test, derive(Debug, PartialEq))]\nstruct AA64Reg {\n aa64isar0: u64,\n aa64isar1: u64,\n #[cfg(test)]\n aa64isar3: u64,\n aa64mmfr2: u64,\n}\n\n#[cold]\n#[must_use]\nfn _detect(mut info: CpuInfo) -> CpuInfo {\n let AA64Reg {\n aa64isar0,\n aa64isar1,\n #[cfg(test)]\n aa64isar3,\n aa64mmfr2,\n } = imp::aa64reg();\n\n // ID_AA64ISAR0_EL1, AArch64 Instruction Set Attribute Register 0\n // https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64ISAR0-EL1--AArch64-Instruction-Set-Attribute-Register-0\n // Atomic, bits [23:20]\n // > FEAT_LSE implements the functionality identified by the value 0b0010.\n // > FEAT_LSE128 implements the functionality identified by the value 0b0011.\n // > From Armv8.1, the value 0b0000 is not permitted.\n let atomic = extract(aa64isar0, 23, 20);\n if atomic >= 0b0010 {\n info.set(CpuInfoFlag::lse);\n if atomic >= 0b0011 {\n info.set(CpuInfoFlag::lse128);\n }\n }\n // ID_AA64ISAR1_EL1, AArch64 Instruction Set Attribute Register 1\n // https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64ISAR1-EL1--AArch64-Instruction-Set-Attribute-Register-1\n // LRCPC, bits [23:20]\n // > FEAT_LRCPC implements the functionality identified by the value 0b0001.\n // > FEAT_LRCPC2 implements the functionality identified by the value 0b0010.\n // > FEAT_LRCPC3 implements the functionality identified by the value 0b0011.\n // > From Armv8.3, the value 0b0000 is not permitted.\n // > From Armv8.4, the value 0b0001 is not permitted.\n let lrcpc = extract(aa64isar1, 23, 20);\n if lrcpc >= 0b0011 {\n info.set(CpuInfoFlag::rcpc3);\n }\n #[cfg(test)]\n if lrcpc >= 0b0001 {\n info.set(CpuInfoFlag::rcpc);\n if lrcpc >= 0b0010 {\n info.set(CpuInfoFlag::rcpc2);\n }\n }\n // LS64, bits [63:60]\n // > FEAT_LS64 implements the functionality identified by 0b0001.\n // > FEAT_LS64_V implements the functionality identified by 0b0010.\n // > FEAT_LS64_ACCDATA implements the functionality identified by 0b0011.\n // > FEAT_LS64WB implements the functionality identified by 0b0100.\n #[cfg(test)]\n if extract(aa64isar1, 63, 60) >= 0b0100 {\n info.set(CpuInfoFlag::ls64wb);\n }\n // ID_AA64ISAR3_EL1, AArch64 Instruction Set Attribute Register 3\n // https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64ISAR3-EL1--AArch64-Instruction-Set-Attribute-Register-3\n // LSFE, bits [19:16]\n // > FEAT_LSFE implements the functionality identified by the value 0b0001\n #[cfg(test)]\n if extract(aa64isar3, 19, 16) >= 0b0001 {\n info.set(CpuInfoFlag::lsfe);\n }\n // ID_AA64MMFR2_EL1, AArch64 Memory Model Feature Register 2\n // https://developer.arm.com/documentation/ddi0601/2025-06/AArch64-Registers/ID-AA64MMFR2-EL1--AArch64-Memory-Model-Feature-Register-2\n // AT, bits [35:32]\n // > FEAT_LSE2 implements the functionality identified by the value 0b0001.\n // > From Armv8.4, the value 0b0000 is not permitted.\n if extract(aa64mmfr2, 35, 32) >= 0b0001 {\n info.set(CpuInfoFlag::lse2);\n }\n info\n}\n\nfn extract(x: u64, high: usize, low: usize) -> u64 {\n (x >> low) & ((1 << (high - low + 1)) - 1)\n}\n\n#[cfg(not(any(target_os = \"netbsd\", target_os = \"openbsd\")))]\nmod imp {\n // This module is test-only. See parent module docs for details.\n\n #[cfg(not(portable_atomic_no_asm))]\n use core::arch::asm;\n\n use super::AA64Reg;\n\n pub(super) fn aa64reg() -> AA64Reg {\n // SAFETY: This is safe on FreeBSD 12.0+. FreeBSD 11 was EoL on 2021-09-30.\n // Note that stdarch has been doing the same thing since before FreeBSD 11 was EoL.\n // https://github.com/rust-lang/stdarch/pull/611\n unsafe {\n let aa64isar0: u64;\n asm!(\n \"mrs {}, ID_AA64ISAR0_EL1\",\n out(reg) aa64isar0,\n options(pure, nomem, nostack, preserves_flags),\n );\n let aa64isar1: u64;\n asm!(\n \"mrs {}, ID_AA64ISAR1_EL1\",\n out(reg) aa64isar1,\n options(pure, nomem, nostack, preserves_flags),\n );\n #[cfg(test)]\n #[cfg(not(portable_atomic_pre_llvm_18))]\n let aa64isar3: u64;\n // ID_AA64ISAR3_EL1 is only recognized on LLVM 18+.\n // https://github.com/llvm/llvm-project/commit/17baba9fa2728b1b1134f9dccb9318debd5a9a1b\n #[cfg(test)]\n #[cfg(not(portable_atomic_pre_llvm_18))]\n asm!(\n \"mrs {}, ID_AA64ISAR3_EL1\",\n out(reg) aa64isar3,\n options(pure, nomem, nostack, preserves_flags),\n );\n let aa64mmfr2: u64;\n asm!(\n \"mrs {}, ID_AA64MMFR2_EL1\",\n out(reg) aa64mmfr2,\n options(pure, nomem, nostack, preserves_flags),\n );\n AA64Reg {\n aa64isar0,\n aa64isar1,\n #[cfg(test)]\n #[cfg(not(portable_atomic_pre_llvm_18))]\n aa64isar3,\n #[cfg(test)]\n #[cfg(portable_atomic_pre_llvm_18)]\n aa64isar3: 0,\n aa64mmfr2,\n }\n }\n }\n}\n#[cfg(target_os = \"netbsd\")]\nmod imp {\n // NetBSD doesn't trap the mrs instruction, but exposes the system registers through sysctl.\n // https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb\n // https://github.com/golang/sys/commit/ef9fd89ba245e184bdd308f7f2b4f3c551fa5b0f\n\n use core::{mem, ptr};\n\n use super::AA64Reg;\n\n // libc requires Rust 1.63\n #[allow(non_camel_case_types)]\n pub(super) mod ffi {\n pub(crate) use crate::utils::ffi::{CStr, c_char, c_int, c_size_t, c_void};\n\n sys_struct!({\n // Defined in machine/armreg.h.\n // https://github.com/NetBSD/src/blob/121914f187d0f46c2ce43f00531d2c500d8e81e5/sys/arch/aarch64/include/armreg.h#L1863\n pub(crate) struct aarch64_sysctl_cpu_id {\n // NetBSD 9.0+\n // https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb\n pub(crate) ac_midr: u64,\n pub(crate) ac_revidr: u64,\n pub(crate) ac_mpidr: u64,\n pub(crate) ac_aa64dfr0: u64,\n pub(crate) ac_aa64dfr1: u64,\n pub(crate) ac_aa64isar0: u64,\n pub(crate) ac_aa64isar1: u64,\n pub(crate) ac_aa64mmfr0: u64,\n pub(crate) ac_aa64mmfr1: u64,\n pub(crate) ac_aa64mmfr2: u64,\n pub(crate) ac_aa64pfr0: u64,\n pub(crate) ac_aa64pfr1: u64,\n pub(crate) ac_aa64zfr0: u64,\n pub(crate) ac_mvfr0: u32,\n pub(crate) ac_mvfr1: u32,\n pub(crate) ac_mvfr2: u32,\n // NetBSD 10.0+\n // https://github.com/NetBSD/src/commit/0c7bdc13f0e332cccec56e307f023b4888638973\n pub(crate) ac_pad: u32,\n pub(crate) ac_clidr: u64,\n pub(crate) ac_ctr: u64,\n }\n });\n\n sys_fn!({\n extern \"C\" {\n // Defined in sys/sysctl.h.\n // https://man.netbsd.org/sysctl.3\n // https://github.com/NetBSD/src/blob/121914f187d0f46c2ce43f00531d2c500d8e81e5/sys/sys/sysctl.h\n pub(crate) fn sysctlbyname(\n name: *const c_char,\n old_p: *mut c_void,\n old_len_p: *mut c_size_t,\n new_p: *const c_void,\n new_len: c_size_t,\n ) -> c_int;\n }\n });\n }\n\n pub(super) fn sysctl_cpu_id(name: &ffi::CStr) -> Option {\n const OUT_LEN: ffi::c_size_t =\n mem::size_of::() as ffi::c_size_t;\n\n // SAFETY: all fields of aarch64_sysctl_cpu_id are zero-able and we use\n // the result when machdep.cpuN.cpu_id sysctl was successful.\n let mut buf: ffi::aarch64_sysctl_cpu_id = unsafe { mem::zeroed() };\n let mut out_len = OUT_LEN;\n // SAFETY:\n // - `name` a valid C string.\n // - `out_len` does not exceed the size of the value at `buf`.\n // - `sysctlbyname` is thread-safe.\n let res = unsafe {\n ffi::sysctlbyname(\n name.as_ptr(),\n (&mut buf as *mut ffi::aarch64_sysctl_cpu_id).cast::(),\n &mut out_len,\n ptr::null_mut(),\n 0,\n )\n };\n if res != 0 {\n return None;\n }\n Some(AA64Reg {\n aa64isar0: buf.ac_aa64isar0,\n aa64isar1: buf.ac_aa64isar1,\n #[cfg(test)]\n aa64isar3: 0,\n aa64mmfr2: buf.ac_aa64mmfr2,\n })\n }\n\n pub(super) fn aa64reg() -> AA64Reg {\n // Get system registers for cpu0.\n // If failed, returns default because machdep.cpuN.cpu_id sysctl is not available.\n // machdep.cpuN.cpu_id sysctl was added in NetBSD 9.0 so it is not available on older versions.\n // It is ok to check only cpu0, even if there are more CPUs.\n // https://github.com/NetBSD/src/commit/bd9707e06ea7d21b5c24df6dfc14cb37c2819416\n // https://github.com/golang/sys/commit/ef9fd89ba245e184bdd308f7f2b4f3c551fa5b0f\n match sysctl_cpu_id(c!(\"machdep.cpu0.cpu_id\")) {\n Some(cpu_id) => cpu_id,\n None => AA64Reg {\n aa64isar0: 0,\n aa64isar1: 0,\n #[cfg(test)]\n aa64isar3: 0,\n aa64mmfr2: 0,\n },\n }\n }\n}\n#[cfg(target_os = \"openbsd\")]\nmod imp {\n // OpenBSD doesn't trap the mrs instruction, but exposes the system registers through sysctl.\n // https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8\n // https://github.com/golang/go/commit/cd54ef1f61945459486e9eea2f016d99ef1da925\n\n use core::{mem, ptr};\n\n use super::AA64Reg;\n\n // libc requires Rust 1.63\n pub(super) mod ffi {\n pub(crate) use crate::utils::ffi::{c_int, c_size_t, c_uint, c_void};\n\n sys_const!({\n // Defined in sys/sysctl.h.\n // https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/sys/sysctl.h\n pub(crate) const CTL_MACHDEP: c_int = 7;\n\n // Defined in machine/cpu.h.\n // https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/arch/arm64/include/cpu.h\n // OpenBSD 7.1+\n // https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8\n pub(crate) const CPU_ID_AA64ISAR0: c_int = 2;\n pub(crate) const CPU_ID_AA64ISAR1: c_int = 3;\n // OpenBSD 7.3+\n // https://github.com/openbsd/src/commit/c7654cd65262d532212f65123ee3905ba200365c\n // However, on OpenBSD 7.3-7.5, querying CPU_ID_AA64MMFR2 always returns 0.\n // https://github.com/openbsd/src/commit/e8331b74e5c20302d4bd948c9db722af688ccfc1\n pub(crate) const CPU_ID_AA64MMFR2: c_int = 7;\n });\n\n sys_fn!({\n extern \"C\" {\n // Defined in sys/sysctl.h.\n // https://man.openbsd.org/sysctl.2\n // https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/sys/sysctl.h\n pub(crate) fn sysctl(\n name: *const c_int,\n name_len: c_uint,\n old_p: *mut c_void,\n old_len_p: *mut c_size_t,\n new_p: *mut c_void,\n new_len: c_size_t,\n ) -> c_int;\n }\n });\n }\n\n // sysctl returns an unsupported error if operation is not supported,\n // so we can safely use this function on older versions of OpenBSD.\n pub(super) fn aa64reg() -> AA64Reg {\n let aa64isar0 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR0]).unwrap_or(0);\n let aa64isar1 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR1]).unwrap_or(0);\n let aa64mmfr2 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64MMFR2]).unwrap_or(0);\n AA64Reg {\n aa64isar0,\n aa64isar1,\n #[cfg(test)]\n aa64isar3: 0,\n aa64mmfr2,\n }\n }\n\n fn sysctl64(mib: &[ffi::c_int]) -> Option {\n const OUT_LEN: ffi::c_size_t = mem::size_of::() as ffi::c_size_t;\n let mut out = 0_u64;\n let mut out_len = OUT_LEN;\n #[allow(clippy::cast_possible_truncation)]\n let mib_len = mib.len() as ffi::c_uint;\n // SAFETY:\n // - `mib_len` does not exceed the size of `mib`.\n // - `out_len` does not exceed the size of `out`.\n // - `sysctl` is thread-safe.\n let res = unsafe {\n ffi::sysctl(\n mib.as_ptr(),\n mib_len,\n (&mut out as *mut u64).cast::(),\n &mut out_len,\n ptr::null_mut(),\n 0,\n )\n };\n // OpenBSD sysctl returns -1 on failure.\n if res == -1 {\n return None;\n }\n debug_assert_eq!(out_len, OUT_LEN);\n Some(out)\n }\n}\n\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks,\n clippy::wildcard_imports\n)]\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n #[cfg_attr(portable_atomic_test_detect_false, ignore = \"detection disabled\")]\n fn test_aa64reg() {\n let AA64Reg { aa64isar0, aa64isar1, aa64isar3, aa64mmfr2 } = imp::aa64reg();\n test_helper::eprintln_nocapture!(\n \"aa64isar0={},aa64isar1={},aa64isar3={},aa64mmfr2={}\",\n aa64isar0,\n aa64isar1,\n aa64isar3,\n aa64mmfr2,\n );\n let atomic = extract(aa64isar0, 23, 20);\n if detect().lse() {\n if detect().lse128() {\n assert_eq!(atomic, 0b0011);\n } else {\n assert_eq!(atomic, 0b0010);\n }\n } else {\n assert_eq!(atomic, 0b0000);\n }\n let lrcpc = extract(aa64isar1, 23, 20);\n if detect().rcpc() {\n if detect().rcpc2() {\n if detect().rcpc3() {\n assert_eq!(lrcpc, 0b0011);\n } else {\n assert_eq!(lrcpc, 0b0010);\n }\n } else {\n assert_eq!(lrcpc, 0b0001);\n }\n } else {\n assert_eq!(lrcpc, 0b0000);\n }\n let ls64 = extract(aa64isar1, 63, 60);\n if detect().ls64wb() {\n assert_eq!(ls64, 0b0100);\n } else {\n assert!(ls64 < 0b0100, \"{}\", ls64);\n }\n let lsfe = extract(aa64isar3, 19, 16);\n if detect().lsfe() {\n assert_eq!(lsfe, 0b0001);\n } else {\n assert_eq!(lsfe, 0b0000);\n }\n let at = extract(aa64mmfr2, 35, 32);\n if detect().lse2() {\n assert_eq!(at, 0b0001);\n } else {\n assert_eq!(at, 0b0000);\n }\n }\n\n #[cfg(target_os = \"openbsd\")]\n #[test]\n fn test_alternative() {\n use std::{format, process::Command, string::String};\n\n // Call sysctl command instead of libc API.\n //\n // This is used only for testing.\n struct SysctlMachdepOutput(String);\n impl SysctlMachdepOutput {\n fn new() -> Self {\n let output = Command::new(\"sysctl\").arg(\"machdep\").output().unwrap();\n assert!(output.status.success());\n let stdout = String::from_utf8(output.stdout).unwrap();\n Self(stdout)\n }\n fn field(&self, name: &str) -> Option {\n Some(\n self.0\n .lines()\n .find_map(|s| s.strip_prefix(&format!(\"{}=\", name)))?\n .parse()\n .unwrap(),\n )\n }\n }\n\n let AA64Reg { aa64isar0, aa64isar1, aa64isar3, aa64mmfr2 } = imp::aa64reg();\n let sysctl_output = SysctlMachdepOutput::new();\n assert_eq!(aa64isar0, sysctl_output.field(\"machdep.id_aa64isar0\").unwrap_or(0));\n assert_eq!(aa64isar1, sysctl_output.field(\"machdep.id_aa64isar1\").unwrap_or(0));\n assert_eq!(aa64isar3, sysctl_output.field(\"machdep.id_aa64isar3\").unwrap_or(0));\n assert_eq!(aa64mmfr2, sysctl_output.field(\"machdep.id_aa64mmfr2\").unwrap_or(0));\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/detect/aarch64_apple.rs", "sha256": "149a79530b9bdb11b1521918a4d61ece2be2cb99ab4c86cb6765c3acde1c727a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nRun-time CPU feature detection on AArch64 Apple targets by using sysctlbyname.\n\nOn macOS, this module is currently only enabled on tests because there are no\ninstructions that were not available on the M1 but are now available on the\nlatest Apple hardware and this library currently wants to use:\n\n```console\n$ LC_ALL=C comm -23 <(rustc --print cfg --target aarch64-apple-darwin -C target-cpu=apple-m5 | grep -F target_feature) <(rustc --print cfg --target aarch64-apple-darwin | grep -F target_feature)\ntarget_feature=\"bf16\"\ntarget_feature=\"bti\"\ntarget_feature=\"cssc\"\ntarget_feature=\"ecv\"\ntarget_feature=\"hbc\"\ntarget_feature=\"i8mm\"\ntarget_feature=\"mte\"\ntarget_feature=\"sme\"\ntarget_feature=\"sme-b16b16\"\ntarget_feature=\"sme-f16f16\"\ntarget_feature=\"sme-f64f64\"\ntarget_feature=\"sme-i16i64\"\ntarget_feature=\"sme2\"\ntarget_feature=\"sme2p1\"\ntarget_feature=\"sve-b16b16\"\ntarget_feature=\"v8.5a\"\ntarget_feature=\"v8.6a\"\ntarget_feature=\"v8.7a\"\ntarget_feature=\"wfxt\"\n```\n\nNon-macOS targets doesn't always supports FEAT_LSE2, so we use this module on them.\nAs used in the example in https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms#Enable-DIT-for-constant-time-cryptographic-operations,\nsysctlbyname is supported on all Apple platforms.\n\nRefs: https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics\n*/\n\ninclude!(\"common.rs\");\n\nuse core::{mem, ptr};\n\n// libc requires Rust 1.63\nmod ffi {\n pub(crate) use crate::utils::ffi::{CStr, c_char, c_int, c_size_t, c_void};\n\n #[cfg(test)] // test-only\n sys_const!({\n // https://github.com/apple-oss-distributions/xnu/blob/xnu-12377.121.6/osfmk/arm/cpu_capabilities_public.h\n pub(crate) const CAP_BIT_FEAT_LSE: usize = 6;\n pub(crate) const CAP_BIT_FEAT_LRCPC: usize = 15;\n pub(crate) const CAP_BIT_FEAT_LRCPC2: usize = 16;\n pub(crate) const CAP_BIT_FEAT_LSE2: usize = 30;\n pub(crate) const CAP_BIT_NB: usize = 92;\n });\n\n sys_fn!({\n extern \"C\" {\n // https://developer.apple.com/documentation/kernel/1387446-sysctlbyname\n // https://github.com/apple-oss-distributions/xnu/blob/xnu-12377.121.6/bsd/sys/sysctl.h\n pub(crate) fn sysctlbyname(\n name: *const c_char,\n old_p: *mut c_void,\n old_len_p: *mut c_size_t,\n new_p: *mut c_void,\n new_len: c_size_t,\n ) -> c_int;\n }\n });\n}\n\nfn sysctlbyname32(name: &ffi::CStr) -> Option {\n const OUT_LEN: ffi::c_size_t = mem::size_of::() as ffi::c_size_t;\n\n let mut out = 0_u32;\n let mut out_len = OUT_LEN;\n // SAFETY:\n // - `name` a valid C string.\n // - `out_len` does not exceed the size of `out`.\n // - `sysctlbyname` is thread-safe.\n let res = unsafe {\n ffi::sysctlbyname(\n name.as_ptr(),\n (&mut out as *mut u32).cast::(),\n &mut out_len,\n ptr::null_mut(),\n 0,\n )\n };\n if res != 0 {\n return None;\n }\n debug_assert_eq!(out_len, OUT_LEN);\n Some(out)\n}\n\n#[cold]\n#[must_use]\nfn _detect(mut info: CpuInfo) -> CpuInfo {\n macro_rules! check {\n ($flag:ident, $($name:tt) ||+) => {\n if $(sysctlbyname32(c!($name)).unwrap_or(0) != 0) ||+ {\n info.set(CpuInfoFlag::$flag);\n }\n };\n }\n\n // On macOS, AArch64 support was added in macOS 11,\n // hw.optional.armv8_1_atomics is available on macOS 11+,\n // hw.optional.arm.FEAT_* are only available on macOS 12+.\n // Query both names in case future versions of macOS remove the old name.\n // https://github.com/golang/go/commit/c15593197453b8bf90fc3a9080ba2afeaf7934ea\n // https://github.com/google/boringssl/commit/91e0b11eba517d83b910b20fe3740eeb39ecb37e\n // hw.optional.arm.caps is available on macOS 15+.\n // https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc#L56\n check!(lse, \"hw.optional.arm.FEAT_LSE\" || \"hw.optional.armv8_1_atomics\");\n check!(lse2, \"hw.optional.arm.FEAT_LSE2\");\n #[cfg(test)]\n check!(rcpc, \"hw.optional.arm.FEAT_LRCPC\");\n #[cfg(test)]\n check!(rcpc2, \"hw.optional.arm.FEAT_LRCPC2\");\n info\n}\n\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks,\n clippy::wildcard_imports\n)]\n#[cfg(test)]\nmod tests {\n use std::{format, process::Command, str, string::String};\n\n use super::*;\n\n #[test]\n fn test_alternative() {\n use crate::utils::ffi::*;\n\n const CAPS_LEN: usize = (ffi::CAP_BIT_NB + 63) / 64;\n fn arm_caps(caps: &mut [u64; CAPS_LEN]) -> bool {\n let mut caps_len = CAPS_LEN * 8;\n // SAFETY:\n // - name a valid C string.\n // - `caps_len` does not exceed the size of `caps`.\n // - `sysctlbyname` is thread-safe.\n let res = unsafe {\n ffi::sysctlbyname(\n c!(\"hw.optional.arm.caps\").as_ptr(),\n caps.as_mut_ptr().cast::(),\n &mut caps_len,\n ptr::null_mut(),\n 0,\n )\n };\n res == 0\n }\n\n // Call sysctl command instead of libc API.\n //\n // This is used only for testing.\n struct SysctlHwOptionalOutput(String);\n impl SysctlHwOptionalOutput {\n fn new() -> Self {\n let output = Command::new(\"sysctl\").arg(\"hw.optional\").output().unwrap();\n assert!(output.status.success());\n let stdout = String::from_utf8(output.stdout).unwrap();\n test_helper::eprintln_nocapture!(\"sysctl hw.optional:\\n{}\", stdout);\n Self(stdout)\n }\n fn field(&self, name: &CStr) -> Option {\n let name = name.to_bytes_with_nul();\n let name = str::from_utf8(&name[..name.len() - 1]).unwrap();\n let prefix = format!(\"{}: \", name);\n Some(self.0.lines().find_map(|s| s.strip_prefix(&prefix))?.parse().unwrap())\n }\n }\n\n let mut caps = [0_u64; CAPS_LEN];\n let has_caps = arm_caps(&mut caps);\n let sysctl_output = SysctlHwOptionalOutput::new();\n for (name, expected_on_macos, cap_bit) in [\n (c!(\"hw.optional.arm.FEAT_LSE\"), Some(1), ffi::CAP_BIT_FEAT_LSE),\n (c!(\"hw.optional.armv8_1_atomics\"), Some(1), ffi::CAP_BIT_FEAT_LSE),\n (c!(\"hw.optional.arm.FEAT_LSE2\"), Some(1), ffi::CAP_BIT_FEAT_LSE2),\n (c!(\"hw.optional.arm.FEAT_LSE128\"), None, 0),\n (c!(\"hw.optional.arm.FEAT_LSFE\"), None, 0),\n (c!(\"hw.optional.arm.FEAT_LRCPC\"), Some(1), ffi::CAP_BIT_FEAT_LRCPC),\n (c!(\"hw.optional.arm.FEAT_LRCPC2\"), Some(1), ffi::CAP_BIT_FEAT_LRCPC2),\n (c!(\"hw.optional.arm.FEAT_LRCPC3\"), None, 0),\n ] {\n let res = sysctlbyname32(name);\n if res.is_none() {\n assert_eq!(std::io::Error::last_os_error().kind(), std::io::ErrorKind::NotFound);\n }\n if cfg!(any(target_os = \"macos\", target_abi = \"macabi\")) {\n assert_eq!(\n res,\n expected_on_macos,\n \"{}\",\n str::from_utf8(name.to_bytes_with_nul()).unwrap()\n );\n }\n if let Some(res) = res {\n assert_eq!(res, sysctl_output.field(name).unwrap());\n if has_caps {\n assert_eq!(caps[cap_bit / 64] & (1 << (cap_bit & 63)) != 0, res != 0);\n }\n } else {\n assert!(sysctl_output.field(name).is_none());\n }\n }\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/detect/aarch64_fuchsia.rs", "sha256": "edb86bdc2630329b96396d3da23d3ee2ca55d3d45737df2de76ac2bfbce4964f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nRun-time CPU feature detection on AArch64 Fuchsia by using zx_system_get_features.\n\nAs of Rust 1.94, is_aarch64_feature_detected doesn't support run-time detection on Fuchsia.\nhttps://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs\n\nRefs:\n- https://fuchsia.dev/reference/syscalls/system_get_features\n- https://github.com/llvm/llvm-project/commit/4e731abc55681751b5d736b613f7720e50eb1ad4\n*/\n\ninclude!(\"common.rs\");\n\n#[allow(non_camel_case_types)]\nmod ffi {\n sys_type!({\n // https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/types.h\n pub(crate) type zx_status_t = i32;\n });\n\n sys_const!({\n // https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/errors.h\n pub(crate) const ZX_OK: zx_status_t = 0;\n\n // https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/features.h\n pub(crate) const ZX_FEATURE_KIND_CPU: u32 = 0;\n pub(crate) const ZX_ARM64_FEATURE_ISA_ATOMICS: u32 = 1 << 8;\n });\n\n // TODO: use sys_fn!\n #[link(name = \"zircon\")]\n extern \"C\" {\n // https://fuchsia.dev/reference/syscalls/system_get_features\n pub(crate) fn zx_system_get_features(kind: u32, features: *mut u32) -> zx_status_t;\n }\n}\n\nfn zx_system_get_features(kind: u32) -> u32 {\n let mut out = 0_u32;\n // SAFETY: the pointer is valid because we got it from a reference.\n let res = unsafe { ffi::zx_system_get_features(kind, &mut out) };\n if res != ffi::ZX_OK {\n return 0;\n }\n out\n}\n\n#[cold]\n#[must_use]\nfn _detect(mut info: CpuInfo) -> CpuInfo {\n let features = zx_system_get_features(ffi::ZX_FEATURE_KIND_CPU);\n macro_rules! check {\n ($flag:ident, $bit:ident) => {\n if features & ffi::$bit != 0 {\n info.set(CpuInfoFlag::$flag);\n }\n };\n }\n check!(lse, ZX_ARM64_FEATURE_ISA_ATOMICS);\n info\n}\n\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks,\n clippy::wildcard_imports\n)]\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn test_fuchsia() {\n let features = zx_system_get_features(ffi::ZX_FEATURE_KIND_CPU);\n test_helper::eprintln_nocapture!(\n \"zx_system_get_features(ZX_FEATURE_KIND_CPU): {:b}\",\n features\n );\n assert_ne!(features, 0);\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/detect/aarch64_illumos.rs", "sha256": "50b95302831a5fe2837fc13b882d6d4e92f79e05b004c4c6cd2a6402d86eefc3", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nRun-time CPU feature detection on AArch64 illumos by using getisax.\n\nAs of Rust 1.94, is_aarch64_feature_detected doesn't support run-time detection on illumos.\nhttps://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs\n*/\n\ninclude!(\"common.rs\");\n\n// libc requires Rust 1.63\nmod ffi {\n pub(crate) use crate::utils::ffi::c_uint;\n\n sys_const!({\n // Defined in sys/auxv_aarch64.h.\n // https://github.com/richlowe/illumos-gate/blob/arm64-gate/usr/src/uts/common/sys/auxv_aarch64.h\n pub(crate) const AV_AARCH64_LSE: u32 = 1 << 15;\n #[cfg(test)]\n pub(crate) const AV_AARCH64_LRCPC: u32 = 1 << 28;\n #[cfg(test)]\n pub(crate) const AV_AARCH64_2_ILRCPC: u32 = 1 << 1;\n pub(crate) const AV_AARCH64_2_LSE2: u32 = 1 << 2;\n });\n\n sys_fn!({\n extern \"C\" {\n // Defined in sys/auxv.h.\n // https://illumos.org/man/2/getisax\n // https://github.com/richlowe/illumos-gate/blob/arm64-gate/usr/src/uts/common/sys/auxv.h\n pub(crate) fn getisax(array: *mut u32, n: c_uint) -> c_uint;\n }\n });\n}\n\n#[cold]\n#[must_use]\nfn _detect(mut info: CpuInfo) -> CpuInfo {\n const OUT_LEN: ffi::c_uint = 2;\n let mut out = [0_u32; OUT_LEN as usize];\n // SAFETY: the pointer is valid because we got it from a reference.\n unsafe {\n ffi::getisax(out.as_mut_ptr(), OUT_LEN);\n }\n macro_rules! check {\n ($x:ident, $flag:ident, $bit:ident) => {\n if $x & ffi::$bit != 0 {\n info.set(CpuInfoFlag::$flag);\n }\n };\n }\n let v1 = out[0];\n check!(v1, lse, AV_AARCH64_LSE);\n #[cfg(test)]\n check!(v1, rcpc, AV_AARCH64_LRCPC);\n let v2 = out[1];\n #[cfg(test)]\n check!(v2, rcpc2, AV_AARCH64_2_ILRCPC);\n check!(v2, lse2, AV_AARCH64_2_LSE2);\n info\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/detect/aarch64_windows.rs", "sha256": "2cfc29df37c09aa8685b47e4973ab11fdb18f66ca9bbc2346e2713ef6898e1e1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nRun-time CPU feature detection on AArch64 Windows by using IsProcessorFeaturePresent.\n\nRun-time detection of FEAT_LSE on Windows by is_aarch64_feature_detected is supported on Rust 1.70+.\nhttps://github.com/rust-lang/stdarch/pull/1373\n\nRefs: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent\n*/\n\ninclude!(\"common.rs\");\n\n// windows-sys requires Rust 1.71\n#[allow(non_camel_case_types, clippy::upper_case_acronyms)]\nmod ffi {\n sys_type!({\n pub(crate) type [Win32::System::Threading] PROCESSOR_FEATURE_ID = u32;\n pub(crate) type [core] BOOL = i32;\n });\n\n sys_const!({\n pub(crate) const [Win32::Foundation] FALSE: BOOL = 0;\n\n // Defined in winnt.h of Windows SDK.\n pub(crate) const [Win32::System::Threading]\n PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE: PROCESSOR_FEATURE_ID = 34;\n #[cfg(test)]\n pub(crate) const [Win32::System::Threading]\n PF_ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE: PROCESSOR_FEATURE_ID = 45;\n });\n // TODO: put in sys_const! once windows-sys updated.\n pub(crate) const PF_ARM_LSE2_AVAILABLE: PROCESSOR_FEATURE_ID = 62;\n\n sys_fn!({\n extern \"system\" {\n // https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent\n pub(crate) fn [Win32::System::Threading] IsProcessorFeaturePresent(\n ProcessorFeature: PROCESSOR_FEATURE_ID,\n ) -> BOOL;\n }\n });\n}\n\n#[cold]\n#[must_use]\nfn _detect(mut info: CpuInfo) -> CpuInfo {\n macro_rules! check {\n ($flag:ident, $bit:ident) => {\n // SAFETY: calling IsProcessorFeaturePresent is safe, and FALSE is also\n // returned if the HAL does not support detection of the specified feature.\n if unsafe { ffi::IsProcessorFeaturePresent(ffi::$bit) != ffi::FALSE } {\n info.set(CpuInfoFlag::$flag);\n }\n };\n }\n check!(lse, PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE);\n check!(lse2, PF_ARM_LSE2_AVAILABLE);\n #[cfg(test)]\n check!(rcpc, PF_ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE);\n info\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/detect/auxv.rs", "sha256": "4897e7d126b69bc7d8121bdc7516aca794c361ed767d6031e61b1e4456f9cae6", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nRun-time CPU feature detection on AArch64/Arm/PowerPC64 Linux/Android/FreeBSD/OpenBSD by parsing ELF auxiliary vectors.\n\nSupported platforms:\n- Linux 6.4+ (through prctl)\n https://github.com/torvalds/linux/commit/ddc65971bb677aa9f6a4c21f76d3133e106f88eb\n prctl returns an unsupported error if operation is not supported,\n so we can safely use this on older versions.\n- glibc 2.16+ (through getauxval)\n https://sourceware.org/git/?p=glibc.git;a=blob;f=NEWS;hb=glibc-2.16#l83\n Always available on:\n - aarch64 (glibc 2.17+ https://sourceware.org/git/?p=glibc.git;a=blob;f=NEWS;hb=glibc-2.17#l36)\n - powerpc64 (le) (glibc 2.19+ or RHEL/CentOS's patched glibc 2.17+ https://sourceware.org/git/?p=glibc.git;a=blob;f=NEWS;hb=glibc-2.19#l108)\n Not always available on:\n - arm (glibc 2.1+ https://sourceware.org/git/?p=glibc.git;a=blob;f=NEWS;hb=glibc-2.1#l97)\n - powerpc64 (be) (glibc 2.3+ https://sourceware.org/git/?p=glibc.git;a=blob;f=NEWS;hb=glibc-2.3#l56)\n Since Rust 1.64, std requires glibc 2.17+ https://blog.rust-lang.org/2022/08/01/Increasing-glibc-kernel-requirements\n- musl 1.1.0+ (through getauxval)\n https://git.musl-libc.org/cgit/musl/tree/WHATSNEW?h=v1.1.0#n1197\n Always available on:\n - aarch64 (musl 1.1.7+ https://git.musl-libc.org/cgit/musl/tree/WHATSNEW?h=v1.1.7#n1422)\n - powerpc64 (musl 1.1.15+ https://git.musl-libc.org/cgit/musl/tree/WHATSNEW?h=v1.1.15#n1702)\n Not always available on:\n - arm (musl 0.8.3+ https://git.musl-libc.org/cgit/musl/tree/WHATSNEW?h=v0.8.3#n354)\n At least since Rust 1.15, std requires musl 1.1.14+ https://github.com/rust-lang/rust/blob/1.15.0/src/ci/docker/x86_64-musl/build-musl.sh#L15\n Since Rust 1.18, std requires musl 1.1.16+ https://github.com/rust-lang/rust/pull/41089\n Since Rust 1.23, std requires musl 1.1.17+ https://github.com/rust-lang/rust/pull/45393\n Since Rust 1.25, std requires musl 1.1.18+ https://github.com/rust-lang/rust/pull/47283\n Since Rust 1.29, std requires musl 1.1.19+ https://github.com/rust-lang/rust/pull/52087\n Since Rust 1.31, std requires musl 1.1.20+ https://github.com/rust-lang/rust/pull/54430\n Since Rust 1.37, std requires musl 1.1.22+ https://github.com/rust-lang/rust/pull/61252\n Since Rust 1.46, std requires musl 1.1.24+ https://github.com/rust-lang/rust/pull/73089\n Since Rust 1.71, std requires musl 1.2.3+ https://blog.rust-lang.org/2023/05/09/Updating-musl-targets\n Since Rust 1.93, std requires musl 1.2.5+ https://blog.rust-lang.org/2025/12/05/Updating-musl-1.2.5/\n OpenHarmony uses a fork of musl 1.2 https://gitee.com/openharmony/docs/blob/master/en/application-dev/reference/native-lib/musl.md\n- uClibc-ng 1.0.43+ (through getauxval)\n https://github.com/wbx-github/uclibc-ng/commit/d869bb1600942c01a77539128f9ba5b5b55ad647\n Not always available on:\n - aarch64 (uClibc-ng 1.0.22+ https://github.com/wbx-github/uclibc-ng/commit/dba942c80dc2cfa5768a856fff98e22a755fdd27)\n - arm (uClibc-ng 1.0.0+ https://github.com/wbx-github/uclibc-ng/tree/v1.0.0/libc/sysdeps/linux)\n (powerpc64 is not supported https://github.com/wbx-github/uclibc-ng/commit/d4d4f37fda7fa57e57132ff2f0d735ce7cc2178e)\n- Picolibc 1.4.6+ (through getauxval)\n https://github.com/picolibc/picolibc/commit/19bfe51d62ad7e32533c7f664b5bca8e26286e31\n The implementation always return 0 (as of 1.8.12): https://github.com/picolibc/picolibc/blob/1.8.12/libc/misc/getauxval.c\n- Android 4.3+ (API level 18+) (through getauxval)\n https://android.googlesource.com/platform/bionic.git/+/2c5153b043b44e9935a334ae9b2d5a4bc5258b40\n https://android.googlesource.com/platform/bionic.git/+/655e430b28d7404f763e7ebefe84fba5a387666d\n Always available on:\n - 64-bit architectures (Android 5.0+ (API level 21+) https://android-developers.googleblog.com/2014/10/whats-new-in-android-50-lollipop.html)\n Not always available on:\n - arm\n Since Rust 1.68, std requires API level 19+ https://blog.rust-lang.org/2023/01/09/android-ndk-update-r25\n Since Rust 1.82, std requires API level 21+ https://github.com/rust-lang/rust/pull/120593\n- FreeBSD 12.0+ and 11.4+ (through elf_aux_info)\n https://github.com/freebsd/freebsd-src/commit/0b08ae2120cdd08c20a2b806e2fcef4d0a36c470\n https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/sys/auxv.h\n Always available on:\n - arm (v7) (FreeBSD 12.0+ https://www.freebsd.org/releases/12.0R/announce, https://man.freebsd.org/cgi/man.cgi?arch)\n - powerpc64 (le) (FreeBSD 12.4+ https://www.freebsd.org/releases/12.4R/announce, https://man.freebsd.org/cgi/man.cgi?arch)\n Not always available on:\n - aarch64 (FreeBSD 11.0+ https://www.freebsd.org/releases/11.0R/announce, https://man.freebsd.org/cgi/man.cgi?arch)\n - arm (v6) (FreeBSD 10.1+ https://www.freebsd.org/releases/10.1R/announce, https://man.freebsd.org/cgi/man.cgi?arch)\n - powerpc64 (be) (FreeBSD 9.0+ https://www.freebsd.org/releases/9.0R/announce, https://man.freebsd.org/cgi/man.cgi?arch)\n Since Rust 1.75, std requires FreeBSD 12+ https://github.com/rust-lang/rust/pull/114521\n Since Rust 1.97, std requires FreeBSD 14+ https://github.com/rust-lang/rust/pull/156607\n- OpenBSD 7.6+ (through elf_aux_info)\n https://github.com/openbsd/src/commit/ef873df06dac50249b2dd380dc6100eee3b0d23d\n Not always available on:\n - aarch64 (OpenBSD 6.1+ https://www.openbsd.org/61.html)\n - arm (OpenBSD 6.0+ https://www.openbsd.org/60.html)\n - powerpc64 (OpenBSD 6.8+ https://www.openbsd.org/68.html)\n\nOn L4Re which uses uClibc-ng, a dummy getauxval was added first in 2020 that always returns 0 (https://github.com/kernkonzept/l4re-core/commit/e88fa67198074d3e6b4983c5c8af1538e2089ff3),\nthen implemented in 2024 (https://github.com/kernkonzept/l4re-core/commit/3ee2a50dd1b3bc22955e593004990887a0a5b4a3).\nHowever, getauxval(AT_HWCAP*) always returns 0 (as of 2025-12-25). (see tests/l4re test)\nOn Redox, getauxval is available since 0.5.0 (https://github.com/redox-os/relibc/commit/f9f752d74c4f1f56a89c0fcdd5cab63d2380fe09),\nbut the implementation always return 0 (as of 2026-08-08): https://github.com/redox-os/relibc/blob/49aeb9470a885030ed43a70d0b708f3caec19478/src/header/sys_auxv/mod.rs\n\nOn platforms that we can assume that getauxval/elf_aux_info is always available, we directly call\nthem on except for musl with static linking. (At this time, we also retain compatibility with\nversions that reached EoL or no longer supported by `std`, with the exception of AArch64 FreeBSD described below.)\n\nOn musl with static linking, it seems that getauxval is not always available, independent of version\nrequirements: https://github.com/rust-lang/rust/issues/89626\n(That problem may have been fixed in https://github.com/rust-lang/rust/commit/9a04ae4997493e9260352064163285cddc43de3c,\nbut even in the version containing that patch, [there is report](https://github.com/rust-lang/rust/issues/89626#issuecomment-1242636038)\nof the same error.)\nThis seems to be due to the fact that compiler-builtins is built before libc (which is a dependency\nof std) links musl. And the std and its dependent can use getauxval without this problem at least\nsince the rust-lang/rust patch mentioned above:\nhttps://github.com/rust-lang/rust/blob/1.85.0/library/std/src/sys/pal/unix/stack_overflow.rs#L268\n(According to https://github.com/rust-lang/rust/issues/89626#issuecomment-2420469392, this problem\nmay have been fixed in https://github.com/rust-lang/rust/commit/9ed0d11efbec18a1fa4155576a3bcb685676d23c.)\nSee also https://github.com/rust-lang/stdarch/pull/1746.\nSo as for musl with static linking, we assume that getauxval is always available also when `std` feature enabled.\n\nOn platforms that we cannot assume that getauxval/elf_aux_info is always available, so we use dlsym\ninstead of directly calling getauxval/elf_aux_info. (You can force getauxval/elf_aux_info to be\ncalled directly instead of using dlsym by `--cfg portable_atomic_outline_atomics`).\n\nAlso, note that dlsym usually not working with static linking.\n\n# Linux/Android\n\nAs of Rust 1.69, is_aarch64_feature_detected always uses dlsym by default\non AArch64 Linux/Android, but on some platforms, we can safely assume\ngetauxval is linked to the binary (see the above).\n\nSee also https://github.com/rust-lang/stdarch/pull/1375\n\nSee tests::test_alternative and aarch64_aa64reg.rs for (test-only) alternative implementations.\n\n# FreeBSD\n\nAs of Rust 1.94, is_aarch64_feature_detected always uses mrs on\nAArch64 FreeBSD. However, they do not work on FreeBSD 12 on QEMU (confirmed\non FreeBSD 12.{2,3,4}), and we got SIGILL (worked on FreeBSD 13 and 14).\n\nSo use elf_aux_info instead of mrs like compiler-rt does.\nhttps://reviews.llvm.org/D109330\n\nelf_aux_info is available on FreeBSD 12.0+ and 11.4+:\nhttps://github.com/freebsd/freebsd-src/commit/0b08ae2120cdd08c20a2b806e2fcef4d0a36c470\nhttps://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/sys/auxv.h\nOn FreeBSD, [AArch64 support is available on FreeBSD 11.0+](https://www.freebsd.org/releases/11.0R/announce),\nbut FreeBSD 11 (11.4) was EoL on 2021-09-30, and FreeBSD 11.3 was EoL on 2020-09-30:\nhttps://www.freebsd.org/security/unsupported\nSee also https://github.com/rust-lang/stdarch/pull/611#issuecomment-445464613\n\nSee tests::test_alternative and aarch64_aa64reg.rs for (test-only) alternative implementations.\n\n# OpenBSD\n\nelf_aux_info is available on OpenBSD 7.6+:\nhttps://github.com/openbsd/src/commit/ef873df06dac50249b2dd380dc6100eee3b0d23d\n\nOn AArch64, there is an alternative that available on older version,\nso we use it (see aarch64_aa64reg.rs).\n\n*/\n\ninclude!(\"common.rs\");\n\nuse self::os::ffi;\n#[cfg(any(target_os = \"linux\", target_os = \"android\"))]\nmod os {\n // libc requires Rust 1.63\n #[cfg_attr(test, allow(dead_code))]\n pub(super) mod ffi {\n pub(crate) use crate::utils::ffi::c_ulong;\n #[allow(unused_imports)]\n pub(crate) use crate::utils::ffi::{c_char, c_int, c_void};\n\n sys_const!({\n // https://github.com/torvalds/linux/blob/v6.19/include/uapi/linux/auxvec.h\n pub(crate) const AT_HWCAP: c_ulong = 16;\n #[cfg(any(\n test,\n all(target_arch = \"aarch64\", target_pointer_width = \"64\"),\n target_arch = \"powerpc64\",\n ))]\n pub(crate) const AT_HWCAP2: c_ulong = 26;\n // Linux 6.9+\n // https://github.com/torvalds/linux/commit/3281366a8e79a512956382885091565db1036b64\n #[cfg(test)]\n #[cfg(not(all(target_arch = \"aarch64\", target_pointer_width = \"32\")))]\n pub(crate) const AT_HWCAP3: c_ulong = 29;\n #[cfg(test)]\n #[cfg(not(all(target_arch = \"aarch64\", target_pointer_width = \"32\")))]\n pub(crate) const AT_HWCAP4: c_ulong = 30;\n\n // Defined in dlfcn.h.\n // https://sourceware.org/git/?p=glibc.git;a=blob;f=dlfcn/dlfcn.h;hb=glibc-2.43\n // https://git.musl-libc.org/cgit/musl/tree/include/dlfcn.h?h=v1.2.5\n // https://github.com/wbx-github/uclibc-ng/blob/v1.0.47/include/dlfcn.h\n // https://github.com/kernkonzept/l4re-core/blob/r-2026-W07/libc/uclibc-ng/contrib/uclibc/include/dlfcn.h\n // https://android.googlesource.com/platform/bionic.git/+/refs/tags/android-16.0.0_r1/libc/include/dlfcn.h\n #[cfg(any(\n test,\n not(any(\n all(\n target_os = \"linux\",\n any(\n all(\n target_env = \"gnu\",\n any(\n target_arch = \"aarch64\",\n all(target_arch = \"powerpc64\", target_endian = \"little\"),\n ),\n ),\n target_env = \"musl\",\n target_env = \"ohos\",\n ),\n ),\n all(target_os = \"android\", target_pointer_width = \"64\"),\n portable_atomic_outline_atomics,\n )),\n ))]\n #[cfg(not(all(target_os = \"android\", target_pointer_width = \"32\")))]\n pub(crate) const RTLD_DEFAULT: *mut c_void = core::ptr::null_mut();\n #[cfg(all(target_os = \"android\", target_pointer_width = \"32\"))]\n #[allow(clippy::cast_sign_loss)]\n pub(crate) const RTLD_DEFAULT: *mut c_void =\n crate::utils::ptr::without_provenance_mut(-1_isize as usize);\n\n // Defined in sys/system_properties.h.\n // https://android.googlesource.com/platform/bionic.git/+/refs/tags/android-16.0.0_r1/libc/include/sys/system_properties.h\n #[cfg(all(target_arch = \"aarch64\", target_os = \"android\"))]\n pub(crate) const PROP_VALUE_MAX: c_int = 92;\n });\n\n sys_fn!({\n extern \"C\" {\n // Defined in sys/auxv.h.\n // https://man7.org/linux/man-pages/man3/getauxval.3.html\n // https://sourceware.org/git/?p=glibc.git;a=blob;f=misc/sys/auxv.h;hb=glibc-2.43\n // https://git.musl-libc.org/cgit/musl/tree/include/sys/auxv.h?h=v1.2.5\n // https://github.com/wbx-github/uclibc-ng/blob/v1.0.47/include/sys/auxv.h\n // https://github.com/kernkonzept/l4re-core/blob/r-2026-W07/libc/uclibc-ng/contrib/uclibc/include/sys/auxv.h\n // https://android.googlesource.com/platform/bionic.git/+/refs/tags/android-16.0.0_r1/libc/include/sys/auxv.h\n // https://github.com/picolibc/picolibc/blob/1.8.12/libc/include/sys/auxv.h\n #[cfg(any(\n test,\n all(\n target_os = \"linux\",\n any(\n all(\n target_env = \"gnu\",\n any(\n target_arch = \"aarch64\",\n all(target_arch = \"powerpc64\", target_endian = \"little\"),\n ),\n ),\n target_env = \"musl\",\n target_env = \"ohos\",\n ),\n ),\n all(target_os = \"android\", target_pointer_width = \"64\"),\n portable_atomic_outline_atomics,\n ))]\n pub(crate) fn getauxval(type_: c_ulong) -> c_ulong;\n\n // Defined in dlfcn.h.\n // https://man7.org/linux/man-pages/man3/dlsym.3.html\n // https://sourceware.org/git/?p=glibc.git;a=blob;f=dlfcn/dlfcn.h;hb=glibc-2.43\n // https://git.musl-libc.org/cgit/musl/tree/include/dlfcn.h?h=v1.2.5\n // https://github.com/wbx-github/uclibc-ng/blob/v1.0.47/include/dlfcn.h\n // https://github.com/kernkonzept/l4re-core/blob/r-2026-W07/libc/uclibc-ng/contrib/uclibc/include/dlfcn.h\n // https://android.googlesource.com/platform/bionic.git/+/refs/tags/android-16.0.0_r1/libc/include/dlfcn.h\n #[cfg(any(\n test,\n not(any(\n all(\n target_os = \"linux\",\n any(\n all(\n target_env = \"gnu\",\n any(\n target_arch = \"aarch64\",\n all(target_arch = \"powerpc64\", target_endian = \"little\"),\n ),\n ),\n target_env = \"musl\",\n target_env = \"ohos\",\n ),\n ),\n all(target_os = \"android\", target_pointer_width = \"64\"),\n portable_atomic_outline_atomics,\n )),\n ))]\n pub(crate) fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void;\n\n // Defined in sys/system_properties.h.\n // https://android.googlesource.com/platform/bionic.git/+/refs/tags/android-16.0.0_r1/libc/include/sys/system_properties.h\n #[cfg(all(target_arch = \"aarch64\", target_os = \"android\"))]\n pub(crate) fn __system_property_get(\n name: *const c_char,\n value: *mut c_char,\n ) -> c_int;\n }\n });\n }\n\n pub(super) type GetauxvalTy = unsafe extern \"C\" fn(ffi::c_ulong) -> ffi::c_ulong;\n pub(super) fn getauxval(type_: ffi::c_ulong) -> ffi::c_ulong {\n cfg_sel!({\n #[cfg(any(\n all(\n target_os = \"linux\",\n any(\n all(\n target_env = \"gnu\",\n any(\n target_arch = \"aarch64\",\n all(target_arch = \"powerpc64\", target_endian = \"little\"),\n ),\n ),\n target_env = \"musl\",\n target_env = \"ohos\",\n ),\n ),\n all(target_os = \"android\", target_pointer_width = \"64\"),\n portable_atomic_outline_atomics,\n ))]\n {\n let getauxval: GetauxvalTy = ffi::getauxval;\n }\n #[cfg(else)]\n {\n // SAFETY: we passed a valid C string to dlsym, and a pointer returned by dlsym\n // is a valid pointer to the function if it is non-null.\n let getauxval: GetauxvalTy = unsafe {\n let ptr = ffi::dlsym(ffi::RTLD_DEFAULT, c!(\"getauxval\").as_ptr());\n if ptr.is_null() {\n return 0;\n }\n core::mem::transmute::<*mut ffi::c_void, GetauxvalTy>(ptr)\n };\n }\n });\n\n // SAFETY: `getauxval` is thread-safe.\n unsafe { getauxval(type_) }\n }\n}\n#[cfg(any(target_os = \"freebsd\", target_os = \"openbsd\"))]\nmod os {\n use core::mem;\n\n // libc requires Rust 1.63\n #[cfg_attr(test, allow(dead_code))]\n pub(super) mod ffi {\n #[allow(unused_imports)]\n pub(crate) use crate::utils::ffi::c_char;\n pub(crate) use crate::utils::ffi::{c_int, c_ulong, c_void};\n\n sys_const!({\n // FreeBSD\n // Defined in sys/elf_common.h.\n // https://github.com/freebsd/freebsd-src/blob/release/15.1.0/sys/sys/elf_common.h\n // OpenBSD\n // Defined in sys/auxv.h.\n // https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/sys/auxv.h\n pub(crate) const AT_HWCAP: c_int = 25;\n #[cfg(any(\n test,\n all(target_os = \"freebsd\", target_arch = \"aarch64\", target_pointer_width = \"64\"),\n target_arch = \"powerpc64\",\n ))]\n pub(crate) const AT_HWCAP2: c_int = 26;\n // FreeBSD 15.0+\n // https://github.com/freebsd/freebsd-src/commit/85007872d1227006adf2ce119fe30de856cbe12d\n #[cfg(test)]\n #[cfg(not(target_os = \"openbsd\"))]\n pub(crate) const AT_HWCAP3: c_int = 38;\n #[cfg(test)]\n #[cfg(not(target_os = \"openbsd\"))]\n pub(crate) const AT_HWCAP4: c_int = 39;\n\n // FreeBSD\n // Defined in dlfcn.h.\n // https://man.freebsd.org/dlsym(3)\n // https://github.com/freebsd/freebsd-src/blob/release/15.1.0/include/dlfcn.h\n // OpenBSD\n // Defined in dlfcn.h.\n // https://man.openbsd.org/dlsym.3\n // https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/include/dlfcn.h\n #[cfg(any(\n test,\n not(any(\n all(\n target_os = \"freebsd\",\n any(\n target_arch = \"aarch64\",\n all(target_arch = \"powerpc64\", target_endian = \"little\"),\n ),\n ),\n portable_atomic_outline_atomics,\n )),\n ))]\n #[allow(clippy::cast_sign_loss)]\n pub(crate) const RTLD_DEFAULT: *mut c_void =\n crate::utils::ptr::without_provenance_mut(-2_isize as usize);\n });\n\n sys_fn!({\n extern \"C\" {\n // FreeBSD\n // Defined in sys/auxv.h.\n // https://man.freebsd.org/elf_aux_info(3)\n // https://github.com/freebsd/freebsd-src/blob/release/15.1.0/sys/sys/auxv.h\n // OpenBSD\n // Defined in sys/auxv.h.\n // https://man.openbsd.org/elf_aux_info.3\n // https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/sys/auxv.h\n #[cfg(any(\n test,\n any(\n all(\n target_os = \"freebsd\",\n any(\n target_arch = \"aarch64\",\n all(target_arch = \"powerpc64\", target_endian = \"little\"),\n ),\n ),\n portable_atomic_outline_atomics,\n ),\n ))]\n pub(crate) fn elf_aux_info(aux: c_int, buf: *mut c_void, buf_len: c_int) -> c_int;\n\n // FreeBSD\n // Defined in dlfcn.h.\n // https://man.freebsd.org/dlsym(3)\n // https://github.com/freebsd/freebsd-src/blob/release/15.1.0/include/dlfcn.h\n // OpenBSD\n // Defined in dlfcn.h.\n // https://man.openbsd.org/dlsym.3\n // https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/include/dlfcn.h\n #[cfg(any(\n test,\n not(any(\n all(\n target_os = \"freebsd\",\n any(\n target_arch = \"aarch64\",\n all(target_arch = \"powerpc64\", target_endian = \"little\"),\n ),\n ),\n portable_atomic_outline_atomics,\n )),\n ))]\n pub(crate) fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void;\n }\n });\n }\n\n pub(super) type ElfAuxInfoTy =\n unsafe extern \"C\" fn(ffi::c_int, *mut ffi::c_void, ffi::c_int) -> ffi::c_int;\n pub(super) fn getauxval(aux: ffi::c_int) -> ffi::c_ulong {\n #[allow(clippy::cast_possible_wrap, clippy::cast_possible_truncation)]\n const OUT_LEN: ffi::c_int = mem::size_of::() as ffi::c_int;\n\n cfg_sel!({\n #[cfg(any(\n all(\n target_os = \"freebsd\",\n any(\n target_arch = \"aarch64\",\n all(target_arch = \"powerpc64\", target_endian = \"little\"),\n ),\n ),\n portable_atomic_outline_atomics,\n ))]\n {\n let elf_aux_info: ElfAuxInfoTy = ffi::elf_aux_info;\n }\n #[cfg(else)]\n {\n // SAFETY: we passed a valid C string to dlsym, and a pointer returned by dlsym\n // is a valid pointer to the function if it is non-null.\n let elf_aux_info: ElfAuxInfoTy = unsafe {\n let ptr = ffi::dlsym(ffi::RTLD_DEFAULT, c!(\"elf_aux_info\").as_ptr());\n if ptr.is_null() {\n return 0;\n }\n mem::transmute::<*mut ffi::c_void, ElfAuxInfoTy>(ptr)\n };\n }\n });\n\n let mut out: ffi::c_ulong = 0;\n // SAFETY:\n // - the pointer is valid because we got it from a reference.\n // - `OUT_LEN` is the same as the size of `out`.\n // - `elf_aux_info` is thread-safe.\n let res = unsafe {\n elf_aux_info(aux, (&mut out as *mut ffi::c_ulong).cast::(), OUT_LEN)\n };\n // If elf_aux_info fails, `out` will be left at zero (which is the proper default value).\n debug_assert!(res == 0 || out == 0);\n out\n }\n}\n\n// Basically, Linux/FreeBSD/OpenBSD use the same hwcap values.\n// FreeBSD and OpenBSD usually support a subset of the hwcap values supported by Linux.\nuse self::arch::_detect;\n#[cfg(target_arch = \"aarch64\")]\nmod arch {\n use super::{CpuInfo, CpuInfoFlag, ffi, os};\n\n sys_const!({\n // Linux\n // https://github.com/torvalds/linux/blob/v6.19/arch/arm64/include/uapi/asm/hwcap.h\n // https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/arm64/elf_hwcaps.rst\n // FreeBSD\n // Defined in machine/elf.h.\n // https://github.com/freebsd/freebsd-src/blob/release/15.1.0/sys/arm64/include/elf.h\n // OpenBSD\n // Defined in machine/elf.h.\n // https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/arch/arm64/include/elf.h\n // Linux 4.3+\n // https://github.com/torvalds/linux/commit/40a1db2434a1b62332b1af25cfa14d7b8c0301fe\n // FreeBSD 13.0+/12.2+\n // https://github.com/freebsd/freebsd-src/blob/release/13.0.0/sys/arm64/include/elf.h\n // https://github.com/freebsd/freebsd-src/blob/release/12.2.0/sys/arm64/include/elf.h\n // OpenBSD 7.6+\n // https://github.com/openbsd/src/commit/ef873df06dac50249b2dd380dc6100eee3b0d23d\n pub(crate) const HWCAP_ATOMICS: ffi::c_ulong = 1 << 8;\n // Linux 4.11+\n // https://github.com/torvalds/linux/commit/77c97b4ee21290f5f083173d957843b615abbff2\n // FreeBSD 13.0+/12.2+\n // https://github.com/freebsd/freebsd-src/blob/release/13.0.0/sys/arm64/include/elf.h\n // https://github.com/freebsd/freebsd-src/blob/release/12.2.0/sys/arm64/include/elf.h\n // OpenBSD 7.6+\n // https://github.com/openbsd/src/commit/ef873df06dac50249b2dd380dc6100eee3b0d23d\n #[cfg(test)]\n pub(crate) const HWCAP_CPUID: ffi::c_ulong = 1 << 11;\n // Linux 4.12+\n // https://github.com/torvalds/linux/commit/c651aae5a7732287c1c9bc974ece4ed798780544\n // FreeBSD 13.0+/12.2+\n // https://github.com/freebsd/freebsd-src/blob/release/13.0.0/sys/arm64/include/elf.h\n // https://github.com/freebsd/freebsd-src/blob/release/12.2.0/sys/arm64/include/elf.h\n // OpenBSD 7.6+\n // https://github.com/openbsd/src/commit/ef873df06dac50249b2dd380dc6100eee3b0d23d\n #[cfg(test)]\n pub(crate) const HWCAP_LRCPC: ffi::c_ulong = 1 << 15;\n // Linux 4.17+\n // https://github.com/torvalds/linux/commit/7206dc93a58fb76421c4411eefa3c003337bcb2d\n // FreeBSD 13.0+/12.2+\n // https://github.com/freebsd/freebsd-src/blob/release/13.0.0/sys/arm64/include/elf.h\n // https://github.com/freebsd/freebsd-src/blob/release/12.2.0/sys/arm64/include/elf.h\n // OpenBSD 7.6+\n // https://github.com/openbsd/src/commit/ef873df06dac50249b2dd380dc6100eee3b0d23d\n pub(crate) const HWCAP_USCAT: ffi::c_ulong = 1 << 25;\n #[cfg(test)]\n pub(crate) const HWCAP_ILRCPC: ffi::c_ulong = 1 << 26;\n // Linux 6.7+\n // https://github.com/torvalds/linux/commit/338a835f40a849cd89b993e342bd9fbd5684825c\n // FreeBSD 15.0+\n // https://github.com/freebsd/freebsd-src/commit/94686b081fdb0c1bb0fc1dfeda14bd53f26ce7c5\n #[cfg(not(target_os = \"openbsd\"))]\n #[cfg(target_pointer_width = \"64\")]\n pub(crate) const HWCAP2_LRCPC3: ffi::c_ulong = 1 << 46;\n // Linux 6.7+\n // https://github.com/torvalds/linux/commit/94d0657f9f0d311489606589133ebf49e28104d8\n // FreeBSD 15.0+\n // https://github.com/freebsd/freebsd-src/commit/94686b081fdb0c1bb0fc1dfeda14bd53f26ce7c5\n #[cfg(not(target_os = \"openbsd\"))]\n #[cfg(target_pointer_width = \"64\")]\n pub(crate) const HWCAP2_LSE128: ffi::c_ulong = 1 << 47;\n // Linux 6.18+\n // https://github.com/torvalds/linux/commit/220928e52cb03d223b3acad3888baf0687486d21\n // FreeBSD 16.0+\n // https://github.com/freebsd/freebsd-src/commit/3a960425df759a7bb8f946d23f035c63f3a5de7a\n #[cfg(test)]\n #[cfg(not(target_os = \"openbsd\"))]\n #[cfg(target_pointer_width = \"64\")]\n pub(crate) const HWCAP3_LSFE: ffi::c_ulong = 1 << 2;\n });\n\n #[cold]\n #[must_use]\n pub(crate) fn _detect(mut info: CpuInfo) -> CpuInfo {\n #[cfg(target_os = \"android\")]\n {\n // Samsung Exynos 9810 has a bug that big and little cores have different\n // ISAs. And on older Android (pre-9), the kernel incorrectly reports\n // that features available only on some cores are available on all cores.\n // https://reviews.llvm.org/D114523\n let mut arch = [0_u8; ffi::PROP_VALUE_MAX as usize];\n // SAFETY: we've passed a valid C string and a buffer with max length.\n let len = unsafe {\n ffi::__system_property_get(\n c!(\"ro.arch\").as_ptr(),\n arch.as_mut_ptr().cast::(),\n )\n };\n // On Exynos, ro.arch is not available on Android 12+, but it is fine\n // because Android 9+ includes the fix.\n if len > 0 && arch.starts_with(b\"exynos9810\") {\n return info;\n }\n }\n\n macro_rules! check {\n ($x:ident, $flag:ident, $bit:ident) => {\n if $x & $bit != 0 {\n info.set(CpuInfoFlag::$flag);\n }\n };\n }\n let hwcap = os::getauxval(ffi::AT_HWCAP);\n check!(hwcap, lse, HWCAP_ATOMICS);\n check!(hwcap, lse2, HWCAP_USCAT);\n #[cfg(test)]\n check!(hwcap, rcpc, HWCAP_LRCPC);\n #[cfg(test)]\n check!(hwcap, rcpc2, HWCAP_ILRCPC);\n #[cfg(test)]\n check!(hwcap, cpuid, HWCAP_CPUID);\n #[cfg(not(target_os = \"openbsd\"))]\n // HWCAP2 is not yet available on ILP32: https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git/tree/arch/arm64/include/uapi/asm/hwcap.h?h=staging/ilp32-5.1\n #[cfg(target_pointer_width = \"64\")]\n {\n let hwcap2 = os::getauxval(ffi::AT_HWCAP2);\n check!(hwcap2, rcpc3, HWCAP2_LRCPC3);\n check!(hwcap2, lse128, HWCAP2_LSE128);\n #[cfg(test)]\n {\n let hwcap3 = os::getauxval(ffi::AT_HWCAP3);\n check!(hwcap3, lsfe, HWCAP3_LSFE);\n }\n }\n info\n }\n}\n#[cfg(target_arch = \"arm\")]\nmod arch {\n use super::{CpuInfo, CpuInfoFlag, ffi, os};\n\n sys_const!({\n // Linux\n // https://github.com/torvalds/linux/blob/v6.19/arch/arm/include/uapi/asm/hwcap.h\n // FreeBSD\n // Defined in machine/elf.h.\n // https://github.com/freebsd/freebsd-src/blob/release/15.1.0/sys/arm/include/elf.h\n // OpenBSD\n // Defined in machine/elf.h.\n // https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/arch/arm/include/elf.h\n // Linux 3.11+\n // https://github.com/torvalds/linux/commit/a469abd0f868c902b75532579bf87553dcf1b360\n // FreeBSD 12.0+/11.2+\n // https://github.com/freebsd/freebsd-src/commit/0cbf724ed03571bc90ed22c3b4bf8c6c7b2da564\n // https://github.com/freebsd/freebsd-src/blob/release/11.2.0/sys/arm/include/elf.h\n // OpenBSD 7.6+\n // https://github.com/openbsd/src/commit/ef873df06dac50249b2dd380dc6100eee3b0d23d\n #[cfg(test)]\n pub(crate) const HWCAP_LPAE: ffi::c_ulong = 1 << 20;\n });\n\n #[cold]\n #[must_use]\n pub(crate) fn _detect(mut info: CpuInfo) -> CpuInfo {\n macro_rules! check {\n ($x:ident, $flag:ident, $($bit:ident) ||+) => {\n if $x & ($($bit) |+) != 0 {\n info.set(CpuInfoFlag::$flag);\n }\n };\n ($x:ident, $flag:ident, $($bit:ident) &&+) => {\n if $x & ($($bit) |+) == ($($bit) |+) {\n info.set(CpuInfoFlag::$flag);\n }\n };\n }\n let hwcap = os::getauxval(ffi::AT_HWCAP);\n #[cfg(test)]\n check!(hwcap, lpae, HWCAP_LPAE);\n info\n }\n}\n#[cfg(target_arch = \"powerpc64\")]\nmod arch {\n use super::{CpuInfo, CpuInfoFlag, ffi, os};\n\n sys_const!({\n // Linux\n // https://github.com/torvalds/linux/blob/v6.19/arch/powerpc/include/uapi/asm/cputable.h\n // https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/powerpc/elf_hwcaps.rst\n // FreeBSD\n // Defined in machine/cpu.h.\n // https://github.com/freebsd/freebsd-src/blob/release/15.1.0/sys/powerpc/include/cpu.h\n // OpenBSD\n // Defined in machine/elf.h.\n // https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/arch/powerpc64/include/elf.h\n // Linux 2.6.16+\n // https://github.com/torvalds/linux/commit/80f15dc703b3677d0b025bafd215f1f3664c8978\n // FreeBSD 11.0+\n // https://github.com/freebsd/freebsd-src/commit/b0bf7fcd298133457991b27625bbed766e612730\n // OpenBSD 7.6+\n // https://github.com/openbsd/src/commit/0b0568a19fc4c197871ceafbabc91fabf17ca152\n pub(crate) const PPC_FEATURE_BOOKE: ffi::c_ulong = 0x00008000;\n // Linux 3.10+\n // https://github.com/torvalds/linux/commit/cbbc6f1b1433ef553d57826eee87a84ca49645ce\n // FreeBSD 11.0+\n // https://github.com/freebsd/freebsd-src/commit/b0bf7fcd298133457991b27625bbed766e612730\n // OpenBSD 7.6+\n // https://github.com/openbsd/src/commit/0b0568a19fc4c197871ceafbabc91fabf17ca152\n pub(crate) const PPC_FEATURE2_ARCH_2_07: ffi::c_ulong = 0x80000000;\n // Linux 4.5+\n // https://github.com/torvalds/linux/commit/e708c24cd01ce80b1609d8baccee40ccc3608a01\n // FreeBSD 12.0+\n // https://github.com/freebsd/freebsd-src/commit/18f48e0c72f91bc2d4373078a3f1ab1bcab4d8b3\n // OpenBSD 7.6+\n // https://github.com/openbsd/src/commit/0b0568a19fc4c197871ceafbabc91fabf17ca152\n pub(crate) const PPC_FEATURE2_ARCH_3_00: ffi::c_ulong = 0x00800000;\n // Linux 5.8+\n // https://github.com/torvalds/linux/commit/ee988c11acf6f9464b7b44e9a091bf6afb3b3a49\n // FreeBSD 15.0+/14.2+\n // https://github.com/freebsd/freebsd-src/commit/1e434da3b065ef96b389e5e0b604ae05a51e794e\n // https://github.com/freebsd/freebsd-src/blob/release/14.2.0/sys/powerpc/include/cpu.h\n // OpenBSD 7.7+\n // https://github.com/openbsd/src/commit/483a78e15aaa23c010911940770c1c97db5c1287\n pub(crate) const PPC_FEATURE2_ARCH_3_1: ffi::c_ulong = 0x00040000;\n });\n\n #[cold]\n #[must_use]\n pub(crate) fn _detect(mut info: CpuInfo) -> CpuInfo {\n let hwcap = os::getauxval(ffi::AT_HWCAP);\n if hwcap & PPC_FEATURE_BOOKE != 0 {\n // quadword-atomics (Load/Store Quadword category in ISA 2.07) is requirement of ISA 2.07\n // server processors. It is always optional in ISA 2.07 BookE (embedded category)\n // processors and there is no corresponding HWCAP bit. (Although there are no ISA 2.07\n // BookE processors that appear to be supported on these platforms.)\n // Refs: Appendix B \"Platform Support Requirements\" of Power ISA 2.07B\n // https://ibm.ent.box.com/s/jd5w15gz301s5b5dt375mshpq9c3lh4u\n return info;\n }\n let hwcap2 = os::getauxval(ffi::AT_HWCAP2);\n // Check both 2_07 and later ISAs (which are superset of 2_07) because\n // OpenBSD (as of 8.0-beta) doesn't set 2_07 even when 3_00 is set.\n // https://github.com/openbsd/src/blob/d2e81eb18e50973c87ae2c9c2ba348ff3ff89cf9/sys/arch/powerpc64/powerpc64/cpu.c#L222-L238\n // Other OSes should be fine, but check all OSs in the same way just in case.\n let isa_2_07_or_later =\n PPC_FEATURE2_ARCH_2_07 | PPC_FEATURE2_ARCH_3_00 | PPC_FEATURE2_ARCH_3_1;\n if hwcap2 & isa_2_07_or_later != 0 {\n info.set(CpuInfoFlag::quadword_atomics);\n }\n info\n }\n}\n\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks,\n clippy::wildcard_imports\n)]\n#[cfg(test)]\nmod tests {\n use std::mem;\n\n use super::*;\n\n #[allow(clippy::cast_sign_loss)]\n #[cfg(all(target_arch = \"aarch64\", target_os = \"android\"))]\n #[test]\n fn test_android() {\n use std::{slice, str};\n unsafe {\n let mut arch = [1; ffi::PROP_VALUE_MAX as usize];\n let len = ffi::__system_property_get(\n c!(\"ro.arch\").as_ptr(),\n arch.as_mut_ptr().cast::(),\n );\n assert!(len >= 0);\n test_helper::eprintln_nocapture!(\"ro.arch=raw={:?},len={}\", arch, len);\n test_helper::eprintln_nocapture!(\n \"ro.arch={:?}\",\n str::from_utf8(slice::from_raw_parts(arch.as_ptr(), len as usize)).unwrap()\n );\n }\n }\n\n #[cfg(any(target_os = \"linux\", target_os = \"android\"))]\n #[test]\n fn test_dlsym_getauxval() {\n unsafe {\n let ptr = ffi::dlsym(ffi::RTLD_DEFAULT, c!(\"getauxval\").as_ptr());\n if cfg!(target_feature = \"crt-static\") {\n assert!(ptr.is_null());\n } else if cfg!(any(\n all(\n target_os = \"linux\",\n any(target_env = \"gnu\", target_env = \"musl\", target_env = \"ohos\"),\n ),\n target_os = \"android\",\n )) {\n assert!(!ptr.is_null());\n } else if option_env!(\"CI\").is_some() {\n assert!(ptr.is_null());\n }\n if ptr.is_null() {\n return;\n }\n let dlsym_getauxval = mem::transmute::<*mut ffi::c_void, os::GetauxvalTy>(ptr);\n for &at in &[ffi::AT_HWCAP, ffi::AT_HWCAP2] {\n assert_eq!(dlsym_getauxval(at), ffi::getauxval(at));\n }\n #[cfg(not(all(target_arch = \"aarch64\", target_pointer_width = \"32\")))]\n for &at in &[ffi::AT_HWCAP3, ffi::AT_HWCAP4] {\n assert_eq!(dlsym_getauxval(at), ffi::getauxval(at));\n }\n }\n }\n #[cfg(any(target_os = \"freebsd\", target_os = \"openbsd\"))]\n #[test]\n fn test_dlsym_elf_aux_info() {\n unsafe {\n let ptr = ffi::dlsym(ffi::RTLD_DEFAULT, c!(\"elf_aux_info\").as_ptr());\n if cfg!(target_feature = \"crt-static\") {\n assert!(ptr.is_null());\n } else if cfg!(target_os = \"freebsd\") || option_env!(\"CI\").is_some() {\n assert!(!ptr.is_null());\n }\n if ptr.is_null() {\n return;\n }\n let dlsym_elf_aux_info = mem::transmute::<*mut ffi::c_void, os::ElfAuxInfoTy>(ptr);\n #[allow(clippy::cast_possible_wrap, clippy::cast_possible_truncation)]\n let out_len = mem::size_of::() as ffi::c_int;\n for &at in &[ffi::AT_HWCAP, ffi::AT_HWCAP2] {\n let mut out: ffi::c_ulong = 0;\n let mut dlsym_out: ffi::c_ulong = 0;\n assert_eq!(\n ffi::elf_aux_info(\n at,\n (&mut out as *mut ffi::c_ulong).cast::(),\n out_len,\n ),\n dlsym_elf_aux_info(\n at,\n (&mut dlsym_out as *mut ffi::c_ulong).cast::(),\n out_len,\n ),\n );\n assert_eq!(out, dlsym_out);\n }\n #[cfg(not(target_os = \"openbsd\"))]\n for &at in &[ffi::AT_HWCAP3, ffi::AT_HWCAP4] {\n let mut out: ffi::c_ulong = 0;\n let mut dlsym_out: ffi::c_ulong = 0;\n assert_eq!(\n ffi::elf_aux_info(\n at,\n (&mut out as *mut ffi::c_ulong).cast::(),\n out_len,\n ),\n dlsym_elf_aux_info(\n at,\n (&mut dlsym_out as *mut ffi::c_ulong).cast::(),\n out_len,\n ),\n );\n assert_eq!(out, dlsym_out);\n }\n }\n }\n\n #[cfg(any(target_os = \"linux\", target_os = \"android\"))]\n #[cfg(not(all(target_arch = \"aarch64\", target_pointer_width = \"32\")))] // no PR_GET_AUXV\n #[test]\n fn test_alternative() {\n use std::{eprintln, str, vec};\n\n #[cfg(target_pointer_width = \"32\")]\n use sys::Elf32_auxv_t as Elf_auxv_t;\n #[cfg(target_pointer_width = \"64\")]\n use sys::Elf64_auxv_t as Elf_auxv_t;\n use test_helper::sys;\n\n use crate::utils::ffi::*;\n\n // Linux kernel 6.4 has added a way to read auxv without depending on either libc or mrs trap.\n // https://github.com/torvalds/linux/commit/ddc65971bb677aa9f6a4c21f76d3133e106f88eb\n // (Actually 6.5? https://github.com/torvalds/linux/commit/636e348353a7cc52609fdba5ff3270065da140d5)\n //\n // This is currently used only for testing.\n #[cfg(not(target_os = \"android\"))]\n #[cfg(not(portable_atomic_no_asm_syscall))]\n fn getauxval_pr_get_auxv_no_libc(type_: c_ulong) -> Result {\n use crate::utils::{RegISize, RegSize};\n\n unsafe fn prctl_get_auxv(out: *mut c_void, len: usize) -> Result {\n #[cfg_attr(target_arch = \"arm\", allow(unused_variables))]\n let number = sys::__NR_prctl as RegSize;\n let arg1 = sys::PR_GET_AUXV as RegSize;\n let arg2 = ptr_reg!(out);\n let arg3 = len as RegSize;\n let arg4: RegSize = 0;\n let arg5: RegSize = 0;\n let r: RegISize;\n // SAFETY: the caller must uphold the safety contract.\n // we've extended the syscall number and arguments to the register size.\n unsafe {\n #[cfg(target_arch = \"arm\")]\n asm_syscall!(sys::__NR_prctl, 172, r, arg1, arg2, arg3, arg4, arg5);\n // POWER9+ has fast syscall using SCV, but since detection is cold path,\n // there’s no need to consider it here.\n #[cfg(not(target_arch = \"arm\"))]\n asm_syscall!(number, r, arg1, arg2, arg3, arg4, arg5);\n }\n #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]\n if (r as c_int) < 0 { Err(r as c_int) } else { Ok(r as usize) }\n }\n\n let mut auxv = vec![unsafe { mem::zeroed::() }; 38];\n\n let old_len = auxv.len() * mem::size_of::();\n\n // SAFETY:\n // - `out_len` does not exceed the size of `auxv`.\n let _len = unsafe { prctl_get_auxv(auxv.as_mut_ptr().cast::(), old_len)? };\n\n for aux in &auxv {\n if aux.a_type == type_ {\n // SAFETY: aux.a_un is #[repr(C)] union and all fields have\n // the same size and can be safely transmuted to integers.\n return Ok(unsafe { aux.a_un.a_val });\n }\n }\n Err(0)\n }\n // Similar to the above, but call libc prctl instead of syscall using asm.\n //\n // This is used only for testing.\n fn getauxval_pr_get_auxv_libc(type_: c_ulong) -> Result {\n unsafe fn prctl_get_auxv(out: *mut c_void, len: usize) -> Result {\n // arg4 and arg5 must be zero.\n #[allow(clippy::cast_possible_wrap)]\n let r = unsafe { libc::prctl(sys::PR_GET_AUXV as c_int, out, len, 0, 0) };\n #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]\n if (r as c_int) < 0 { Err(r as c_int) } else { Ok(r as usize) }\n }\n\n let mut auxv = vec![unsafe { mem::zeroed::() }; 38];\n\n let old_len = auxv.len() * mem::size_of::();\n\n // SAFETY:\n // - `out_len` does not exceed the size of `auxv`.\n let _len = unsafe { prctl_get_auxv(auxv.as_mut_ptr().cast::(), old_len)? };\n\n for aux in &auxv {\n if aux.a_type == type_ {\n // SAFETY: aux.a_un is #[repr(C)] union and all fields have\n // the same size and can be safely transmuted to integers.\n return Ok(unsafe { aux.a_un.a_val });\n }\n }\n Err(0)\n }\n\n unsafe {\n let mut u = mem::zeroed();\n assert_eq!(libc::uname(&mut u), 0);\n let release = std::ffi::CStr::from_ptr(u.release.as_ptr());\n let release = str::from_utf8(release.to_bytes()).unwrap();\n let mut digits = release.split('.');\n let major = digits.next().unwrap().parse::().unwrap();\n let minor = digits.next().unwrap().parse::().unwrap();\n // TODO: qemu-user bug (fails even on kernel >= 6.4) (as of 9.2)\n if (major, minor) < (6, 4) || cfg!(qemu) {\n eprintln!(\"kernel version: {}.{} (no pr_get_auxv)\", major, minor);\n for &at in &[ffi::AT_HWCAP, ffi::AT_HWCAP2, ffi::AT_HWCAP3, ffi::AT_HWCAP4] {\n assert_eq!(getauxval_pr_get_auxv_libc(at).unwrap_err(), -1);\n #[cfg(not(target_os = \"android\"))]\n #[cfg(not(portable_atomic_no_asm_syscall))]\n assert_eq!(getauxval_pr_get_auxv_no_libc(at).unwrap_err(), -libc::EINVAL);\n }\n } else {\n eprintln!(\"kernel version: {}.{} (has pr_get_auxv)\", major, minor);\n for &at in &[ffi::AT_HWCAP, ffi::AT_HWCAP2] {\n if cfg!(all(valgrind, target_arch = \"powerpc64\")) {\n // TODO: valgrind bug (as of Valgrind 3.26)\n assert_eq!(getauxval_pr_get_auxv_libc(at).unwrap_err(), -1);\n #[cfg(not(target_os = \"android\"))]\n #[cfg(not(portable_atomic_no_asm_syscall))]\n assert_eq!(getauxval_pr_get_auxv_no_libc(at).unwrap_err(), -libc::EINVAL);\n } else if cfg!(all(valgrind, target_arch = \"aarch64\"))\n || cfg!(all(valgrind, target_arch = \"arm\")) && at == ffi::AT_HWCAP2\n {\n // TODO: valgrind bug (result value mismatch) (as of Valgrind 3.26)\n assert_ne!(os::getauxval(at), getauxval_pr_get_auxv_libc(at).unwrap());\n #[cfg(not(target_os = \"android\"))]\n #[cfg(not(portable_atomic_no_asm_syscall))]\n assert_ne!(os::getauxval(at), getauxval_pr_get_auxv_no_libc(at).unwrap());\n } else {\n assert_eq!(os::getauxval(at), getauxval_pr_get_auxv_libc(at).unwrap());\n #[cfg(not(target_os = \"android\"))]\n #[cfg(not(portable_atomic_no_asm_syscall))]\n assert_eq!(os::getauxval(at), getauxval_pr_get_auxv_no_libc(at).unwrap());\n }\n }\n for &at in &[ffi::AT_HWCAP3, ffi::AT_HWCAP4] {\n assert_eq!(\n os::getauxval(at),\n getauxval_pr_get_auxv_libc(at).unwrap_or_default()\n );\n #[cfg(not(target_os = \"android\"))]\n #[cfg(not(portable_atomic_no_asm_syscall))]\n assert_eq!(\n os::getauxval(at),\n getauxval_pr_get_auxv_no_libc(at).unwrap_or_default()\n );\n }\n }\n }\n }\n #[allow(clippy::cast_possible_wrap)]\n #[cfg(target_os = \"freebsd\")]\n #[test]\n fn test_alternative() {\n use std::ptr;\n\n use test_helper::sys;\n\n use crate::utils::ffi::*;\n\n // This is almost equivalent to what elf_aux_info does.\n // https://man.freebsd.org/elf_aux_info(3)\n // On FreeBSD, [AArch64 support is available on FreeBSD 11.0+](https://www.freebsd.org/releases/11.0R/announce),\n // but elf_aux_info is available on FreeBSD 12.0+ and 11.4+:\n // https://github.com/freebsd/freebsd-src/commit/0b08ae2120cdd08c20a2b806e2fcef4d0a36c470\n // https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/sys/auxv.h\n // so use sysctl instead of elf_aux_info.\n // Note that FreeBSD 11 (11.4) was EoL on 2021-09-30, and FreeBSD 11.3 was EoL on 2020-09-30:\n // https://www.freebsd.org/security/unsupported\n //\n // This is currently used only for testing.\n // If you want us to use this implementation for compatibility with the older FreeBSD\n // version that came to EoL a few years ago, please open an issue.\n fn getauxval_sysctl_libc(type_: ffi::c_int) -> Result {\n let mut auxv: [sys::Elf_Auxinfo; sys::AT_COUNT as usize] = unsafe { mem::zeroed() };\n\n let mut len = mem::size_of_val(&auxv) as c_size_t;\n\n // SAFETY: calling getpid is safe.\n let pid = unsafe { libc::getpid() };\n let mib = [\n sys::CTL_KERN as c_int,\n sys::KERN_PROC as c_int,\n sys::KERN_PROC_AUXV as c_int,\n pid,\n ];\n #[allow(clippy::cast_possible_truncation)]\n let mib_len = mib.len() as c_uint;\n // SAFETY:\n // - `mib_len` does not exceed the size of `mib`.\n // - `len` does not exceed the size of `auxv`.\n // - `sysctl` is thread-safe.\n let res = unsafe {\n libc::sysctl(\n mib.as_ptr(),\n mib_len,\n auxv.as_mut_ptr().cast::(),\n &mut len,\n ptr::null_mut(),\n 0,\n )\n };\n // FreeBSD sysctl returns 0 on success, -1 on failure.\n if res != 0 {\n return Err(res);\n }\n\n for aux in &auxv {\n #[allow(clippy::cast_sign_loss)]\n if aux.a_type == type_ as c_long {\n // SAFETY: aux.a_un is #[repr(C)] union and all fields have\n // the same size and can be safely transmuted to integers.\n return Ok(unsafe { aux.a_un.a_val as c_ulong });\n }\n }\n Err(0)\n }\n\n // AT_HWCAP2 is only available on FreeBSD 13+ on AArch64.\n let hwcap2_else = |e| if cfg!(target_arch = \"aarch64\") { 0 } else { panic!(\"{:?}\", e) };\n let at = ffi::AT_HWCAP;\n assert_eq!(os::getauxval(at), getauxval_sysctl_libc(at).unwrap());\n let at = ffi::AT_HWCAP2;\n assert_eq!(os::getauxval(at), getauxval_sysctl_libc(at).unwrap_or_else(hwcap2_else));\n for &at in &[ffi::AT_HWCAP3, ffi::AT_HWCAP4] {\n assert_eq!(os::getauxval(at), getauxval_sysctl_libc(at).unwrap_or_default());\n }\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/detect/common.rs", "sha256": "735925ba2717d649f49a7550d78fd523b002fb8c030cb7d52e542d982ada43a9", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#[derive(Clone, Copy)]\n#[repr(transparent)]\npub(crate) struct CpuInfo(u32);\n\nimpl CpuInfo {\n #[inline]\n fn set(&mut self, bit: CpuInfoFlag) {\n self.0 = set(self.0, bit as u32);\n }\n #[inline]\n #[must_use]\n fn test(self, bit: CpuInfoFlag) -> bool {\n test(self.0, bit as u32)\n }\n}\n\n#[inline]\n#[must_use]\nfn set(x: u32, bit: u32) -> u32 {\n x | (1 << bit)\n}\n#[inline]\n#[must_use]\nfn test(x: u32, bit: u32) -> bool {\n x & (1 << bit) != 0\n}\n\n#[inline]\npub(crate) fn detect() -> CpuInfo {\n use core::sync::atomic::{AtomicU32, Ordering};\n\n static CACHE: AtomicU32 = AtomicU32::new(0);\n let mut info = CpuInfo(CACHE.load(Ordering::Relaxed));\n if info.0 != 0 {\n return info;\n }\n info.set(CpuInfoFlag::Init);\n // Note: detect_false cfg is intended to make it easy for developers to test\n // cases where features usually available is not available, and is not a public API.\n if !cfg!(portable_atomic_test_detect_false) {\n info = _detect(info);\n }\n if cfg!(any(\n all(\n target_arch = \"x86_64\",\n not(any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\")),\n ),\n all(\n target_arch = \"powerpc64\",\n not(any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n )),\n ),\n all(\n any(target_arch = \"riscv32\", target_arch = \"riscv64\"),\n not(any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\")),\n ),\n )) {\n // Use CAS if the detection result affects lock-freeness.\n match CACHE.compare_exchange(0, info.0, Ordering::Relaxed, Ordering::Relaxed) {\n Ok(_) => info,\n Err(info) => {\n debug_assert_ne!(info, 0);\n CpuInfo(info)\n }\n }\n } else {\n CACHE.store(info.0, Ordering::Relaxed);\n info\n }\n}\n\nmacro_rules! flags {\n ($(\n $(#[$attr:meta])*\n $func:ident($name:literal, any($($cfg:ident),*)),\n )*) => {\n #[allow(dead_code, non_camel_case_types)]\n #[derive(Clone, Copy)]\n #[cfg_attr(test, derive(PartialEq, Eq, PartialOrd, Ord))]\n #[repr(u32)]\n enum CpuInfoFlag {\n Init = 0,\n $($func,)*\n }\n impl CpuInfo {\n $(\n $(#[$attr])*\n #[cfg(any(test, not(any($($cfg = $name),*))))]\n #[inline]\n #[must_use]\n pub(crate) fn $func(self) -> bool {\n self.test(CpuInfoFlag::$func)\n }\n )*\n #[cfg(test)] // for test\n const ALL_FLAGS: &'static [(&'static str, CpuInfoFlag, bool)] = &[$(\n ($name, CpuInfoFlag::$func, cfg!(any($($cfg = $name),*))),\n )*];\n }\n #[test]\n #[cfg_attr(portable_atomic_test_detect_false, ignore = \"detection disabled\")]\n fn test_detect() {$(\n $(#[$attr])*\n {\n const _: u32 = 1_u32 << CpuInfoFlag::$func as u32;\n assert_eq!($name.replace(|c: char| c == '-' || c == '.', \"_\"), stringify!($func));\n if detect().$func() {\n assert!(detect().test(CpuInfoFlag::$func));\n } else {\n assert!(!detect().test(CpuInfoFlag::$func));\n }\n }\n )*}\n };\n}\n\n// rustc definitions: https://github.com/rust-lang/rust/blob/ddaf12390d3ffb7d5ba74491a48f3cd528e5d777/compiler/rustc_target/src/target_features.rs\n\n// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/AArch64/AArch64Features.td\n#[cfg(any(target_arch = \"aarch64\", target_arch = \"arm64ec\"))]\nflags! {\n // The Armv8.1 architecture extension\n // https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-1-architecture-extension\n // FEAT_LSE, Large System Extensions\n // > This feature is supported in AArch64 state only.\n // > FEAT_LSE is OPTIONAL from Armv8.0.\n // > FEAT_LSE is mandatory from Armv8.1.\n lse(\"lse\", any(target_feature /* 1.61+ */, portable_atomic_target_feature)),\n\n // The Armv8.3 architecture extension\n // https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-3-architecture-extension\n // FEAT_LRCPC, Load-Acquire RCpc instructions\n // > These instructions are added to the A64 instruction set only.\n // > FEAT_LRCPC is OPTIONAL from Armv8.2.\n // > FEAT_LRCPC is mandatory from Armv8.3.\n #[cfg(test)] // test-only\n rcpc(\"rcpc\", any(target_feature /* 1.61+ */)),\n\n // The Armv8.4 architecture extension\n // https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-4-architecture-extension\n // FEAT_LSE2, Large System Extensions version 2\n // > This feature is supported in AArch64 state only.\n // > FEAT_LSE2 is OPTIONAL from Armv8.2.\n // > FEAT_LSE2 is mandatory from Armv8.4.\n #[cfg_attr(not(test), allow(dead_code))]\n lse2(\"lse2\", any(target_feature /* nightly */, portable_atomic_target_feature)),\n // FEAT_LRCPC2, Load-Acquire RCpc instructions version 2\n // > These instructions are added to the A64 instruction set only.\n // > FEAT_LRCPC2 is OPTIONAL from Armv8.2.\n // > FEAT_LRCPC2 is mandatory from Armv8.4.\n // > If FEAT_LRCPC2 is implemented, then FEAT_LRCPC is implemented.\n #[cfg(test)] // test-only\n rcpc2(\"rcpc2\", any(target_feature /* 1.61+ */)),\n\n // The Armv8.9 architecture extension\n // https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv8-9-architecture-extension\n // FEAT_LRCPC3, Load-Acquire RCpc instructions version 3\n // > This feature is supported in AArch64 state only.\n // > FEAT_LRCPC3 is OPTIONAL from Armv8.2.\n // > If FEAT_LRCPC3 is implemented, then FEAT_LRCPC2 is implemented.\n #[cfg_attr(not(test), allow(dead_code))]\n rcpc3(\"rcpc3\", any(target_feature /* nightly */, portable_atomic_target_feature)),\n\n // The Armv9.4 architecture extension\n // https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv9-4-architecture-extension\n // FEAT_LSE128, 128-bit Atomics\n // > This feature is supported in AArch64 state only.\n // > FEAT_LSE128 is OPTIONAL from Armv9.3.\n // > If FEAT_LSE128 is implemented, then FEAT_LSE is implemented.\n #[cfg_attr(not(test), allow(dead_code))]\n lse128(\"lse128\", any(target_feature /* nightly */, portable_atomic_target_feature)),\n\n // The Armv9.6 architecture extension\n // https://developer.arm.com/documentation/109697/2025_06/Feature-descriptions/The-Armv9-6-architecture-extension\n // FEAT_LS64WB, LS64 for Write-back cacheable memory\n // > This feature is supported in AArch64 state only.\n // > FEAT_LS64WB is OPTIONAL from Armv9.2.\n // > If FEAT_LS64WB is implemented, then FEAT_LS64_ACCDATA is implemented.\n #[cfg(test)] // test-only\n ls64wb(\"ls64wb\", any(/* no corresponding target feature */)),\n // FEAT_LSFE, Large System Float Extension\n // > This feature is supported in AArch64 state only.\n // > FEAT_LSFE is OPTIONAL from Armv9.3.\n // > If FEAT_LSFE is implemented, then FEAT_FP is implemented.\n #[cfg(test)] // test-only\n lsfe(\"lsfe\", any(target_feature /* N/A */, portable_atomic_target_feature)),\n\n #[cfg(test)] // test-only\n cpuid(\"cpuid\", any(/* no corresponding target feature */)),\n}\n\n// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/ARM/ARMFeatures.td\n#[cfg(target_arch = \"arm\")]\nflags! {\n #[cfg(test)] // test-only\n lpae(\"lpae\", any(/* no corresponding target feature */)),\n}\n\n// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/PowerPC/PPC.td\n#[cfg(target_arch = \"powerpc64\")]\nflags! {\n // lqarx and stqcx.\n quadword_atomics(\"quadword-atomics\", any(target_feature /* nightly */, portable_atomic_target_feature)),\n}\n\n// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/RISCV/RISCVFeatures.td\n#[cfg(any(target_arch = \"riscv32\", target_arch = \"riscv64\"))]\nflags! {\n // amocas.{w,d,q}\n zacas(\"zacas\", any(target_feature /* 1.94+ */, portable_atomic_target_feature)),\n #[cfg(test)] // test-only\n zabha(\"zabha\", any(target_feature /* 1.94+ */, portable_atomic_target_feature)),\n #[cfg(test)] // test-only\n zalasr(\"zalasr\", any(/* no corresponding target feature */)),\n}\n\n// LLVM definitions: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/Target/X86/X86.td\n#[cfg(target_arch = \"x86_64\")]\nflags! {\n // avx\n #[cfg(target_feature = \"sse\")]\n avx(\"avx\", any(target_feature)),\n // cmpxchg16b\n cmpxchg16b(\"cmpxchg16b\", any(target_feature /* 1.69+ */, portable_atomic_target_feature)),\n}\n\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks,\n clippy::wildcard_imports\n)]\n#[cfg(test)]\nmod tests_common {\n use std::{collections::BTreeSet, vec};\n\n use super::*;\n\n #[test]\n fn test_bit_flags() {\n let mut flags = vec![(\"init\", CpuInfoFlag::Init)];\n flags.extend(CpuInfo::ALL_FLAGS.iter().map(|&(name, flag, _)| (name, flag)));\n let flag_set = flags.iter().map(|(_, flag)| flag).collect::>();\n let name_set = flags.iter().map(|(name, _)| name).collect::>();\n if flag_set.len() != flags.len() {\n panic!(\"CpuInfo flag values must be unique\")\n }\n if name_set.len() != flags.len() {\n panic!(\"CpuInfo flag names must be unique\")\n }\n\n let mut x = CpuInfo(0);\n for &(_, f) in &flags {\n assert!(!x.test(f));\n }\n for i in 0..flags.len() {\n x.set(flags[i].1);\n for &(_, f) in &flags[..i + 1] {\n assert!(x.test(f));\n }\n for &(_, f) in &flags[i + 1..] {\n assert!(!x.test(f));\n }\n }\n for &(_, f) in &flags {\n assert!(x.test(f));\n }\n }\n\n #[test]\n fn print_features() {\n use std::{fmt::Write as _, string::String};\n\n let mut features = String::new();\n features.push_str(\"\\nfeatures:\\n\");\n for &(name, flag, compile_time) in CpuInfo::ALL_FLAGS {\n let run_time = detect().test(flag);\n if run_time == compile_time {\n let _ = writeln!(features, \" {}: {}\", name, run_time);\n } else {\n let _ = writeln!(\n features,\n \" {}: {} (compile-time), {} (run-time)\",\n name, compile_time, run_time\n );\n }\n }\n test_helper::eprintln_nocapture!(\"{}\", features);\n }\n\n // Static assertions for C type definitions.\n // Assertions with core::ffi types are in crate::utils::ffi module.\n #[cfg(not(any(windows, target_arch = \"x86\", target_arch = \"x86_64\")))]\n const _: fn() = || {\n use test_helper::sys;\n let _: crate::utils::ffi::c_char = 0 as sys::c_char;\n };\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/detect/powerpc64_aix.rs", "sha256": "b635538662e9add807cebd26c5742eae6dd764e11fd2c30fc6661baa03b69d01", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nRun-time CPU feature detection on PowerPC64 AIX by using getsystemcfg.\n\nRefs:\n- https://github.com/golang/go/blob/go1.26.0/src/internal/cpu/cpu_ppc64x_aix.go\n\nAs of Rust 1.94, is_powerpc_feature_detected doesn't support run-time detection on AIX.\nhttps://github.com/rust-lang/rust/blob/1.94.0/library/std_detect/src/detect/mod.rs\n\nRun-time detection on PowerPC64 AIX is currently disabled by default as experimental\nbecause we cannot run tests on the VM or real machine.\n*/\n\ninclude!(\"common.rs\");\n\n// libc requires Rust 1.63\nmod ffi {\n pub(crate) use crate::utils::ffi::{c_int, c_ulong};\n\n sys_const!({\n // https://github.com/rust-lang/libc/blob/0.2.158/src/unix/aix/mod.rs#L2058\n // https://github.com/golang/go/blob/go1.26.0/src/internal/cpu/cpu_ppc64x_aix.go\n pub(crate) const SC_IMPL: c_int = 2;\n pub(crate) const POWER_8: c_ulong = 0x10000;\n pub(crate) const POWER_9: c_ulong = 0x20000;\n });\n // TODO: use sys_const! once libc crate defined it.\n pub(crate) const POWER_10: c_ulong = 0x40000;\n\n sys_fn!({\n extern \"C\" {\n // https://www.ibm.com/docs/en/aix/7.3?topic=g-getsystemcfg-subroutine\n // https://github.com/rust-lang/libc/blob/0.2.158/src/unix/aix/powerpc64.rs#L643\n pub(crate) fn getsystemcfg(name: c_int) -> c_ulong;\n }\n });\n}\n\n#[cold]\n#[must_use]\nfn _detect(mut info: CpuInfo) -> CpuInfo {\n // SAFETY: calling getsystemcfg is safe.\n let impl_ = unsafe { ffi::getsystemcfg(ffi::SC_IMPL) };\n if impl_ == ffi::c_ulong::MAX {\n return info;\n }\n // Check both POWER_8 and later ISAs (which are superset of POWER_8) because\n // AIX currently doesn't set POWER_8 when POWER_9 is set.\n // https://github.com/golang/go/commit/51859ec2292d9c1d82a7054ec672ff551a0d7497\n if impl_ & (ffi::POWER_8 | ffi::POWER_9 | ffi::POWER_10) != 0 {\n info.set(CpuInfoFlag::quadword_atomics);\n }\n info\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/detect/riscv_linux.rs", "sha256": "9e8b214013992dcf7ff5e0beb9228db4f2125aa134afdc573781788349f3a159", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nRun-time CPU feature detection on RISC-V Linux/Android by using riscv_hwprobe.\n\nOn RISC-V, detection using auxv only supports single-letter extensions.\nSo, we use riscv_hwprobe that supports multi-letter extensions.\n\nRefs: https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/riscv/hwprobe.rst\n*/\n\ninclude!(\"common.rs\");\n\nuse core::ptr;\n\n// libc requires Rust 1.63\n#[allow(non_camel_case_types, non_upper_case_globals)]\nmod ffi {\n pub(crate) use crate::utils::ffi::{c_long, c_size_t, c_uint, c_ulong};\n\n sys_struct!({\n // https://github.com/torvalds/linux/blob/v6.19/arch/riscv/include/uapi/asm/hwprobe.h\n pub(crate) struct riscv_hwprobe {\n pub(crate) key: i64,\n pub(crate) value: u64,\n }\n });\n\n sys_const!({\n pub(crate) const __NR_riscv_hwprobe: c_long = 258;\n\n // https://github.com/torvalds/linux/blob/v6.19/arch/riscv/include/uapi/asm/hwprobe.h\n // Linux 6.4+\n // https://github.com/torvalds/linux/commit/00e76e2c6a2bd3976d44d4a1fdd0b7a3c2566607\n pub(crate) const RISCV_HWPROBE_KEY_BASE_BEHAVIOR: i64 = 3;\n pub(crate) const RISCV_HWPROBE_BASE_BEHAVIOR_IMA: u64 = 1 << 0;\n pub(crate) const RISCV_HWPROBE_KEY_IMA_EXT_0: i64 = 4;\n // Linux 6.8+\n // https://github.com/torvalds/linux/commit/154a3706122978eeb34d8223d49285ed4f3c61fa\n pub(crate) const RISCV_HWPROBE_EXT_ZACAS: u64 = 1 << 34;\n // Linux 6.16+\n // https://github.com/torvalds/linux/commit/415a8c81da3dab0a585bd4f8d505a11ad5a171a7\n #[cfg(test)]\n pub(crate) const RISCV_HWPROBE_EXT_ZABHA: u64 = 1 << 58;\n // Linux 6.19+\n // https://github.com/torvalds/linux/commit/f4922b69165735e81752ee47d174f873e989a449\n #[cfg(test)]\n pub(crate) const RISCV_HWPROBE_EXT_ZALASR: u64 = 1 << 59;\n });\n\n cfg_sel!({\n // Use asm-based syscall on Linux for compatibility with non-libc targets if possible.\n // Do not use this on Android. See comments on syscall_helper module in src/utils.rs for details.\n #[cfg(all(\n target_os = \"linux\",\n any(\n target_arch = \"riscv32\",\n all(target_arch = \"riscv64\", target_pointer_width = \"64\"),\n ),\n not(portable_atomic_no_asm_syscall),\n ))]\n {\n use crate::utils::{RegISize, RegSize};\n\n #[inline]\n pub(crate) unsafe fn syscall5(\n number: c_long,\n arg1: *mut riscv_hwprobe,\n arg2: c_size_t,\n arg3: c_size_t,\n arg4: *mut c_ulong,\n arg5: c_uint,\n ) -> c_long {\n #[allow(clippy::cast_possible_truncation)]\n let number = number as RegISize;\n let arg1 = ptr_reg!(arg1);\n let arg2 = arg2 as RegSize;\n let arg3 = arg3 as RegSize;\n let arg4 = ptr_reg!(arg4);\n let arg5 = arg5 as RegSize;\n let r: RegISize;\n // SAFETY: the caller must uphold the safety contract.\n // we've extended the syscall number and arguments to the register size.\n unsafe {\n asm_syscall!(number, r, arg1, arg2, arg3, arg4, arg5);\n }\n #[allow(clippy::cast_possible_truncation)]\n {\n r as c_long\n }\n }\n }\n #[cfg(else)]\n {\n sys_fn!({\n extern \"C\" {\n // https://man7.org/linux/man-pages/man2/syscall.2.html\n pub(crate) fn syscall(number: c_long, ...) -> c_long;\n }\n });\n pub(crate) use self::syscall as syscall5;\n }\n });\n\n // https://github.com/torvalds/linux/blob/v6.19/Documentation/arch/riscv/hwprobe.rst\n pub(crate) unsafe fn __riscv_hwprobe(\n pairs: *mut riscv_hwprobe,\n pair_count: c_size_t,\n cpu_set_size: c_size_t,\n cpus: *mut c_ulong,\n flags: c_uint,\n ) -> c_long {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { syscall5(__NR_riscv_hwprobe, pairs, pair_count, cpu_set_size, cpus, flags) }\n }\n}\n\n// syscall returns an unsupported error if riscv_hwprobe is not supported,\n// so we can safely use this function on older versions of Linux.\nfn riscv_hwprobe(out: &mut [ffi::riscv_hwprobe]) -> bool {\n let len = out.len();\n // SAFETY: We've passed the valid pointer and length,\n // passing null ptr for cpus is safe because cpu_set_size is zero.\n unsafe { ffi::__riscv_hwprobe(out.as_mut_ptr(), len, 0, ptr::null_mut(), 0) == 0 }\n}\n\n#[cold]\n#[must_use]\nfn _detect(mut info: CpuInfo) -> CpuInfo {\n let mut out = [\n ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_BASE_BEHAVIOR, value: 0 },\n ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_IMA_EXT_0, value: 0 },\n ];\n if riscv_hwprobe(&mut out)\n && out[0].key != -1\n && out[0].value & ffi::RISCV_HWPROBE_BASE_BEHAVIOR_IMA != 0\n && out[1].key != -1\n {\n let value = out[1].value;\n macro_rules! check {\n ($flag:ident, $bit:ident) => {\n if value & ffi::$bit != 0 {\n info.set(CpuInfoFlag::$flag);\n }\n };\n }\n check!(zacas, RISCV_HWPROBE_EXT_ZACAS);\n #[cfg(test)]\n check!(zabha, RISCV_HWPROBE_EXT_ZABHA);\n #[cfg(test)]\n check!(zalasr, RISCV_HWPROBE_EXT_ZALASR);\n }\n info\n}\n\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks,\n clippy::wildcard_imports\n)]\n#[cfg(test)]\nmod tests {\n use super::*;\n\n // We use asm-based syscall for compatibility with non-libc targets.\n // This test tests that our ones and libc::syscall returns the same result.\n #[test]\n fn test_alternative() {\n unsafe fn __riscv_hwprobe_libc(\n pairs: *mut ffi::riscv_hwprobe,\n pair_count: ffi::c_size_t,\n cpu_set_size: ffi::c_size_t,\n cpus: *mut ffi::c_ulong,\n flags: ffi::c_uint,\n ) -> ffi::c_long {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n libc::syscall(ffi::__NR_riscv_hwprobe, pairs, pair_count, cpu_set_size, cpus, flags)\n }\n }\n fn riscv_hwprobe_libc(out: &mut [ffi::riscv_hwprobe]) -> bool {\n let len = out.len();\n unsafe { __riscv_hwprobe_libc(out.as_mut_ptr(), len, 0, ptr::null_mut(), 0) == 0 }\n }\n let mut out = [\n ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_BASE_BEHAVIOR, value: 0 },\n ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_IMA_EXT_0, value: 0 },\n ];\n let mut libc_out = out;\n assert_eq!(riscv_hwprobe(&mut out), riscv_hwprobe_libc(&mut libc_out));\n assert_eq!(out, libc_out);\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/detect/x86_64.rs", "sha256": "8b315b61a409d3350b19c02e98a741f4c1c1e3861e6792594137267f4595eeb1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nRun-time CPU feature detection on x86_64 by using CPUID.\n\nAdapted from https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs.\n*/\n\n#![cfg_attr(portable_atomic_sanitize_thread, allow(dead_code))]\n\n// Miri doesn't support inline assembly used in __cpuid: https://github.com/rust-lang/miri/issues/932\n// SGX doesn't support CPUID: https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs#L30-L33\n#[cfg(any(target_env = \"sgx\", miri))]\ncompile_error!(\"internal error: this module is not supported on this environment\");\n\ninclude!(\"common.rs\");\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\nuse core::arch::x86_64::CpuidResult;\n\n// Workaround for https://github.com/rust-lang/rust/issues/101346\n// It is not clear if our use cases are affected, but we implement this just in case.\n//\n// Refs:\n// - https://www.felixcloutier.com/x86/cpuid\n// - https://en.wikipedia.org/wiki/CPUID\n// - https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs\n#[cfg(not(target_env = \"sgx\"))]\nfn __cpuid(leaf: u32) -> CpuidResult {\n let eax;\n let mut ebx;\n let ecx;\n let edx;\n // SAFETY: Calling `__cpuid` is safe on all x86_64 CPUs except for SGX,\n // which doesn't support `cpuid`.\n // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L109\n unsafe {\n asm!(\n \"mov r8, rbx\", // save rbx which is reserved by LLVM\n \"cpuid\",\n \"xchg r8, rbx\", // restore rbx\n out(\"r8\") ebx,\n inout(\"eax\") leaf => eax,\n inout(\"ecx\") 0 => ecx,\n out(\"edx\") edx,\n options(nostack, preserves_flags),\n );\n }\n CpuidResult { eax, ebx, ecx, edx }\n}\n\n#[cold]\n#[must_use]\nfn _detect(mut info: CpuInfo) -> CpuInfo {\n let CpuidResult { ecx: proc_info_ecx, .. } = __cpuid(1);\n\n // https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs#L104\n if test(proc_info_ecx, 13) {\n info.set(CpuInfoFlag::cmpxchg16b);\n }\n\n // We only use VMOVDQA when SSE is enabled. See _atomic_load_vmovdqa() in atomic128/x86_64.rs for more.\n #[cfg(target_feature = \"sse\")]\n {\n use core::arch::x86_64::_xgetbv;\n\n // https://github.com/rust-lang/rust/blob/1.92.0/library/std_detect/src/detect/os/x86.rs#L166-L236\n let cpu_xsave = test(proc_info_ecx, 26);\n if cpu_xsave {\n let cpu_osxsave = test(proc_info_ecx, 27);\n if cpu_osxsave {\n // SAFETY: Calling `_xgetbv` is safe because the CPU has `xsave` support\n // and OS has set `osxsave`.\n let xcr0 = unsafe { _xgetbv(0) };\n let os_avx_support = xcr0 & 6 == 6;\n if os_avx_support && test(proc_info_ecx, 28) {\n info.set(CpuInfoFlag::avx);\n }\n }\n }\n }\n info\n}\n\n#[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks,\n clippy::wildcard_imports\n)]\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n #[cfg_attr(portable_atomic_test_detect_false, ignore = \"detection disabled\")]\n fn test_cpuid() {\n // The recent Rosetta 2 unofficially implements AVX support.\n // (The OS reports it as unsupported, likely due to poor performance.)\n #[cfg(target_vendor = \"apple\")]\n assert_eq!(\n std::is_x86_feature_detected!(\"avx\"),\n detect().avx() || cfg!(target_feature = \"avx\")\n );\n #[cfg(not(target_vendor = \"apple\"))]\n assert_eq!(std::is_x86_feature_detected!(\"avx\"), detect().avx());\n assert_eq!(std::is_x86_feature_detected!(\"cmpxchg16b\"), detect().cmpxchg16b());\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/fallback/mod.rs", "sha256": "970934078d4cf02e424f2bc80ad7724e188bc8fe4c883898311ef32b4601d826", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nFallback implementation using global locks.\n\nThis implementation uses seqlock for global locks.\n\nThis is basically based on global locks in crossbeam-utils's `AtomicCell`,\nbut seqlock is implemented in a way that does not depend on UB\n(see comments in optimistic_read method in atomic! macro for details).\n\nNote that we cannot use a lock per atomic type, since the in-memory representation of the atomic\ntype and the value type must be the same.\n*/\n\n#![cfg_attr(\n any(\n all(\n target_arch = \"riscv32\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n not(portable_atomic_no_outline_atomics),\n any(target_os = \"linux\", target_os = \"android\"),\n ),\n all(\n target_arch = \"arm\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_os = \"linux\", target_os = \"android\"),\n any(test, not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\"))),\n not(portable_atomic_no_outline_atomics),\n ),\n ),\n allow(dead_code)\n)]\n\n// This module requires CAS and this crate only provides atomics up to 128-bit.\n// I don't believe there are any 16-bit multi-core systems with CAS, and\n// at least no such architecture is currently supported in Rust.\n// 128-bit targets that lack atomic usize CAS also do not reach this module.\n#[cfg(target_pointer_width = \"16\")]\ncompile_error!(\n \"internal error: unreachable since atomics for 16-bit targets can always be provided by disable interrupts\"\n);\n#[cfg(target_pointer_width = \"128\")]\ncompile_error!(\n \"internal error: unreachable since 128-bit target either has atomic CAS for the pointer width or does not have CAS\"\n);\n\nmod utils;\n\n// Use \"wide\" sequence lock if the pointer width <= 32 for preventing its counter against wrap\n// around.\n//\n// Some 64-bit architectures have ABI with 32-bit pointer width (e.g., x86_64 X32 ABI,\n// AArch64 ILP32 ABI, mips64 N32 ABI). On those targets, AtomicU64 is available and fast,\n// so use it to implement normal sequence lock and reduce chunks of byte-wise atomic memcpy.\ncfg_has_fast_atomic_64!({\n mod seq_lock;\n type AtomicChunk = core::sync::atomic::AtomicU64;\n type Chunk = u64;\n});\ncfg_no_fast_atomic_64!({\n #[path = \"seq_lock_wide.rs\"]\n mod seq_lock;\n type AtomicChunk = core::sync::atomic::AtomicU32;\n type Chunk = u32;\n});\n\nuse core::{cell::UnsafeCell, mem, sync::atomic::Ordering};\n\nuse self::{\n seq_lock::{SeqLock, SeqLockWriteGuard},\n utils::CachePadded,\n};\n#[cfg(portable_atomic_no_strict_provenance)]\nuse crate::utils::ptr::PtrExt as _;\nuse crate::utils::unlikely;\n\n// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/atomic_cell.rs#L970-L1010.\n#[inline]\n#[must_use]\nfn lock(addr: usize) -> &'static SeqLock {\n // The number of locks is a prime number because we want to make sure `addr % LEN` gets\n // dispersed across all locks.\n const LEN: usize = 67;\n const L: CachePadded = CachePadded::new(SeqLock::new());\n static LOCKS: [CachePadded; LEN] = [\n L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L,\n L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L, L,\n L, L, L, L, L, L, L,\n ];\n\n // If the modulus is a constant number, the compiler will use crazy math to transform this into\n // a sequence of cheap arithmetic operations rather than using the slow modulo instruction.\n &LOCKS[addr % LEN]\n}\n\n// Emit SeqCst fence to ensure SeqCst semantics when ordering is SeqCst.\n#[must_use]\nstruct ScFenceGuard;\nimpl ScFenceGuard {\n #[inline]\n fn new(emit_sc_fence: bool) -> Option {\n if unlikely(emit_sc_fence) {\n crate::fence(Ordering::SeqCst);\n Some(ScFenceGuard)\n } else {\n None\n }\n }\n}\nimpl Drop for ScFenceGuard {\n #[inline]\n fn drop(&mut self) {\n crate::fence(Ordering::SeqCst);\n }\n}\n\nmacro_rules! atomic {\n ($atomic_type:ident, $int_type:ident, $align:literal) => {\n #[repr(C, align($align))]\n pub(crate) struct $atomic_type {\n v: UnsafeCell<$int_type>,\n }\n\n impl $atomic_type {\n const LEN: usize = mem::size_of::<$int_type>() / mem::size_of::();\n\n #[inline]\n unsafe fn chunks(&self) -> &[AtomicChunk; Self::LEN] {\n static_assert!($atomic_type::LEN > 1);\n static_assert!(mem::size_of::<$int_type>() % mem::size_of::() == 0);\n\n // SAFETY: the caller must uphold the safety contract for `chunks`.\n unsafe { &*(self.v.get() as *const $int_type as *const [AtomicChunk; Self::LEN]) }\n }\n\n #[inline]\n fn optimistic_read(&self) -> $int_type {\n // Using `MaybeUninit<[usize; Self::LEN]>` here doesn't change codegen: https://godbolt.org/z/86f8s733M\n let mut dst: [Chunk; Self::LEN] = [0; Self::LEN];\n // SAFETY:\n // - There are no threads that perform non-atomic concurrent write operations.\n // - There is no writer that updates the value using atomic operations of different granularity.\n //\n // If the atomic operation is not used here, it will cause a data race\n // when `write` performs concurrent write operation.\n // Such a data race is sometimes considered virtually unproblematic\n // in SeqLock implementations:\n //\n // - https://github.com/Amanieu/seqlock/issues/2\n // - https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/atomic_cell.rs#L1063-L1069\n // - https://rust-lang.zulipchat.com/#narrow/stream/136281-t-lang.2Fwg-unsafe-code-guidelines/topic/avoiding.20UB.20due.20to.20races.20by.20discarding.20result.3F\n //\n // However, in our use case, the implementation that loads/stores value as\n // chunks of usize is enough fast and sound, so we use that implementation.\n //\n // See also atomic-memcpy crate, a generic implementation of this pattern:\n // https://github.com/taiki-e/atomic-memcpy\n let chunks = unsafe { self.chunks() };\n for i in 0..Self::LEN {\n dst[i] = chunks[i].load(Ordering::Relaxed);\n }\n // SAFETY: integers are plain old data types so we can always transmute to them.\n unsafe { mem::transmute::<[Chunk; Self::LEN], $int_type>(dst) }\n }\n\n #[inline]\n fn read(&self, _guard: &SeqLockWriteGuard<'static>) -> $int_type {\n // This calls optimistic_read that can return teared value, but the resulting value\n // is guaranteed not to be teared because we hold the lock to write.\n self.optimistic_read()\n }\n\n #[inline]\n fn write(&self, val: $int_type, _guard: &SeqLockWriteGuard<'static>) {\n // SAFETY: integers are plain old data types so we can always transmute them to arrays of integers.\n let val = unsafe { mem::transmute::<$int_type, [Chunk; Self::LEN]>(val) };\n // SAFETY:\n // - The guard guarantees that we hold the lock to write.\n // - There are no threads that perform non-atomic concurrent read or write operations.\n //\n // See optimistic_read for the reason that atomic operations are used here.\n let chunks = unsafe { self.chunks() };\n for i in 0..Self::LEN {\n chunks[i].store(val[i], Ordering::Relaxed);\n }\n }\n }\n\n // Send is implicitly implemented.\n // SAFETY: any data races are prevented by the lock and atomic operation.\n unsafe impl Sync for $atomic_type {}\n\n #[cfg(any(\n test,\n not(any(\n all(\n target_arch = \"x86_64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n portable_atomic_no_cmpxchg16b_intrinsic,\n )),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n not(portable_atomic_no_outline_atomics),\n not(any(target_env = \"sgx\", miri)),\n ),\n all(\n target_arch = \"powerpc64\",\n not(portable_atomic_no_asm),\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n all(\n target_env = \"gnu\",\n any(target_endian = \"little\", not(target_feature = \"crt-static\")),\n ),\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n all(\n target_os = \"freebsd\",\n any(\n target_endian = \"little\",\n not(target_feature = \"crt-static\"),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"openbsd\",\n all(\n target_os = \"aix\",\n not(portable_atomic_pre_llvm_20),\n portable_atomic_outline_atomics, // TODO(aix): currently disabled by default\n ),\n ),\n not(any(miri, portable_atomic_sanitize_thread)),\n ),\n all(\n target_arch = \"riscv64\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n not(portable_atomic_no_outline_atomics),\n any(target_os = \"linux\", target_os = \"android\"),\n ),\n )),\n ))]\n items!({\n impl_default_no_fetch_ops!($atomic_type, $int_type);\n impl_default_bit_opts!($atomic_type, $int_type);\n impl $atomic_type {\n #[inline]\n pub(crate) const fn new(v: $int_type) -> Self {\n Self { v: UnsafeCell::new(v) }\n }\n\n #[inline]\n pub(crate) fn is_lock_free() -> bool {\n Self::IS_ALWAYS_LOCK_FREE\n }\n pub(crate) const IS_ALWAYS_LOCK_FREE: bool = false;\n\n #[inline]\n #[cfg_attr(\n all(debug_assertions, not(portable_atomic_no_track_caller)),\n track_caller\n )]\n pub(crate) fn compare_exchange_weak(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n self.compare_exchange(current, new, success, failure)\n }\n\n #[inline]\n pub(crate) fn not(&self, order: Ordering) {\n self.fetch_not(order);\n }\n #[inline]\n pub(crate) fn neg(&self, order: Ordering) {\n self.fetch_neg(order);\n }\n\n #[inline]\n pub(crate) const fn as_ptr(&self) -> *mut $int_type {\n self.v.get()\n }\n }\n });\n #[cfg_attr(\n any(\n all(\n target_arch = \"x86_64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n portable_atomic_no_cmpxchg16b_intrinsic,\n )),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n not(portable_atomic_no_outline_atomics),\n not(any(target_env = \"sgx\", miri)),\n ),\n all(\n target_arch = \"powerpc64\",\n not(portable_atomic_no_asm),\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n all(\n target_env = \"gnu\",\n any(target_endian = \"little\", not(target_feature = \"crt-static\")),\n ),\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n all(\n target_os = \"freebsd\",\n any(\n target_endian = \"little\",\n not(target_feature = \"crt-static\"),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"openbsd\",\n all(\n target_os = \"aix\",\n not(portable_atomic_pre_llvm_20),\n portable_atomic_outline_atomics, // TODO(aix): currently disabled by default\n ),\n ),\n not(any(miri, portable_atomic_sanitize_thread)),\n ),\n all(\n target_arch = \"riscv64\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n not(portable_atomic_no_outline_atomics),\n any(target_os = \"linux\", target_os = \"android\"),\n ),\n ),\n allow(dead_code)\n )]\n impl $atomic_type {\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n pub(crate) fn load(&self, order: Ordering) -> $int_type {\n crate::utils::assert_load_ordering(order);\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let lock = lock(self.v.get().addr());\n\n // Try doing an optimistic read first.\n if let Some(stamp) = lock.optimistic_read() {\n let val = self.optimistic_read();\n\n if lock.validate_read(stamp) {\n return val;\n }\n }\n\n // Grab a regular write lock so that writers don't starve this load.\n let guard = lock.write();\n let val = self.read(&guard);\n // The value hasn't been changed. Drop the guard without incrementing the stamp.\n guard.abort();\n val\n }\n\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n pub(crate) fn store(&self, val: $int_type, order: Ordering) {\n crate::utils::assert_store_ordering(order);\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n self.write(val, &guard)\n }\n\n #[inline]\n pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type {\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n self.write(val, &guard);\n prev\n }\n\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n pub(crate) fn compare_exchange(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure);\n let _sc_fence = ScFenceGuard::new(\n success == Ordering::SeqCst || failure == Ordering::SeqCst\n );\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n if prev == current {\n self.write(new, &guard);\n Ok(prev)\n } else {\n // The value hasn't been changed. Drop the guard without incrementing the stamp.\n guard.abort();\n Err(prev)\n }\n }\n\n #[inline]\n pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type {\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n self.write(prev.wrapping_add(val), &guard);\n prev\n }\n\n #[inline]\n pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type {\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n self.write(prev.wrapping_sub(val), &guard);\n prev\n }\n\n #[inline]\n pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n self.write(prev & val, &guard);\n prev\n }\n\n #[inline]\n pub(crate) fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type {\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n self.write(!(prev & val), &guard);\n prev\n }\n\n #[inline]\n pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n self.write(prev | val, &guard);\n prev\n }\n\n #[inline]\n pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n self.write(prev ^ val, &guard);\n prev\n }\n\n #[inline]\n pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n self.write(!prev, &guard);\n prev\n }\n\n #[inline]\n pub(crate) fn fetch_neg(&self, order: Ordering) -> $int_type {\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n self.write(prev.wrapping_neg(), &guard);\n prev\n }\n }\n impl $atomic_type {\n #[inline]\n pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n self.write(core::cmp::max(prev, val), &guard);\n prev\n }\n\n #[inline]\n pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {\n let _sc_fence = ScFenceGuard::new(order == Ordering::SeqCst);\n let guard = lock(self.v.get().addr()).write();\n let prev = self.read(&guard);\n self.write(core::cmp::min(prev, val), &guard);\n prev\n }\n }\n };\n}\n\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(\n test,\n not(any(\n not(portable_atomic_no_atomic_64),\n all(\n target_arch = \"riscv32\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"),\n ),\n ))\n ))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n test,\n not(any(\n target_has_atomic = \"64\",\n all(\n target_arch = \"riscv32\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"),\n ),\n ))\n ))\n)]\ncfg_no_fast_atomic_64!({\n atomic!(AtomicI64, i64, 8);\n atomic!(AtomicU64, u64, 8);\n});\n\natomic!(AtomicI128, i128, 16);\natomic!(AtomicU128, u128, 16);\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n cfg_no_fast_atomic_64!({\n test_atomic_int!(i64);\n test_atomic_int!(u64);\n });\n test_atomic_int!(i128);\n test_atomic_int!(u128);\n\n // load/store/swap implementation is not affected by signedness, so it is\n // enough to test only unsigned types.\n cfg_no_fast_atomic_64!({\n stress_test!(u64);\n });\n stress_test!(u128);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/fallback/outline_atomics.rs", "sha256": "041d872affc4ad0b6b452f695a1373a1f76fe75d1b2377c24c3c0d199d89e383", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nHelper for outline-atomics.\n\nOn architectures where DW atomics are not supported on older CPUs, we use\nfallback implementation when DW atomic instructions are not supported and\noutline-atomics is enabled.\n\nThis module provides helpers to implement them.\n*/\n\nuse core::sync::atomic::Ordering;\n\n#[cfg(any(target_arch = \"x86_64\", target_arch = \"powerpc64\", target_arch = \"riscv64\"))]\npub(crate) type Udw = u128;\n#[cfg(any(target_arch = \"x86_64\", target_arch = \"powerpc64\", target_arch = \"riscv64\"))]\npub(crate) type AtomicUdw = super::super::super::fallback::AtomicU128;\n#[cfg(any(target_arch = \"x86_64\", target_arch = \"powerpc64\", target_arch = \"riscv64\"))]\npub(crate) type AtomicIdw = super::super::super::fallback::AtomicI128;\n\n#[cfg(any(target_arch = \"arm\", target_arch = \"riscv32\"))]\npub(crate) type Udw = u64;\n#[cfg(any(target_arch = \"arm\", target_arch = \"riscv32\"))]\npub(crate) type AtomicUdw = super::super::super::fallback::AtomicU64;\n#[cfg(any(target_arch = \"arm\", target_arch = \"riscv32\"))]\npub(crate) type AtomicIdw = super::super::super::fallback::AtomicI64;\n\n// Asserts that the function is called in the correct context.\nmacro_rules! debug_assert_outline_atomics {\n () => {\n #[cfg(target_arch = \"x86_64\")]\n {\n debug_assert!(!super::detect::detect().cmpxchg16b());\n }\n #[cfg(target_arch = \"powerpc64\")]\n {\n debug_assert!(!super::detect::detect().quadword_atomics());\n }\n #[cfg(any(target_arch = \"riscv32\", target_arch = \"riscv64\"))]\n {\n debug_assert!(!super::detect::detect().zacas());\n }\n #[cfg(target_arch = \"arm\")]\n {\n debug_assert!(!super::has_kuser_cmpxchg64());\n }\n };\n}\n\n#[cold]\npub(crate) unsafe fn atomic_load(src: *mut Udw, order: Ordering) -> Udw {\n debug_assert_outline_atomics!();\n #[allow(clippy::cast_ptr_alignment)]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n (*(src as *const AtomicUdw)).load(order)\n }\n}\nfn_alias! {\n #[cold]\n pub(crate) unsafe fn(src: *mut Udw) -> Udw;\n // fallback's atomic load has at least acquire semantics.\n #[cfg(not(any(target_arch = \"arm\", target_arch = \"x86_64\")))]\n atomic_load_non_seqcst = atomic_load(Ordering::Acquire);\n atomic_load_seqcst = atomic_load(Ordering::SeqCst);\n}\n\n#[cfg(not(any(target_arch = \"arm\", target_arch = \"riscv32\", target_arch = \"riscv64\")))]\n#[cold]\npub(crate) unsafe fn atomic_store(dst: *mut Udw, val: Udw, order: Ordering) {\n debug_assert_outline_atomics!();\n #[allow(clippy::cast_ptr_alignment)]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n (*(dst as *const AtomicUdw)).store(val, order);\n }\n}\n#[cfg(not(any(target_arch = \"arm\", target_arch = \"riscv32\", target_arch = \"riscv64\")))]\nfn_alias! {\n #[cold]\n pub(crate) unsafe fn(dst: *mut Udw, val: Udw);\n // fallback's atomic store has at least release semantics.\n atomic_store_non_seqcst = atomic_store(Ordering::Release);\n atomic_store_seqcst = atomic_store(Ordering::SeqCst);\n}\n\n#[cold]\npub(crate) unsafe fn atomic_compare_exchange(\n dst: *mut Udw,\n old: Udw,\n new: Udw,\n success: Ordering,\n failure: Ordering,\n) -> (Udw, bool) {\n debug_assert_outline_atomics!();\n #[allow(clippy::cast_ptr_alignment)]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n match (*(dst as *const AtomicUdw)).compare_exchange(old, new, success, failure) {\n Ok(v) => (v, true),\n Err(v) => (v, false),\n }\n }\n}\nfn_alias! {\n #[cold]\n pub(crate) unsafe fn(dst: *mut Udw, old: Udw, new: Udw) -> (Udw, bool);\n // fallback's atomic CAS has at least AcqRel semantics.\n #[cfg(not(any(target_arch = \"arm\", target_arch = \"x86_64\")))]\n atomic_compare_exchange_non_seqcst\n = atomic_compare_exchange(Ordering::AcqRel, Ordering::Acquire);\n atomic_compare_exchange_seqcst\n = atomic_compare_exchange(Ordering::SeqCst, Ordering::SeqCst);\n}\n\nmacro_rules! atomic_rmw_3 {\n (\n $name:ident($atomic_type:ident::$method_name:ident),\n $non_seqcst_alias:ident, $seqcst_alias:ident\n ) => {\n #[cold]\n pub(crate) unsafe fn $name(dst: *mut Udw, val: Udw, order: Ordering) -> Udw {\n debug_assert_outline_atomics!();\n #[allow(\n clippy::as_underscore,\n clippy::cast_possible_wrap,\n clippy::cast_ptr_alignment,\n clippy::cast_sign_loss\n )]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n (*(dst as *const $atomic_type)).$method_name(val as _, order) as Udw\n }\n }\n fn_alias! {\n #[cold]\n pub(crate) unsafe fn(dst: *mut Udw, val: Udw) -> Udw;\n // fallback's atomic RMW has at least AcqRel semantics.\n #[cfg(not(any(target_arch = \"arm\", target_arch = \"x86_64\")))]\n $non_seqcst_alias = $name(Ordering::AcqRel);\n $seqcst_alias = $name(Ordering::SeqCst);\n }\n };\n}\nmacro_rules! atomic_rmw_2 {\n (\n $name:ident($atomic_type:ident::$method_name:ident),\n $non_seqcst_alias:ident, $seqcst_alias:ident\n ) => {\n #[cold]\n pub(crate) unsafe fn $name(dst: *mut Udw, order: Ordering) -> Udw {\n debug_assert_outline_atomics!();\n #[allow(clippy::cast_ptr_alignment)]\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n (*(dst as *const $atomic_type)).$method_name(order) as Udw\n }\n }\n fn_alias! {\n #[cold]\n pub(crate) unsafe fn(dst: *mut Udw) -> Udw;\n // fallback's atomic RMW has at least AcqRel semantics.\n #[cfg(not(any(target_arch = \"arm\", target_arch = \"x86_64\")))]\n $non_seqcst_alias = $name(Ordering::AcqRel);\n $seqcst_alias = $name(Ordering::SeqCst);\n }\n };\n}\n\natomic_rmw_3!(atomic_swap(AtomicUdw::swap), atomic_swap_non_seqcst, atomic_swap_seqcst);\natomic_rmw_3!(atomic_add(AtomicUdw::fetch_add), atomic_add_non_seqcst, atomic_add_seqcst);\natomic_rmw_3!(atomic_sub(AtomicUdw::fetch_sub), atomic_sub_non_seqcst, atomic_sub_seqcst);\natomic_rmw_3!(atomic_and(AtomicUdw::fetch_and), atomic_and_non_seqcst, atomic_and_seqcst);\natomic_rmw_3!(atomic_nand(AtomicUdw::fetch_nand), atomic_nand_non_seqcst, atomic_nand_seqcst);\natomic_rmw_3!(atomic_or(AtomicUdw::fetch_or), atomic_or_non_seqcst, atomic_or_seqcst);\natomic_rmw_3!(atomic_xor(AtomicUdw::fetch_xor), atomic_xor_non_seqcst, atomic_xor_seqcst);\natomic_rmw_3!(atomic_max(AtomicIdw::fetch_max), atomic_max_non_seqcst, atomic_max_seqcst);\natomic_rmw_3!(atomic_umax(AtomicUdw::fetch_max), atomic_umax_non_seqcst, atomic_umax_seqcst);\natomic_rmw_3!(atomic_min(AtomicIdw::fetch_min), atomic_min_non_seqcst, atomic_min_seqcst);\natomic_rmw_3!(atomic_umin(AtomicUdw::fetch_min), atomic_umin_non_seqcst, atomic_umin_seqcst);\n\natomic_rmw_2!(atomic_not(AtomicUdw::fetch_not), atomic_not_non_seqcst, atomic_not_seqcst);\natomic_rmw_2!(atomic_neg(AtomicUdw::fetch_neg), atomic_neg_non_seqcst, atomic_neg_seqcst);\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/fallback/seq_lock.rs", "sha256": "f099568551ecd866b8deff71655421f7eff21d045af598debe44d7a355c7bb39", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock.rs.\n\ninclude!(\"seq_lock_common.rs\");\n\nuse crate::imp::core_atomic::AtomicU64;\n\npub(super) type State = u64;\n\n/// A simple stamped lock.\npub(super) struct SeqLock {\n /// The current state of the lock.\n ///\n /// All bits except the least significant one hold the current stamp. When locked, the state\n /// equals 1 and doesn't contain a valid stamp.\n state: AtomicU64,\n}\n\nimpl SeqLock {\n #[inline]\n pub(super) const fn new() -> Self {\n Self { state: AtomicU64::new(0) }\n }\n\n /// If not locked, returns the current stamp.\n ///\n /// This method should be called before optimistic reads.\n #[inline]\n pub(super) fn optimistic_read(&self) -> Option {\n // The acquire load from `state` synchronizes with the release stores in\n // `SeqLockWriteGuard::drop` and `SeqLockWriteGuard::abort`.\n let state = self.state.load(Ordering::Acquire);\n if state == LOCKED { None } else { Some(state) }\n }\n\n /// Returns `true` if the current stamp is equal to `stamp`.\n ///\n /// This method should be called after optimistic reads to check whether they are valid. The\n /// argument `stamp` should correspond to the one returned by method `optimistic_read`.\n #[inline]\n pub(super) fn validate_read(&self, stamp: State) -> bool {\n // Thanks to the fence, if we're noticing any modification to the data at the critical\n // section of `stamp`, then the critical section's write of 1 to state\n // should be visible.\n crate::fence(Ordering::Acquire);\n self.state.load(Ordering::Relaxed) == stamp\n }\n}\n\nimpl SeqLockWriteGuard<'_> {\n #[inline]\n fn next_stamp(&self) -> State {\n self.state.wrapping_add(2)\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/fallback/seq_lock_common.rs", "sha256": "2c519835f7a9b20b64b9cc3428dcf9ed18ad7e56fd73791404d4f7f87975dd3e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock.rs\n// and https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock_wide.rs.\n\nuse core::{mem::ManuallyDrop, sync::atomic::Ordering};\n\nuse super::utils::Backoff;\n#[cfg(portable_atomic_unsafe_assume_privileged)]\nuse crate::imp::interrupt::arch as interrupt;\n\nconst LOCKED: State = 1;\n\nimpl SeqLock {\n /// Grabs the lock for writing.\n #[inline]\n pub(super) fn write(&self) -> SeqLockWriteGuard<'_> {\n // Get current interrupt state and disable interrupts when the user\n // explicitly declares that privileged instructions are available.\n #[cfg(portable_atomic_unsafe_assume_privileged)]\n let interrupt_state = interrupt::disable();\n\n let mut backoff = Backoff::new();\n loop {\n let previous = self.state.swap(LOCKED, Ordering::Acquire);\n\n if previous != LOCKED {\n // To synchronize with the acquire fence in `validate_read` via any modification to\n // the data at the critical section of `previous` (seq_lock.rs) or `(state_hi, previous)`\n // (seq_lock_wide.rs).\n crate::fence(Ordering::Release);\n\n return SeqLockWriteGuard {\n lock: self,\n state: previous,\n #[cfg(portable_atomic_unsafe_assume_privileged)]\n interrupt_state,\n };\n }\n\n while self.state.load(Ordering::Relaxed) == LOCKED {\n backoff.snooze();\n }\n }\n }\n}\n\n/// An RAII guard that releases the lock and increments the stamp when dropped.\n#[must_use]\npub(crate) struct SeqLockWriteGuard<'a> {\n /// The parent lock.\n lock: &'a SeqLock,\n\n /// The stamp before locking.\n state: State,\n\n /// The interrupt state before disabling.\n #[cfg(portable_atomic_unsafe_assume_privileged)]\n interrupt_state: interrupt::State,\n}\n\nimpl SeqLockWriteGuard<'_> {\n /// Releases the lock without incrementing the stamp.\n #[inline]\n pub(crate) fn abort(self) {\n // We specifically don't want to call drop(), since that's\n // what increments the stamp.\n let this = ManuallyDrop::new(self);\n\n // Restore the stamp.\n //\n // Release ordering for synchronizing with `optimistic_read`.\n this.lock.state.store(this.state, Ordering::Release);\n\n // Restore interrupt state.\n // SAFETY: the state was retrieved by the previous `disable`.\n #[cfg(portable_atomic_unsafe_assume_privileged)]\n unsafe {\n interrupt::restore(this.interrupt_state);\n }\n }\n}\n\nimpl Drop for SeqLockWriteGuard<'_> {\n #[inline]\n fn drop(&mut self) {\n let state = self.next_stamp();\n\n // Release the lock and increment the stamp.\n //\n // Release ordering for synchronizing with `optimistic_read`.\n self.lock.state.store(state, Ordering::Release);\n\n // Restore interrupt state.\n // SAFETY: the state was retrieved by the previous `disable`.\n #[cfg(portable_atomic_unsafe_assume_privileged)]\n unsafe {\n interrupt::restore(self.interrupt_state);\n }\n }\n}\n\n#[cfg(test)]\n#[allow(clippy::items_after_test_module)]\nmod tests {\n use core::sync::atomic::Ordering;\n\n use super::SeqLock;\n\n #[test]\n fn smoke() {\n let lock = SeqLock::new();\n let before = lock.optimistic_read().unwrap();\n assert!(lock.validate_read(before));\n {\n let _guard = lock.write();\n assert!(lock.optimistic_read().is_none());\n }\n assert!(!lock.validate_read(before));\n let after = lock.optimistic_read().unwrap();\n assert_ne!(before, after);\n }\n\n #[test]\n fn test_wrap() {\n let lock = SeqLock::new();\n let zero = lock.optimistic_read().unwrap();\n lock.state.store(0xFFFF_FFFE, Ordering::Relaxed); // u32::MAX & !1\n let before = lock.optimistic_read().unwrap();\n assert!(lock.validate_read(before));\n {\n let _guard = lock.write();\n assert!(lock.optimistic_read().is_none());\n }\n assert!(!lock.validate_read(before));\n let after = lock.optimistic_read().unwrap();\n assert_ne!(before, after);\n assert_ne!(after, zero);\n #[allow(clippy::identity_op)]\n {\n assert_eq!(lock.state.load(Ordering::Relaxed) & 0xFFFF_FFFF, 0);\n }\n }\n\n #[test]\n fn test_abort() {\n let lock = SeqLock::new();\n let before = lock.optimistic_read().unwrap();\n {\n let guard = lock.write();\n assert!(lock.optimistic_read().is_none());\n guard.abort();\n }\n let after = lock.optimistic_read().unwrap();\n assert_eq!(before, after, \"aborted write does not update the stamp\");\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/fallback/seq_lock_wide.rs", "sha256": "b2870e179e7d903327ef4ca52e13471e771622ebefd8b06940a5584d8e83416c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/atomic/seq_lock_wide.rs.\n\ninclude!(\"seq_lock_common.rs\");\n\nuse crate::imp::core_atomic::AtomicU32;\n\npub(super) type State = u32;\n\n/// A simple stamped lock.\n///\n/// The state is represented as two `AtomicU32`: `state_hi` for high bits and `state_lo` for low\n/// bits.\npub(super) struct SeqLock {\n /// The high bits of the current state of the lock.\n state_hi: AtomicU32,\n\n /// The low bits of the current state of the lock.\n ///\n /// All bits except the least significant one hold the current stamp. When locked, the state_lo\n /// equals 1 and doesn't contain a valid stamp.\n state: AtomicU32,\n}\n\nimpl SeqLock {\n #[inline]\n pub(super) const fn new() -> Self {\n Self { state_hi: AtomicU32::new(0), state: AtomicU32::new(0) }\n }\n\n /// If not locked, returns the current stamp.\n ///\n /// This method should be called before optimistic reads.\n #[inline]\n pub(super) fn optimistic_read(&self) -> Option<(State, State)> {\n // The acquire loads from `state_hi` and `state_lo` synchronize with the release stores in\n // `SeqLockWriteGuard::drop` and `SeqLockWriteGuard::abort`.\n //\n // As a consequence, we can make sure that (1) all writes within the era of `state_hi - 1`\n // happens before now; and therefore, (2) if `state_lo` is even, all writes within the\n // critical section of (`state_hi`, `state_lo`) happens before now.\n let state_hi = self.state_hi.load(Ordering::Acquire);\n let state_lo = self.state.load(Ordering::Acquire);\n if state_lo == LOCKED { None } else { Some((state_hi, state_lo)) }\n }\n\n /// Returns `true` if the current stamp is equal to `stamp`.\n ///\n /// This method should be called after optimistic reads to check whether they are valid. The\n /// argument `stamp` should correspond to the one returned by method `optimistic_read`.\n #[inline]\n pub(super) fn validate_read(&self, stamp: (State, State)) -> bool {\n // Thanks to the fence, if we're noticing any modification to the data at the critical\n // section of `(stamp.0, stamp.1)`, then the critical section's write of 1 to state_lo\n // should be visible.\n crate::fence(Ordering::Acquire);\n\n // So if `state_lo` coincides with `stamp.1`, then either (1) we're noticing no modification\n // to the data after the critical section of `(stamp.0, stamp.1)`, or (2) `state_lo` wrapped\n // around.\n //\n // If (2) is the case, the acquire ordering ensures we see the new value of `state_hi`.\n let state_lo = self.state.load(Ordering::Acquire);\n\n // If (2) is the case and `state_hi` coincides with `stamp.0`, then `state_hi` also wrapped\n // around, which we give up to correctly validate the read.\n let state_hi = self.state_hi.load(Ordering::Relaxed);\n\n // Except for the case that both `state_hi` and `state_lo` wrapped around, the following\n // condition implies that we're noticing no modification to the data after the critical\n // section of `(stamp.0, stamp.1)`.\n (state_hi, state_lo) == stamp\n }\n}\n\nimpl SeqLockWriteGuard<'_> {\n #[inline]\n fn next_stamp(&self) -> State {\n let state_lo = self.state.wrapping_add(2);\n\n // Increase the high bits if the low bits wrap around.\n //\n // Release ordering for synchronizing with `optimistic_read`.\n if state_lo == 0 {\n let state_hi = self.lock.state_hi.load(Ordering::Relaxed);\n self.lock.state_hi.store(state_hi.wrapping_add(1), Ordering::Release);\n }\n\n state_lo\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/fallback/utils.rs", "sha256": "4b04423031fa66eaf7619e4ddbb30689eb834f74e9dd876b8f4d163cf9c75f73", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse core::ops;\n\n// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/cache_padded.rs.\n/// Pads and aligns a value to the length of a cache line.\n// Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache\n// lines at a time, so we have to align to 128 bytes rather than 64.\n//\n// Sources:\n// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf\n// - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107\n//\n// aarch64/arm64ec's big.LITTLE architecture has asymmetric cores and \"big\" cores have 128-byte cache line size.\n//\n// Sources:\n// - https://www.mono-project.com/news/2016/09/12/arm64-icache/\n//\n// powerpc64 has 128-byte cache line size.\n//\n// Sources:\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/powerpc/include/asm/cache.h#L26\n#[cfg_attr(\n any(\n target_arch = \"x86_64\",\n target_arch = \"aarch64\",\n target_arch = \"arm64ec\",\n target_arch = \"powerpc64\",\n ),\n repr(align(128))\n)]\n// arm, mips, mips64, sparc, and hexagon have 32-byte cache line size.\n//\n// Sources:\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L17\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/hexagon/include/asm/cache.h#L12\n#[cfg_attr(\n any(\n target_arch = \"arm\",\n target_arch = \"mips\",\n target_arch = \"mips32r6\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_arch = \"sparc\",\n target_arch = \"hexagon\",\n ),\n repr(align(32))\n)]\n// m68k has 16-byte cache line size.\n//\n// Sources:\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/m68k/include/asm/cache.h#L9\n#[cfg_attr(target_arch = \"m68k\", repr(align(16)))]\n// s390x has 256-byte cache line size.\n//\n// Sources:\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/s390/include/asm/cache.h#L13\n#[cfg_attr(target_arch = \"s390x\", repr(align(256)))]\n// x86, wasm, riscv, and sparc64 have 64-byte cache line size.\n//\n// Sources:\n// - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9\n// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/riscv/include/asm/cache.h#L10\n// - https://github.com/torvalds/linux/blob/3516bd729358a2a9b090c1905bd2a3fa926e24c6/arch/sparc/include/asm/cache.h#L19\n//\n// All others are assumed to have 64-byte cache line size.\n#[cfg_attr(\n not(any(\n target_arch = \"x86_64\",\n target_arch = \"aarch64\",\n target_arch = \"arm64ec\",\n target_arch = \"powerpc64\",\n target_arch = \"arm\",\n target_arch = \"mips\",\n target_arch = \"mips32r6\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_arch = \"sparc\",\n target_arch = \"hexagon\",\n target_arch = \"m68k\",\n target_arch = \"s390x\",\n )),\n repr(align(64))\n)]\n#[repr(C)]\npub(crate) struct CachePadded {\n value: T,\n}\n\nimpl CachePadded {\n #[inline]\n pub(crate) const fn new(value: T) -> Self {\n Self { value }\n }\n}\n\nimpl ops::Deref for CachePadded {\n type Target = T;\n\n #[inline]\n fn deref(&self) -> &Self::Target {\n &self.value\n }\n}\n\n// Adapted from https://github.com/crossbeam-rs/crossbeam/blob/crossbeam-utils-0.8.21/crossbeam-utils/src/backoff.rs.\n// Adjusted to reduce spinning.\n/// Performs exponential backoff in spin loops.\npub(crate) struct Backoff {\n step: u32,\n}\n\n// https://github.com/oneapi-src/oneTBB/blob/v2021.5.0/include/oneapi/tbb/detail/_utils.h#L46-L48\nconst SPIN_LIMIT: u32 = 4;\n\nimpl Backoff {\n #[inline]\n pub(crate) const fn new() -> Self {\n Self { step: 0 }\n }\n\n #[inline]\n pub(crate) fn snooze(&mut self) {\n if self.step <= SPIN_LIMIT {\n for _ in 0..1 << self.step {\n #[allow(deprecated)]\n core::sync::atomic::spin_loop_hint();\n }\n self.step += 1;\n } else {\n #[cfg(not(feature = \"std\"))]\n for _ in 0..1 << self.step {\n #[allow(deprecated)]\n core::sync::atomic::spin_loop_hint();\n }\n\n #[cfg(feature = \"std\")]\n std::thread::yield_now();\n }\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/float/aarch64.rs", "sha256": "b3d69ef526252b8ee99662f2059898cfa6cb99e1c969828225a2be99de2d45df", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nAtomic float implementation based on AArch64 with FEAT_LSFE.\n\nThis module provides atomic float implementations using FEAT_LSFE instructions.\n\nGenerated asm:\n- aarch64 (+lsfe) https://godbolt.org/z/7vaxeofv1\n*/\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\nuse core::sync::atomic::Ordering;\n\n#[cfg(portable_atomic_unstable_f16)]\nuse super::int::AtomicF16;\n#[cfg(portable_atomic_unstable_f128)]\nuse super::int::AtomicF128;\nuse super::int::{AtomicF32, AtomicF64};\n\n// TODO: optimize no return cases:\n// https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/STFADD--STFADDL--Floating-point-atomic-add-in-memory--without-return-\n// https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/STFMAXNM--STFMAXNML--Floating-point-atomic-maximum-number-in-memory--without-return-\n// https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/STFMINNM--STFMINNML--Floating-point-atomic-minimum-number-in-memory--without-return-\n\n#[cfg(not(portable_atomic_pre_llvm_20))]\nmacro_rules! start_lsfe {\n () => {\n \".arch_extension lsfe\"\n };\n}\n\n#[cfg(not(portable_atomic_pre_llvm_20))]\nmacro_rules! atomic_rmw {\n ($op:ident, $order:ident) => {\n atomic_rmw!($op, $order, write = $order)\n };\n ($op:ident, $order:ident, write = $write:ident) => {\n match $order {\n Ordering::Relaxed => $op!(\"\", \"\", \"\"),\n Ordering::Acquire => $op!(\"a\", \"\", \"\"),\n Ordering::Release => $op!(\"\", \"l\", \"\"),\n Ordering::AcqRel => $op!(\"a\", \"l\", \"\"),\n // In MSVC environments, SeqCst stores/writes by non-LSE* instructions needs fences after writes.\n // https://reviews.llvm.org/D141748\n // https://github.com/llvm/llvm-project/commit/1ea201d73be2fdf03347e9c6be09ebed5f8e0e00\n #[cfg(target_env = \"msvc\")]\n Ordering::SeqCst if $write == Ordering::SeqCst => $op!(\"a\", \"l\", \"dmb ish\"),\n // AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.\n Ordering::SeqCst => $op!(\"a\", \"l\", \"\"),\n _ => unreachable!(),\n }\n };\n}\n#[cfg(portable_atomic_pre_llvm_20)]\nmacro_rules! atomic_rmw_inst {\n ($op:ident, $order:ident) => {\n atomic_rmw_inst!($op, $order, write = $order)\n };\n ($op:ident, $order:ident, write = $write:ident) => {\n match $order {\n Ordering::Relaxed => $op!(\"2\", \"\"), // \"\"\n Ordering::Acquire => $op!(\"a\", \"\"), // \"a\"\n Ordering::Release => $op!(\"6\", \"\"), // \"l\"\n Ordering::AcqRel => $op!(\"e\", \"\"), // \"al\"\n // In MSVC environments, SeqCst stores/writes by non-LSE* instructions needs fences after writes.\n // https://reviews.llvm.org/D141748\n // https://github.com/llvm/llvm-project/commit/1ea201d73be2fdf03347e9c6be09ebed5f8e0e00\n #[cfg(target_env = \"msvc\")]\n Ordering::SeqCst if $write == Ordering::SeqCst => $op!(\"e\", \"dmb ish\"),\n // AcqRel and SeqCst RMWs are equivalent in non-MSVC environments.\n Ordering::SeqCst => $op!(\"e\", \"\"),\n _ => unreachable!(),\n }\n };\n}\n\n#[rustfmt::skip]\nmacro_rules! atomic_float {\n ($atomic_type:ident, $float_type:ident, $modifier:tt, $inst_modifier:tt) => {\n impl $atomic_type {\n #[inline]\n pub(crate) fn fetch_add(&self, val: $float_type, order: Ordering) -> $float_type {\n let dst = self.as_ptr();\n let out;\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n //\n // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/LDFADD--LDFADDA--LDFADDAL--LDFADDL--Floating-point-atomic-add-in-memory-\n unsafe {\n #[cfg(not(portable_atomic_pre_llvm_20))]\n macro_rules! add {\n ($acquire:tt, $release:tt, $_msvc_fence:tt) => {\n asm!(\n start_lsfe!(),\n concat!(\"ldfadd\", $acquire, $release, \" {val:\", $modifier, \"}, {out:\", $modifier, \"}, [{dst}]\"), // atomic { _x = *dst; *dst = _x + val; out = _x }\n dst = in(reg) ptr_reg!(dst),\n val = in(vreg) val,\n out = lateout(vreg) out,\n options(nostack),\n )\n };\n }\n #[cfg(not(portable_atomic_pre_llvm_20))]\n atomic_rmw!(add, order);\n // LLVM supports FEAT_LSFE instructions on LLVM 20+, so use .inst directive on old LLVM.\n // https://github.com/llvm/llvm-project/commit/67ff5ba9af9754261abe11d762af11532a816126\n #[cfg(portable_atomic_pre_llvm_20)]\n macro_rules! add {\n ($order:tt, $_msvc_fence:tt) => {\n asm!(\n // ldfadd{,a,l,al} {h,s,d}0, {h,s,d}1, [x2] // atomic { _x = *x2; *x2 = _x + {h,s,d}0; {h,s,d}1 = _x }\n concat!(\".inst 0x\", $inst_modifier, \"c\", $order, \"00041\"),\n in(\"x2\") ptr_reg!(dst),\n in(\"v0\") val,\n out(\"v1\") out,\n options(nostack),\n )\n };\n }\n #[cfg(portable_atomic_pre_llvm_20)]\n atomic_rmw_inst!(add, order);\n }\n out\n }\n #[inline]\n pub(crate) fn fetch_sub(&self, val: $float_type, order: Ordering) -> $float_type {\n // There is no atomic sub instruction, so add `-val`.\n self.fetch_add(-val, order)\n }\n #[inline]\n pub(crate) fn fetch_max(&self, val: $float_type, order: Ordering) -> $float_type {\n let dst = self.as_ptr();\n let out;\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n //\n // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/LDFMAXNM--LDFMAXNMA--LDFMAXNMAL--LDFMAXNML--Floating-point-atomic-maximum-number-in-memory-\n unsafe {\n #[cfg(not(portable_atomic_pre_llvm_20))]\n macro_rules! max {\n ($acquire:tt, $release:tt, $_msvc_fence:tt) => {\n asm!(\n start_lsfe!(),\n concat!(\"ldfmaxnm\", $acquire, $release, \" {val:\", $modifier, \"}, {out:\", $modifier, \"}, [{dst}]\"), // atomic { _x = *dst; *dst = _x.max(val); out = _x }\n dst = in(reg) ptr_reg!(dst),\n val = in(vreg) val,\n out = lateout(vreg) out,\n options(nostack),\n )\n };\n }\n #[cfg(not(portable_atomic_pre_llvm_20))]\n atomic_rmw!(max, order);\n // LLVM supports FEAT_LSFE instructions on LLVM 20+, so use .inst directive on old LLVM.\n // https://github.com/llvm/llvm-project/commit/67ff5ba9af9754261abe11d762af11532a816126\n #[cfg(portable_atomic_pre_llvm_20)]\n macro_rules! max {\n ($order:tt, $_msvc_fence:tt) => {\n asm!(\n // ldfmaxnm{,a,l,al} {h,s,d}0, {h,s,d}1, [x2] // atomic { _x = *x2; *x2 = _x.max({h,s,d}0); {h,s,d}1 = _x }\n concat!(\".inst 0x\", $inst_modifier, \"c\", $order, \"06041\"),\n in(\"x2\") ptr_reg!(dst),\n in(\"v0\") val,\n out(\"v1\") out,\n options(nostack),\n )\n };\n }\n #[cfg(portable_atomic_pre_llvm_20)]\n atomic_rmw_inst!(max, order);\n }\n out\n }\n #[inline]\n pub(crate) fn fetch_min(&self, val: $float_type, order: Ordering) -> $float_type {\n let dst = self.as_ptr();\n let out;\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n //\n // Refs: https://developer.arm.com/documentation/ddi0602/2025-06/SIMD-FP-Instructions/LDFMINNM--LDFMINNMA--LDFMINNMAL--LDFMINNML--Floating-point-atomic-minimum-number-in-memory-\n unsafe {\n #[cfg(not(portable_atomic_pre_llvm_20))]\n macro_rules! min {\n ($acquire:tt, $release:tt, $_msvc_fence:tt) => {\n asm!(\n start_lsfe!(),\n concat!(\"ldfminnm\", $acquire, $release, \" {val:\", $modifier, \"}, {out:\", $modifier, \"}, [{dst}]\"), // atomic { _x = *dst; *dst = _x.min(val); out = _x }\n dst = in(reg) ptr_reg!(dst),\n val = in(vreg) val,\n out = lateout(vreg) out,\n options(nostack),\n )\n };\n }\n #[cfg(not(portable_atomic_pre_llvm_20))]\n atomic_rmw!(min, order);\n // LLVM supports FEAT_LSFE instructions on LLVM 20+, so use .inst directive on old LLVM.\n // https://github.com/llvm/llvm-project/commit/67ff5ba9af9754261abe11d762af11532a816126\n #[cfg(portable_atomic_pre_llvm_20)]\n macro_rules! min {\n ($order:tt, $_msvc_fence:tt) => {\n asm!(\n // ldfminnm{,a,l,al} {h,s,d}0, {h,s,d}1, [x2] // atomic { _x = *x2; *x2 = _x.min({h,s,d}0); {h,s,d}1 = _x }\n concat!(\".inst 0x\", $inst_modifier, \"c\", $order, \"07041\"),\n in(\"x2\") ptr_reg!(dst),\n in(\"v0\") val,\n out(\"v1\") out,\n options(nostack),\n )\n };\n }\n #[cfg(portable_atomic_pre_llvm_20)]\n atomic_rmw_inst!(min, order);\n }\n out\n }\n }\n };\n}\n\n#[cfg(portable_atomic_unstable_f16)]\natomic_float!(AtomicF16, f16, \"h\", \"7\");\natomic_float!(AtomicF32, f32, \"s\", \"b\");\natomic_float!(AtomicF64, f64, \"d\", \"f\");\n\n#[cfg(portable_atomic_unstable_f128)]\nimpl AtomicF128 {\n #[inline]\n pub(crate) fn fetch_add(&self, val: f128, order: Ordering) -> f128 {\n self.fetch_update_(order, |x| x + val)\n }\n #[inline]\n pub(crate) fn fetch_sub(&self, val: f128, order: Ordering) -> f128 {\n self.fetch_update_(order, |x| x - val)\n }\n #[inline]\n pub(super) fn fetch_update_(&self, order: Ordering, mut f: F) -> f128\n where\n F: FnMut(f128) -> f128,\n {\n // This is a private function and all instances of `f` only operate on the value\n // loaded, so there is no need to synchronize the first load/failed CAS.\n let mut prev = self.load(Ordering::Relaxed);\n loop {\n let next = f(prev);\n match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {\n Ok(x) => return x,\n Err(next_prev) => prev = next_prev,\n }\n }\n }\n #[inline]\n pub(crate) fn fetch_max(&self, val: f128, order: Ordering) -> f128 {\n self.fetch_update_(order, |x| x.max(val))\n }\n #[inline]\n pub(crate) fn fetch_min(&self, val: f128, order: Ordering) -> f128 {\n self.fetch_update_(order, |x| x.min(val))\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/float/int.rs", "sha256": "8801b336650a039bc6e056857c0f65c2f148a0774c83baa9bd9bf415618107e5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nAtomic float implementation based on atomic integer.\n\nThis module provides atomic float implementations using atomic integer.\n\nNote that most of `fetch_*` operations of atomic floats are implemented using\nCAS loops, which can be slower than equivalent operations of atomic integers.\n\nAArch64 with FEAT_LSFE and GPU targets have atomic instructions for float.\nSee aarch64.rs for AArch64 with FEAT_LSFE.\nGPU targets will also use architecture-specific implementations instead of this implementation in the\nfuture: https://github.com/taiki-e/portable-atomic/issues/34 / https://github.com/taiki-e/portable-atomic/pull/45\n*/\n\n// TODO: fetch_{minimum,maximum}* https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p3008r2.html / https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p0493r5.pdf\n\n#![cfg_attr(\n all(target_pointer_width = \"16\", not(feature = \"fallback\")),\n allow(unused_imports, unused_macros)\n)]\n\nuse core::{cell::UnsafeCell, sync::atomic::Ordering};\n\nmacro_rules! atomic_float {\n (\n $atomic_type:ident, $float_type:ident, $atomic_int_type:ident, $int_type:ident,\n $align:literal\n ) => {\n #[repr(C, align($align))]\n pub(crate) struct $atomic_type {\n v: UnsafeCell<$float_type>,\n }\n\n // Send is implicitly implemented.\n // SAFETY: any data races are prevented by atomic operations.\n unsafe impl Sync for $atomic_type {}\n\n impl $atomic_type {\n #[inline]\n pub(crate) const fn new(v: $float_type) -> Self {\n Self { v: UnsafeCell::new(v) }\n }\n\n #[inline]\n pub(crate) fn is_lock_free() -> bool {\n crate::$atomic_int_type::is_lock_free()\n }\n pub(crate) const IS_ALWAYS_LOCK_FREE: bool =\n crate::$atomic_int_type::is_always_lock_free();\n\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn load(&self, order: Ordering) -> $float_type {\n $float_type::from_bits(self.as_bits().load(order))\n }\n\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn store(&self, val: $float_type, order: Ordering) {\n self.as_bits().store(val.to_bits(), order)\n }\n\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];\n #[inline(always)]\n pub(crate) const fn as_bits(&self) -> &crate::$atomic_int_type {\n // SAFETY: $atomic_type and $atomic_int_type have the same layout,\n // and there is no concurrent access to the value that does not go through this method.\n unsafe { &*(self as *const Self as *const crate::$atomic_int_type) }\n }\n }\n\n #[inline]\n pub(crate) const fn as_ptr(&self) -> *mut $float_type {\n self.v.get()\n }\n }\n\n cfg_has_atomic_cas_or_amo32! {\n impl $atomic_type {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn swap(&self, val: $float_type, order: Ordering) -> $float_type {\n $float_type::from_bits(self.as_bits().swap(val.to_bits(), order))\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_neg(&self, order: Ordering) -> $float_type {\n const NEG_MASK: $int_type = !0 / 2 + 1;\n $float_type::from_bits(self.as_bits().fetch_xor(NEG_MASK, order))\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_abs(&self, order: Ordering) -> $float_type {\n const ABS_MASK: $int_type = !0 / 2;\n $float_type::from_bits(self.as_bits().fetch_and(ABS_MASK, order))\n }\n }\n cfg_has_atomic_cas! {\n impl $atomic_type {\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange(\n &self,\n current: $float_type,\n new: $float_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$float_type, $float_type> {\n match self.as_bits().compare_exchange(\n current.to_bits(),\n new.to_bits(),\n success,\n failure,\n ) {\n Ok(v) => Ok($float_type::from_bits(v)),\n Err(v) => Err($float_type::from_bits(v)),\n }\n }\n\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub(crate) fn compare_exchange_weak(\n &self,\n current: $float_type,\n new: $float_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$float_type, $float_type> {\n match self.as_bits().compare_exchange_weak(\n current.to_bits(),\n new.to_bits(),\n success,\n failure,\n ) {\n Ok(v) => Ok($float_type::from_bits(v)),\n Err(v) => Err($float_type::from_bits(v)),\n }\n }\n }\n #[cfg(not(all(\n any(target_arch = \"aarch64\", target_arch = \"arm64ec\"),\n any(target_feature = \"lsfe\", portable_atomic_target_feature = \"lsfe\"),\n target_feature = \"neon\", // for vreg\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n )))]\n impl $atomic_type {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_add(&self, val: $float_type, order: Ordering) -> $float_type {\n self.fetch_update_(order, |x| x + val)\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_sub(&self, val: $float_type, order: Ordering) -> $float_type {\n self.fetch_update_(order, |x| x - val)\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n fn fetch_update_(&self, order: Ordering, mut f: F) -> $float_type\n where\n F: FnMut($float_type) -> $float_type,\n {\n // This is a private function and all instances of `f` only operate on the value\n // loaded, so there is no need to synchronize the first load/failed CAS.\n let mut prev = self.load(Ordering::Relaxed);\n loop {\n let next = f(prev);\n match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) {\n Ok(x) => return x,\n Err(next_prev) => prev = next_prev,\n }\n }\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_max(&self, val: $float_type, order: Ordering) -> $float_type {\n self.fetch_update_(order, |x| x.max(val))\n }\n\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn fetch_min(&self, val: $float_type, order: Ordering) -> $float_type {\n self.fetch_update_(order, |x| x.min(val))\n }\n }\n } // cfg_has_atomic_cas!\n } // cfg_has_atomic_cas_or_amo32!\n };\n}\n\n#[cfg(portable_atomic_unstable_f16)]\ncfg_has_atomic_16! {\n atomic_float!(AtomicF16, f16, AtomicU16, u16, 2);\n}\ncfg_has_atomic_32! {\n atomic_float!(AtomicF32, f32, AtomicU32, u32, 4);\n}\ncfg_has_atomic_64! {\n atomic_float!(AtomicF64, f64, AtomicU64, u64, 8);\n}\n#[cfg(portable_atomic_unstable_f128)]\ncfg_has_atomic_128! {\n atomic_float!(AtomicF128, f128, AtomicU128, u128, 16);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/float/mod.rs", "sha256": "2c722182baee05df65f9f6e2b4de5d5474a3b2bf0c24803a65a771de78b76f95", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nAtomic float implementations\n*/\n\n#![allow(clippy::float_arithmetic)]\n\nmod int;\n\n#[cfg(all(\n any(target_arch = \"aarch64\", target_arch = \"arm64ec\"),\n any(target_feature = \"lsfe\", portable_atomic_target_feature = \"lsfe\"),\n target_feature = \"neon\", // for vreg\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n))]\nmod aarch64;\n\n#[cfg(portable_atomic_unstable_f16)]\ncfg_has_atomic_16! {\n pub(crate) use self::int::AtomicF16;\n}\ncfg_has_atomic_32! {\n pub(crate) use self::int::AtomicF32;\n}\ncfg_has_atomic_64! {\n pub(crate) use self::int::AtomicF64;\n}\n#[cfg(portable_atomic_unstable_f128)]\ncfg_has_atomic_128! {\n pub(crate) use self::int::AtomicF128;\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/interrupt/armv4t.rs", "sha256": "6fb9e7d01f7fb483bc6bec07916fdcba8563b605b53bd87c5310662da04e55a4", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nArm A-Profile Architectures, Arm R-Profile Architectures, Legacy Arm Architectures\n\nRefs: https://developer.arm.com/documentation/ddi0406/cb/System-Level-Architecture/The-System-Level-Programmers--Model/ARM-processor-modes-and-ARM-core-registers/Program-Status-Registers--PSRs-\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\n\n// - 0x80 - I (IRQ mask) bit (1 << 7)\n// - 0x40 - F (FIQ mask) bit (1 << 6)\n// We disable only IRQs by default. See also https://github.com/taiki-e/portable-atomic/pull/28#issuecomment-1214146912.\n#[cfg(not(portable_atomic_disable_fiq))]\nmacro_rules! mask {\n () => {\n \"0x80\"\n };\n}\n#[cfg(portable_atomic_disable_fiq)]\nmacro_rules! mask {\n () => {\n \"0xC0\" // 0x80 | 0x40\n };\n}\n\npub(crate) type State = u32;\n\n/// Disables interrupts and returns the previous interrupt state.\n#[inline]\n#[cfg_attr(\n not(any(target_feature = \"v7\", portable_atomic_target_feature = \"v7\")),\n instruction_set(arm::a32)\n)]\npub(crate) fn disable() -> State {\n let cpsr: State;\n // SAFETY: reading CPSR and disabling interrupts are safe.\n // (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)\n unsafe {\n asm!(\n \"mrs {prev}, cpsr\", // prev = CPSR\n concat!(\"orr {new}, {prev}, \", mask!()), // new = prev | mask\n \"msr cpsr_c, {new}\", // CPSR.{I,F,T,M} = new.{I,F,T,M}\n prev = out(reg) cpsr,\n new = out(reg) _,\n // Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.\n options(nostack, preserves_flags),\n );\n }\n cpsr\n}\n\n/// Restores the previous interrupt state.\n///\n/// # Safety\n///\n/// The state must be the one retrieved by the previous `disable`.\n#[inline]\n#[cfg_attr(\n not(any(target_feature = \"v7\", portable_atomic_target_feature = \"v7\")),\n instruction_set(arm::a32)\n)]\npub(crate) unsafe fn restore(prev_cpsr: State) {\n // SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,\n //\n // This clobbers the control field mask byte of CPSR. See msp430.rs for safety on this.\n // (preserves_flags is fine because we can clobber only the I, F, T, and M bits of CPSR.)\n //\n // Refs: https://developer.arm.com/documentation/dui0473/m/arm-and-thumb-instructions/msr--general-purpose-register-to-psr-\n unsafe {\n asm!(\n \"msr cpsr_c, {prev_cpsr}\", // CPSR.{I,F,T,M} = prev_cpsr.{I,F,T,M}\n prev_cpsr = in(reg) prev_cpsr,\n // Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.\n options(nostack, preserves_flags),\n );\n }\n}\n\n// On pre-v6 Arm, we cannot use core::sync::atomic here because they call the\n// `__sync_*` builtins for non-relaxed load/store (because pre-v6 Arm doesn't\n// have Data Memory Barrier).\n#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(test, not(target_has_atomic = \"ptr\")))\n)]\npub(super) mod atomic {\n #[cfg(not(portable_atomic_no_asm))]\n use core::arch::asm;\n use core::{cell::UnsafeCell, sync::atomic::Ordering};\n\n macro_rules! atomic {\n ($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $suffix:tt) => {\n #[repr(transparent)]\n pub(crate) struct $atomic_type $(<$($generics)*>)? {\n v: UnsafeCell<$value_type>,\n }\n\n // Send is implicitly implemented for atomic integers, but not for atomic pointers.\n // SAFETY: any data races are prevented by atomic operations.\n unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}\n // SAFETY: any data races are prevented by atomic operations.\n unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}\n\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n #[inline]\n pub(crate) fn load(&self, _order: Ordering) -> $value_type {\n let src = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n let out;\n // inline asm without nomem/readonly implies compiler fence.\n // And compiler fence is fine because the user explicitly declares that\n // the system is single-core by using an unsafe cfg.\n asm!(\n concat!(\"ldr\", $suffix, \" {out}, [{src}]\"), // atomic { out = *src }\n src = in(reg) src,\n out = lateout(reg) out,\n options(nostack, preserves_flags),\n );\n out\n }\n }\n\n #[inline]\n pub(crate) fn store(&self, val: $value_type, _order: Ordering) {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n // inline asm without nomem/readonly implies compiler fence.\n // And compiler fence is fine because the user explicitly declares that\n // the system is single-core by using an unsafe cfg.\n asm!(\n concat!(\"str\", $suffix, \" {val}, [{dst}]\"), // atomic { *dst = val }\n dst = in(reg) dst,\n val = in(reg) val,\n options(nostack, preserves_flags),\n );\n }\n }\n }\n };\n }\n\n atomic!(AtomicI8, i8, \"b\");\n atomic!(AtomicU8, u8, \"b\");\n atomic!(AtomicI16, i16, \"h\");\n atomic!(AtomicU16, u16, \"h\");\n atomic!(AtomicI32, i32, \"\");\n atomic!(AtomicU32, u32, \"\");\n atomic!(AtomicIsize, isize, \"\");\n atomic!(AtomicUsize, usize, \"\");\n atomic!([T] AtomicPtr, *mut T, \"\");\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/interrupt/armv6m.rs", "sha256": "364a38f18ffb2b3cd836700dff6a390d9542f0f2a3eda07bc63bc8868dafaad1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nArm M-Profile Architectures\n\nAdapted from https://github.com/rust-embedded/cortex-m.\n\nRefs: https://developer.arm.com/documentation/ddi0419/c/System-Level-Architecture/System-Level-Programmers--Model/Registers/The-special-purpose-mask-register--PRIMASK\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(test, portable_atomic_no_atomic_cas))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(test, not(target_has_atomic = \"ptr\")))\n)]\npub(super) use core::sync::atomic;\n\npub(crate) type State = u32;\n\n/// Disables interrupts and returns the previous interrupt state.\n#[inline(always)]\npub(crate) fn disable() -> State {\n let primask: State;\n // SAFETY: reading the priority mask register and disabling interrupts are safe.\n // (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)\n unsafe {\n asm!(\n \"mrs {primask}, PRIMASK\", // primask = PRIMASK\n \"cpsid i\", // PRIMASK.PM = 1\n primask = out(reg) primask,\n // Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.\n options(nostack, preserves_flags),\n );\n }\n primask\n}\n\n/// Restores the previous interrupt state.\n///\n/// # Safety\n///\n/// The state must be the one retrieved by the previous `disable`.\n#[inline(always)]\npub(crate) unsafe fn restore(prev_primask: State) {\n // SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,\n // and we've checked that interrupts were enabled before disabling interrupts.\n //\n // This clobbers the entire PRIMASK register. See msp430.rs for safety on this.\n unsafe {\n asm!(\n \"msr PRIMASK, {prev_primask}\", // PRIMASK = prev_primask\n prev_primask = in(reg) prev_primask,\n // Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.\n options(nostack, preserves_flags),\n );\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/interrupt/avr.rs", "sha256": "252569da4b194f91d7f52c6480a869e26679dc74c30a3a8fcd6568a39b90440d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nAdapted from https://github.com/Rahix/avr-device.\n\nSee also src/imp/avr.rs.\n\nRefs:\n- AVR® Instruction Set Manual, Rev. DS40002198B\n https://ww1.microchip.com/downloads/en/DeviceDoc/AVR-InstructionSet-Manual-DS40002198.pdf\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\n\n#[cfg(not(portable_atomic_no_asm))]\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(test, portable_atomic_no_atomic_cas))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(test, not(target_has_atomic = \"ptr\")))\n)]\npub(super) use super::super::avr as atomic;\n\npub(crate) type State = u8;\n\n/// Disables interrupts and returns the previous interrupt state.\n#[inline(always)]\npub(crate) fn disable() -> State {\n let sreg: State;\n // SAFETY: reading the status register (SREG) and disabling interrupts are safe.\n // (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)\n unsafe {\n // Refs: https://ww1.microchip.com/downloads/en/DeviceDoc/AVR-InstructionSet-Manual-DS40002198.pdf#page=58\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n \"in {sreg}, 0x3F\", // sreg = SREG\n \"cli\", // SREG.I = 0\n sreg = out(reg) sreg,\n // Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.\n // Do not use `preserves_flags` because CLI modifies the I bit of the status register (SREG).\n options(nostack),\n );\n #[cfg(portable_atomic_no_asm)]\n {\n llvm_asm!(\"in $0, 0x3F\" : \"=r\"(sreg) ::: \"volatile\");\n llvm_asm!(\"cli\" ::: \"memory\" : \"volatile\");\n }\n }\n sreg\n}\n\n/// Restores the previous interrupt state.\n///\n/// # Safety\n///\n/// The state must be the one retrieved by the previous `disable`.\n#[inline(always)]\npub(crate) unsafe fn restore(prev_sreg: State) {\n // SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`.\n //\n // This clobbers the entire status register, but all other flags are the arithmetic flags that\n // are changed as a side effect of arithmetic operations, etc., which LLVM recognizes unless\n // preserves_flags is set, so it is safe to clobber them here.\n // See also the discussion at https://github.com/taiki-e/portable-atomic/pull/40.\n unsafe {\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n \"out 0x3F, {prev_sreg}\", // SREG = prev_sreg\n prev_sreg = in(reg) prev_sreg,\n // Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.\n // Do not use `preserves_flags` because OUT modifies the status register (SREG).\n options(nostack),\n );\n #[cfg(portable_atomic_no_asm)]\n llvm_asm!(\"out 0x3F, $0\" :: \"r\"(prev_sreg) : \"memory\" : \"volatile\");\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/interrupt/mod.rs", "sha256": "256ed2675f921dc2c2fb0ea06b610990e391ec2714dc460383e21d688a0f751f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nFallback implementation based on disabling interrupts or critical-section\n\n- mod.rs contains critical section based fallback implementations.\n- Each architecture modules contain implementations of disabling interrupts.\n\nSee README.md of this directory for details.\n*/\n\n#[cfg(not(feature = \"critical-section\"))]\n#[cfg_attr(\n all(\n target_arch = \"arm\",\n any(target_feature = \"mclass\", portable_atomic_target_feature = \"mclass\"),\n ),\n path = \"armv6m.rs\"\n)]\n#[cfg_attr(\n all(\n target_arch = \"arm\",\n not(any(target_feature = \"mclass\", portable_atomic_target_feature = \"mclass\")),\n ),\n path = \"armv4t.rs\"\n)]\n#[cfg_attr(target_arch = \"avr\", path = \"avr.rs\")]\n#[cfg_attr(target_arch = \"msp430\", path = \"msp430.rs\")]\n#[cfg_attr(any(target_arch = \"riscv32\", target_arch = \"riscv64\"), path = \"riscv.rs\")]\n#[cfg_attr(target_arch = \"xtensa\", path = \"xtensa.rs\")]\npub(super) mod arch;\n\n#[cfg(not(all(\n portable_atomic_unsafe_assume_privileged,\n not(portable_atomic_unsafe_assume_single_core),\n)))]\nitems!({\n use core::{cell::UnsafeCell, sync::atomic::Ordering};\n\n // critical-section implementations might use locks internally.\n #[cfg(feature = \"critical-section\")]\n const IS_ALWAYS_LOCK_FREE: bool = false;\n // Consider atomic operations based on disabling interrupts on single-core\n // systems are lock-free. (We consider the pre-v6 Arm Linux's atomic operations\n // provided in a similar way by the Linux kernel to be lock-free.)\n #[cfg(not(feature = \"critical-section\"))]\n const IS_ALWAYS_LOCK_FREE: bool = true;\n\n // Put this in its own module to prevent guard creation.\n use self::guard::disable;\n mod guard {\n // Note: The caller must NOT explicitly modify registers containing fields modified by disable/restore.\n // (Fields modified as side effects of other operations are covered by the absence of preserves_flags,\n // so they are fine -- see msp430.rs for more.)\n #[inline(always)]\n pub(super) fn disable() -> Guard {\n Guard {\n #[cfg(feature = \"critical-section\")]\n // SAFETY: the state will be restored in the subsequent `release`.\n state: unsafe { critical_section::acquire() },\n #[cfg(not(feature = \"critical-section\"))]\n // Get current interrupt state and disable interrupts.\n state: super::arch::disable(),\n }\n }\n pub(super) struct Guard {\n #[cfg(feature = \"critical-section\")]\n state: critical_section::RestoreState,\n #[cfg(not(feature = \"critical-section\"))]\n state: super::arch::State,\n }\n impl Drop for Guard {\n #[inline(always)]\n fn drop(&mut self) {\n #[cfg(feature = \"critical-section\")]\n // SAFETY: the state was retrieved by the previous `acquire`.\n unsafe {\n critical_section::release(self.state);\n }\n #[cfg(not(feature = \"critical-section\"))]\n // Restore interrupt state.\n // SAFETY: the state was retrieved by the previous `disable`.\n unsafe {\n super::arch::restore(self.state);\n }\n }\n }\n }\n\n macro_rules! atomic_base {\n (base, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $align:literal) => {\n #[repr(C, align($align))]\n pub(crate) struct $atomic_type $(<$($generics)*>)? {\n v: UnsafeCell<$value_type>,\n }\n\n // Send is implicitly implemented for atomic integers, but not for atomic pointers.\n // SAFETY: any data races are prevented by disabling interrupts (or\n // atomic intrinsics) or critical-section (see module-level comments).\n unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}\n // SAFETY: any data races are prevented by disabling interrupts (or\n // atomic intrinsics) or critical-section (see module-level comments).\n unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}\n\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n #[inline]\n pub(crate) const fn new(v: $value_type) -> Self {\n Self { v: UnsafeCell::new(v) }\n }\n\n #[inline]\n pub(crate) fn is_lock_free() -> bool {\n Self::IS_ALWAYS_LOCK_FREE\n }\n pub(crate) const IS_ALWAYS_LOCK_FREE: bool = IS_ALWAYS_LOCK_FREE;\n\n #[inline]\n fn read(&self, _guard: &guard::Guard) -> $value_type {\n // SAFETY: any data races are prevented by disabling interrupts or critical-section (see\n // module-level comments) and the raw pointer is valid because we got it\n // from a reference.\n unsafe { self.v.get().read() }\n }\n #[inline]\n fn write(&self, val: $value_type, _guard: &guard::Guard) {\n // SAFETY: any data races are prevented by disabling interrupts or critical-section (see\n // module-level comments) and the raw pointer is valid because we got it\n // from a reference.\n unsafe { self.v.get().write(val) }\n }\n\n // As for CAS, there is no corresponding atomic operation on all architectures that use this code.\n // (If the CAS instruction exists, all atomic operations can be implemented by it, so this code will not be used.)\n #[inline]\n #[cfg_attr(\n all(debug_assertions, not(portable_atomic_no_track_caller)),\n track_caller\n )]\n pub(crate) fn compare_exchange(\n &self,\n current: $value_type,\n new: $value_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$value_type, $value_type> {\n crate::utils::assert_compare_exchange_ordering(success, failure);\n let guard = disable();\n let prev = self.read(&guard);\n if prev == current {\n self.write(new, &guard);\n Ok(prev)\n } else {\n Err(prev)\n }\n }\n #[inline]\n #[cfg_attr(\n all(debug_assertions, not(portable_atomic_no_track_caller)),\n track_caller\n )]\n pub(crate) fn compare_exchange_weak(\n &self,\n current: $value_type,\n new: $value_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$value_type, $value_type> {\n self.compare_exchange(current, new, success, failure)\n }\n\n #[inline]\n pub(crate) const fn as_ptr(&self) -> *mut $value_type {\n self.v.get()\n }\n }\n };\n (native_load_store, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {\n impl $(<$($generics)*>)? core::ops::Deref for $atomic_type $(<$($generics)*>)? {\n type Target = atomic::$atomic_type $(<$($generics)*>)?;\n #[inline(always)]\n fn deref(&self) -> &Self::Target {\n // SAFETY: $atomic_type and atomic::$atomic_type have the same layout and\n // guarantee atomicity in a compatible way. (see module-level comments)\n unsafe {\n &*(self as *const Self as *const atomic::$atomic_type $(<$($generics)*>)?)\n }\n }\n }\n };\n (emulate_load_store, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n #[inline]\n #[cfg_attr(\n all(debug_assertions, not(portable_atomic_no_track_caller)),\n track_caller\n )]\n pub(crate) fn load(&self, order: Ordering) -> $value_type {\n crate::utils::assert_load_ordering(order);\n let guard = disable();\n self.read(&guard)\n }\n #[inline]\n #[cfg_attr(\n all(debug_assertions, not(portable_atomic_no_track_caller)),\n track_caller\n )]\n pub(crate) fn store(&self, val: $value_type, order: Ordering) {\n crate::utils::assert_store_ordering(order);\n let guard = disable();\n self.write(val, &guard);\n }\n }\n };\n (emulate_swap, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty) => {\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n #[inline]\n pub(crate) fn swap(&self, val: $value_type, _order: Ordering) -> $value_type {\n let guard = disable();\n let prev = self.read(&guard);\n self.write(val, &guard);\n prev\n }\n }\n };\n }\n\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(\n test,\n target_arch = \"avr\",\n target_arch = \"msp430\",\n portable_atomic_no_atomic_cas\n ))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n test,\n target_arch = \"avr\",\n target_arch = \"msp430\",\n not(target_has_atomic = \"ptr\")\n ))\n )]\n items!({\n #[cfg(target_pointer_width = \"16\")]\n atomic_base!(base, [T] AtomicPtr, *mut T, 2);\n #[cfg(target_pointer_width = \"32\")]\n atomic_base!(base, [T] AtomicPtr, *mut T, 4);\n #[cfg(target_pointer_width = \"64\")]\n atomic_base!(base, [T] AtomicPtr, *mut T, 8);\n #[cfg(target_pointer_width = \"128\")]\n atomic_base!(base, [T] AtomicPtr, *mut T, 16);\n\n impl_default_bit_opts!(AtomicPtr, usize);\n\n cfg_sel!({\n #[cfg(any(target_arch = \"avr\", feature = \"critical-section\"))]\n {\n atomic_base!(emulate_load_store, [T] AtomicPtr, *mut T);\n }\n #[cfg(else)]\n {\n atomic_base!(native_load_store, [T] AtomicPtr, *mut T);\n }\n });\n\n #[cfg(not(all(\n any(target_arch = \"riscv32\", target_arch = \"riscv64\"),\n not(feature = \"critical-section\"),\n any(\n portable_atomic_force_amo,\n target_feature = \"zaamo\",\n portable_atomic_target_feature = \"zaamo\",\n ),\n )))]\n items!({\n atomic_base!(emulate_swap, [T] AtomicPtr, *mut T);\n impl AtomicPtr {\n #[inline]\n pub(crate) fn fetch_byte_add(&self, val: usize, _order: Ordering) -> *mut T {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n let guard = disable();\n let prev = self.read(&guard);\n self.write(prev.with_addr(prev.addr().wrapping_add(val)), &guard);\n prev\n }\n #[inline]\n pub(crate) fn fetch_byte_sub(&self, val: usize, _order: Ordering) -> *mut T {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n let guard = disable();\n let prev = self.read(&guard);\n self.write(prev.with_addr(prev.addr().wrapping_sub(val)), &guard);\n prev\n }\n #[inline]\n pub(crate) fn fetch_and(&self, val: usize, _order: Ordering) -> *mut T {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n let guard = disable();\n let prev = self.read(&guard);\n self.write(prev.with_addr(prev.addr() & val), &guard);\n prev\n }\n #[inline]\n pub(crate) fn fetch_or(&self, val: usize, _order: Ordering) -> *mut T {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n let guard = disable();\n let prev = self.read(&guard);\n self.write(prev.with_addr(prev.addr() | val), &guard);\n prev\n }\n #[inline]\n pub(crate) fn fetch_xor(&self, val: usize, _order: Ordering) -> *mut T {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n let guard = disable();\n let prev = self.read(&guard);\n self.write(prev.with_addr(prev.addr() ^ val), &guard);\n prev\n }\n }\n });\n\n impl AtomicPtr {\n #[cfg(test)]\n #[inline]\n fn fetch_ptr_add(&self, val: usize, order: Ordering) -> *mut T {\n self.fetch_byte_add(val.wrapping_mul(core::mem::size_of::()), order)\n }\n #[cfg(test)]\n #[inline]\n fn fetch_ptr_sub(&self, val: usize, order: Ordering) -> *mut T {\n self.fetch_byte_sub(val.wrapping_mul(core::mem::size_of::()), order)\n }\n }\n });\n\n macro_rules! atomic_int {\n (base, $atomic_type:ident, $int_type:ty, $align:literal) => {\n atomic_base!(base, $atomic_type, $int_type, $align);\n // As for nand and neg, there is no corresponding atomic operation on all architectures that use this code.\n impl $atomic_type {\n #[inline]\n pub(crate) fn fetch_nand(&self, val: $int_type, _order: Ordering) -> $int_type {\n let guard = disable();\n let prev = self.read(&guard);\n self.write(!(prev & val), &guard);\n prev\n }\n #[inline]\n pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type {\n let guard = disable();\n let prev = self.read(&guard);\n self.write(prev.wrapping_neg(), &guard);\n prev\n }\n #[inline]\n pub(crate) fn neg(&self, order: Ordering) {\n self.fetch_neg(order);\n }\n }\n };\n (load_store_atomic $([$kind:ident])?, $atomic_type:ident, $int_type:ty, $align:literal) => {\n cfg_sel!({\n #[cfg(feature = \"critical-section\")]\n {\n atomic_int!(all_critical_session, $atomic_type, $int_type, $align);\n }\n #[cfg(else)]\n {\n atomic_int!(base, $atomic_type, $int_type, $align);\n impl_default_bit_opts!($atomic_type, $int_type);\n // load/store\n cfg_sel!({\n // AVR with very old rustc\n #[cfg(all(target_arch = \"avr\", portable_atomic_no_asm))]\n {\n atomic_base!(emulate_load_store, $atomic_type, $int_type);\n }\n #[cfg(else)]\n {\n atomic_base!(native_load_store, $atomic_type, $int_type);\n }\n });\n // RMW\n cfg_sel!({\n // AVR 8-bit RMW with RMW instructions\n #[cfg(all(\n target_arch = \"avr\",\n not(portable_atomic_no_asm),\n any(target_feature = \"rmw\", portable_atomic_target_feature = \"rmw\"),\n ))]\n {\n atomic_int!(emulate_arithmetic, $atomic_type, $int_type);\n }\n // RISC-V RMW with Zaamo extension\n #[cfg(all(\n any(target_arch = \"riscv32\", target_arch = \"riscv64\"),\n any(\n portable_atomic_force_amo,\n target_feature = \"zaamo\",\n portable_atomic_target_feature = \"zaamo\",\n ),\n ))]\n {\n atomic_int!(cas $([$kind])?, $atomic_type, $int_type);\n }\n #[cfg(else)]\n {\n atomic_int!(cas[emulate], $atomic_type, $int_type);\n }\n });\n // RMW (no-fetch)\n #[cfg(not(target_arch = \"msp430\"))]\n items!({\n impl_default_no_fetch_ops!($atomic_type, $int_type);\n impl $atomic_type {\n #[inline]\n pub(crate) fn not(&self, order: Ordering) {\n self.fetch_not(order);\n }\n }\n });\n }\n });\n };\n (all_critical_session, $atomic_type:ident, $int_type:ty, $align:literal) => {\n atomic_int!(base, $atomic_type, $int_type, $align);\n atomic_base!(emulate_load_store, $atomic_type, $int_type);\n atomic_int!(cas[emulate], $atomic_type, $int_type);\n impl_default_no_fetch_ops!($atomic_type, $int_type);\n impl_default_bit_opts!($atomic_type, $int_type);\n impl $atomic_type {\n #[inline]\n pub(crate) fn not(&self, order: Ordering) {\n self.fetch_not(order);\n }\n }\n };\n (emulate_arithmetic, $atomic_type:ident, $int_type:ty) => {\n impl $atomic_type {\n #[inline]\n pub(crate) fn fetch_add(&self, val: $int_type, _order: Ordering) -> $int_type {\n let guard = disable();\n let prev = self.read(&guard);\n self.write(prev.wrapping_add(val), &guard);\n prev\n }\n #[inline]\n pub(crate) fn fetch_sub(&self, val: $int_type, _order: Ordering) -> $int_type {\n let guard = disable();\n let prev = self.read(&guard);\n self.write(prev.wrapping_sub(val), &guard);\n prev\n }\n #[inline]\n pub(crate) fn fetch_max(&self, val: $int_type, _order: Ordering) -> $int_type {\n let guard = disable();\n let prev = self.read(&guard);\n self.write(core::cmp::max(prev, val), &guard);\n prev\n }\n #[inline]\n pub(crate) fn fetch_min(&self, val: $int_type, _order: Ordering) -> $int_type {\n let guard = disable();\n let prev = self.read(&guard);\n self.write(core::cmp::min(prev, val), &guard);\n prev\n }\n }\n };\n (emulate_bit, $atomic_type:ident, $int_type:ty) => {\n impl $atomic_type {\n #[inline]\n pub(crate) fn fetch_and(&self, val: $int_type, _order: Ordering) -> $int_type {\n let guard = disable();\n let prev = self.read(&guard);\n self.write(prev & val, &guard);\n prev\n }\n #[inline]\n pub(crate) fn fetch_or(&self, val: $int_type, _order: Ordering) -> $int_type {\n let guard = disable();\n let prev = self.read(&guard);\n self.write(prev | val, &guard);\n prev\n }\n #[inline]\n pub(crate) fn fetch_xor(&self, val: $int_type, _order: Ordering) -> $int_type {\n let guard = disable();\n let prev = self.read(&guard);\n self.write(prev ^ val, &guard);\n prev\n }\n #[inline]\n pub(crate) fn fetch_not(&self, _order: Ordering) -> $int_type {\n let guard = disable();\n let prev = self.read(&guard);\n self.write(!prev, &guard);\n prev\n }\n }\n };\n (cas[emulate], $atomic_type:ident, $int_type:ty) => {\n atomic_base!(emulate_swap, $atomic_type, $int_type);\n atomic_int!(emulate_arithmetic, $atomic_type, $int_type);\n atomic_int!(emulate_bit, $atomic_type, $int_type);\n };\n // RISC-V 32-bit(RV32)/{32,64}-bit(RV64) RMW with Zaamo extension\n // RISC-V 8-bit/16-bit RMW with Zabha extension\n (cas, $atomic_type:ident, $int_type:ty) => {};\n // RISC-V 8-bit/16-bit RMW with Zaamo extension\n (cas[sub_word], $atomic_type:ident, $int_type:ty) => {\n // RISC-V 8-bit/16-bit RMW with Zaamo+Zabha extension\n #[cfg(any(target_feature = \"zabha\", portable_atomic_target_feature = \"zabha\"))]\n atomic_int!(cas, $atomic_type, $int_type);\n\n // RISC-V 8-bit/16-bit RMW with Zaamo extension\n #[cfg(not(any(target_feature = \"zabha\", portable_atomic_target_feature = \"zabha\")))]\n atomic_base!(emulate_swap, $atomic_type, $int_type);\n #[cfg(not(any(target_feature = \"zabha\", portable_atomic_target_feature = \"zabha\")))]\n atomic_int!(emulate_arithmetic, $atomic_type, $int_type);\n };\n }\n\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(\n test,\n target_arch = \"avr\",\n target_arch = \"msp430\",\n portable_atomic_no_atomic_cas,\n ))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n test,\n target_arch = \"avr\",\n target_arch = \"msp430\",\n not(target_has_atomic = \"ptr\"),\n ))\n )]\n items!({\n // On some platforms, atomic load/store can be implemented in a more efficient\n // way than disabling interrupts. On MSP430, some RMWs that do not return the\n // previous value can also be optimized.\n //\n // Note: On single-core systems, it is okay to use critical session-based\n // CAS together with atomic load/store. The load/store will not be\n // called while interrupts are disabled, and since the load/store is\n // atomic, it is not affected by interrupts even if interrupts are enabled.\n #[cfg(not(any(\n all(target_arch = \"avr\", portable_atomic_no_asm),\n feature = \"critical-section\",\n )))]\n use self::arch::atomic;\n\n #[cfg(target_pointer_width = \"16\")]\n #[cfg(not(target_arch = \"avr\"))]\n atomic_int!(load_store_atomic, AtomicIsize, isize, 2);\n #[cfg(target_pointer_width = \"16\")]\n #[cfg(not(target_arch = \"avr\"))]\n atomic_int!(load_store_atomic, AtomicUsize, usize, 2);\n #[cfg(target_arch = \"avr\")]\n atomic_int!(all_critical_session, AtomicIsize, isize, 2);\n #[cfg(target_arch = \"avr\")]\n atomic_int!(all_critical_session, AtomicUsize, usize, 2);\n #[cfg(target_pointer_width = \"32\")]\n atomic_int!(load_store_atomic, AtomicIsize, isize, 4);\n #[cfg(target_pointer_width = \"32\")]\n atomic_int!(load_store_atomic, AtomicUsize, usize, 4);\n #[cfg(target_pointer_width = \"64\")]\n atomic_int!(load_store_atomic, AtomicIsize, isize, 8);\n #[cfg(target_pointer_width = \"64\")]\n atomic_int!(load_store_atomic, AtomicUsize, usize, 8);\n #[cfg(target_pointer_width = \"128\")]\n atomic_int!(load_store_atomic, AtomicIsize, isize, 16);\n #[cfg(target_pointer_width = \"128\")]\n atomic_int!(load_store_atomic, AtomicUsize, usize, 16);\n\n #[cfg(not(all(target_arch = \"avr\", portable_atomic_no_asm)))]\n atomic_int!(load_store_atomic[sub_word], AtomicI8, i8, 1);\n #[cfg(not(all(target_arch = \"avr\", portable_atomic_no_asm)))]\n atomic_int!(load_store_atomic[sub_word], AtomicU8, u8, 1);\n #[cfg(all(target_arch = \"avr\", portable_atomic_no_asm))]\n atomic_int!(all_critical_session, AtomicI8, i8, 1);\n #[cfg(all(target_arch = \"avr\", portable_atomic_no_asm))]\n atomic_int!(all_critical_session, AtomicU8, u8, 1);\n #[cfg(not(target_arch = \"avr\"))]\n atomic_int!(load_store_atomic[sub_word], AtomicI16, i16, 2);\n #[cfg(not(target_arch = \"avr\"))]\n atomic_int!(load_store_atomic[sub_word], AtomicU16, u16, 2);\n #[cfg(target_arch = \"avr\")]\n atomic_int!(all_critical_session, AtomicI16, i16, 2);\n #[cfg(target_arch = \"avr\")]\n atomic_int!(all_critical_session, AtomicU16, u16, 2);\n\n #[cfg(not(target_pointer_width = \"16\"))]\n atomic_int!(load_store_atomic, AtomicI32, i32, 4);\n #[cfg(not(target_pointer_width = \"16\"))]\n atomic_int!(load_store_atomic, AtomicU32, u32, 4);\n\n cfg_has_fast_atomic_64!({\n atomic_int!(load_store_atomic, AtomicI64, i64, 8);\n atomic_int!(load_store_atomic, AtomicU64, u64, 8);\n });\n });\n\n // Double or more width atomics (require fallback feature for consistency with other situations).\n #[cfg(target_pointer_width = \"16\")]\n #[cfg(any(test, feature = \"fallback\"))]\n items!({\n atomic_int!(all_critical_session, AtomicI32, i32, 4);\n atomic_int!(all_critical_session, AtomicU32, u32, 4);\n });\n #[cfg(any(\n test,\n all(\n feature = \"fallback\",\n not(all(\n target_arch = \"riscv32\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(\n target_feature = \"zacas\",\n portable_atomic_target_feature = \"zacas\",\n all(\n feature = \"fallback\",\n not(portable_atomic_no_outline_atomics),\n any(target_os = \"linux\", target_os = \"android\"),\n ),\n ),\n )),\n ),\n ))]\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(test, portable_atomic_no_atomic_64))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(test, not(target_has_atomic = \"64\")))\n )]\n cfg_no_fast_atomic_64!({\n atomic_int!(all_critical_session, AtomicI64, i64, 8);\n atomic_int!(all_critical_session, AtomicU64, u64, 8);\n });\n #[cfg(any(\n test,\n all(\n feature = \"fallback\",\n not(all(\n target_arch = \"riscv64\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(\n target_feature = \"zacas\",\n portable_atomic_target_feature = \"zacas\",\n all(\n feature = \"fallback\",\n not(portable_atomic_no_outline_atomics),\n any(target_os = \"linux\", target_os = \"android\"),\n ),\n ),\n )),\n ),\n ))]\n items!({\n atomic_int!(all_critical_session, AtomicI128, i128, 16);\n atomic_int!(all_critical_session, AtomicU128, u128, 16);\n });\n});\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n test_atomic_ptr_single_thread!();\n test_atomic_int_single_thread!(i8);\n test_atomic_int_single_thread!(u8);\n test_atomic_int_single_thread!(i16);\n test_atomic_int_single_thread!(u16);\n test_atomic_int_single_thread!(i32);\n test_atomic_int_single_thread!(u32);\n test_atomic_int_single_thread!(i64);\n test_atomic_int_single_thread!(u64);\n test_atomic_int_single_thread!(i128);\n test_atomic_int_single_thread!(u128);\n test_atomic_int_single_thread!(isize);\n test_atomic_int_single_thread!(usize);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/interrupt/msp430.rs", "sha256": "a7131a478fca956de4ca569f340641da2eb6a5cbd86a05a854f1231dd24e62d1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nAdapted from https://github.com/rust-embedded/msp430.\n\nSee also src/imp/msp430.rs.\n\nRefs:\n- MSP430x5xx and MSP430x6xx Family User's Guide, Rev. Q\n https://www.ti.com/lit/ug/slau208q/slau208q.pdf\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\n\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(test, portable_atomic_no_atomic_cas))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(test, not(target_has_atomic = \"ptr\")))\n)]\npub(super) use super::super::msp430 as atomic;\n\npub(crate) type State = u16;\n\n/// Disables interrupts and returns the previous interrupt state.\n#[inline(always)]\npub(crate) fn disable() -> State {\n let sr: State;\n // SAFETY: reading the status register and disabling interrupts are safe.\n // (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)\n //\n // See \"NOTE: Enable and Disable Interrupt\" of User's Guide for NOP: https://www.ti.com/lit/ug/slau208q/slau208q.pdf#page=60\n unsafe {\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n \"mov r2, {sr}\", // sr = SR\n \"dint {{ nop\", // SR.GIE = 0\n sr = out(reg) sr,\n // Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.\n // Do not use `preserves_flags` because DINT modifies the GIE (global interrupt enable) bit of the status register.\n options(nostack),\n );\n #[cfg(portable_atomic_no_asm)]\n {\n llvm_asm!(\"mov r2, $0\" : \"=r\"(sr) ::: \"volatile\");\n llvm_asm!(\"dint { nop\" ::: \"memory\" : \"volatile\");\n }\n }\n sr\n}\n\n/// Restores the previous interrupt state.\n///\n/// # Safety\n///\n/// The state must be the one retrieved by the previous `disable`.\n#[inline(always)]\npub(crate) unsafe fn restore(prev_sr: State) {\n // SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,\n //\n // This clobbers the entire status register, but we never explicitly modify\n // flags within a critical session, and the only flags that may be changed\n // within a critical session are the arithmetic flags that are changed as\n // a side effect of arithmetic operations, etc., which LLVM recognizes unless\n // preserves_flags is set, so it is safe to clobber them here.\n // See also the discussion at https://github.com/taiki-e/portable-atomic/pull/40.\n //\n // See \"NOTE: Enable and Disable Interrupt\" of User's Guide for NOP: https://www.ti.com/lit/ug/slau208q/slau208q.pdf#page=60\n unsafe {\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n \"nop {{ mov {prev_sr}, r2 {{ nop\", // SR = prev_sr\n prev_sr = in(reg) prev_sr,\n // Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.\n // Do not use `preserves_flags` because MOV modifies the status register.\n options(nostack),\n );\n #[cfg(portable_atomic_no_asm)]\n llvm_asm!(\"nop { mov $0, r2 { nop\" :: \"r\"(prev_sr) : \"memory\" : \"volatile\");\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/interrupt/riscv.rs", "sha256": "32a0291b174310385f850c19283d583852b128cb9c037633c8a5e872245842bc", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nRefs:\n- RISC-V Instruction Set Manual\n Machine Status (mstatus and mstatush) Registers\n https://docs.riscv.org/reference/isa/v20260120/priv/machine.html#3-1-1-6-machine-status-mstatus-and-mstatush-registers\n Supervisor Status (sstatus) Register\n https://docs.riscv.org/reference/isa/v20260120/priv/supervisor.html#sstatus\n\nSee also src/imp/riscv.rs.\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\n\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(test, portable_atomic_no_atomic_cas))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(test, not(target_has_atomic = \"ptr\")))\n)]\npub(super) use super::super::riscv as atomic;\n\ncfg_sel!({\n // Supervisor-mode (S-mode)\n #[cfg(portable_atomic_s_mode)]\n {\n // Status register\n macro_rules! status {\n () => {\n \"sstatus\"\n };\n }\n // SIE (Supervisor Interrupt Enable) bit (1 << 1)\n #[cfg(portable_atomic_s_mode)]\n macro_rules! mask {\n () => {\n \"0x2\"\n };\n }\n const MASK: State = 0x2;\n }\n // Machine-mode (M-mode)\n #[cfg(else)]\n {\n // Status register\n macro_rules! status {\n () => {\n \"mstatus\"\n };\n }\n // MIE (Machine Interrupt Enable) bit (1 << 3)\n macro_rules! mask {\n () => {\n \"0x8\"\n };\n }\n const MASK: State = 0x8;\n }\n});\n\npub(crate) type State = crate::utils::RegSize;\n\n/// Disables interrupts and returns the previous interrupt state.\n#[inline(always)]\npub(crate) fn disable() -> State {\n let status: State;\n // SAFETY: reading mstatus/sstatus and disabling interrupts is safe.\n // (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)\n unsafe {\n asm!(\n concat!(\"csrrci {status}, \", status!(), \", \", mask!()), // atomic { status = status!(); status!() &= !mask!() }\n status = out(reg) status,\n // Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.\n options(nostack, preserves_flags),\n );\n }\n status\n}\n\n/// Restores the previous interrupt state.\n///\n/// # Safety\n///\n/// The state must be the one retrieved by the previous `disable`.\n#[inline(always)]\npub(crate) unsafe fn restore(prev_status: State) {\n // SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,\n //\n // This avoids clobbering other fields in the mstatus/sstatus register.\n unsafe {\n asm!(\n concat!(\"csrs \", status!(), \", {mask}\"), // atomic { status!() |= mask }\n mask = in(reg) prev_status & MASK,\n // Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.\n options(nostack, preserves_flags),\n );\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/interrupt/xtensa.rs", "sha256": "7c877f42c237d7ff6564776ee9c4afa5b8143d4fb6b39b4ac869e5e357189b4f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nRefs:\n- Xtensa Instruction Set Architecture (ISA) Summary for all Xtensa LX Processors\n https://www.cadence.com/content/dam/cadence-www/global/en_US/documents/tools/silicon-solutions/compute-ip/isa-summary.pdf\n- Linux kernel's Xtensa atomic implementation\n https://github.com/torvalds/linux/blob/v6.19/arch/xtensa/include/asm/atomic.h\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\nuse core::arch::asm;\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(test, portable_atomic_no_atomic_cas))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(test, not(target_has_atomic = \"ptr\")))\n)]\npub(super) use core::sync::atomic;\n\npub(crate) type State = u32;\n\n/// Disables interrupts and returns the previous interrupt state.\n#[inline(always)]\npub(crate) fn disable() -> State {\n let ps: State;\n // SAFETY: reading the PS special register and disabling all interrupts is safe.\n // (see module-level comments of interrupt/mod.rs on the safety of using privileged instructions)\n //\n // Interrupt level 15 to disable all interrupts.\n // SYNC after RSIL is not required.\n // Use a14 per https://github.com/torvalds/linux/blob/v6.16/arch/xtensa/include/asm/atomic.h#L33-L37.\n unsafe {\n asm!(\n \"rsil a14, 15\", // a14 = PS; PS.INTLEVEL = 15\n out(\"a14\") ps,\n // Do not use `nomem` and `readonly` because prevent subsequent memory accesses from being reordered before interrupts are disabled.\n options(nostack),\n );\n }\n ps\n}\n\n/// Restores the previous interrupt state.\n///\n/// # Safety\n///\n/// The state must be the one retrieved by the previous `disable`.\n#[inline(always)]\npub(crate) unsafe fn restore(prev_ps: State) {\n // SAFETY: the caller must guarantee that the state was retrieved by the previous `disable`,\n // and we've checked that interrupts were enabled before disabling interrupts.\n //\n // SYNC after WSR is required to guarantee that subsequent RSIL read the written value.\n // See also 3.8.10 Processor Control Instructions of Xtensa Instruction Set Architecture (ISA) Summary for all Xtensa LX Processors.\n unsafe {\n asm!(\n \"wsr.ps a14\", // PS = a14\n \"rsync\", // wait\n in(\"a14\") prev_ps,\n // Do not use `nomem` and `readonly` because prevent preceding memory accesses from being reordered after interrupts are enabled.\n options(nostack),\n );\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/mod.rs", "sha256": "d9505be50a63ea4b00590a3637266ccb88dc85078572aa8fd8bf4236bb9ae18c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n// -----------------------------------------------------------------------------\n// Lock-free implementations\n\ncfg_core_atomic!({\n mod core_atomic;\n});\n\n// AVR\n#[cfg(target_arch = \"avr\")]\n#[cfg(not(portable_atomic_no_asm))]\n#[cfg(not(feature = \"critical-section\"))]\nmod avr;\n\n// pre-v6 Arm Linux\n#[cfg(all(\n target_arch = \"arm\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(target_os = \"linux\", target_os = \"android\"),\n not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\")),\n))]\npub(crate) mod arm_linux;\n\n// MSP430\n#[cfg(target_arch = \"msp430\")]\npub(crate) mod msp430;\n\n// RISC-V without A-extension\n#[cfg(any(test, not(feature = \"critical-section\")))]\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(\n all(test, not(any(miri, portable_atomic_sanitize_thread))),\n portable_atomic_no_atomic_cas,\n ))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n all(test, not(any(miri, portable_atomic_sanitize_thread))),\n not(target_has_atomic = \"ptr\"),\n ))\n)]\n#[cfg(any(target_arch = \"riscv32\", target_arch = \"riscv64\"))]\nmod riscv;\n\n// x86-specific optimizations\n// Miri and Sanitizer do not support inline assembly.\n#[cfg(all(\n any(target_arch = \"x86\", target_arch = \"x86_64\"),\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n))]\npub(crate) mod x86;\n\n// 64-bit atomic implementations on 32-bit architectures\n#[cfg(any(target_arch = \"arm\", target_arch = \"riscv32\"))]\nmod atomic64;\n\n// 128-bit atomic implementations on 64-bit architectures\n#[cfg(any(\n target_arch = \"aarch64\",\n target_arch = \"arm64ec\",\n target_arch = \"powerpc64\",\n target_arch = \"riscv64\",\n target_arch = \"s390x\",\n target_arch = \"x86_64\",\n))]\nmod atomic128;\n\n// -----------------------------------------------------------------------------\n// Lock-based fallback implementations\n\n#[cfg(feature = \"fallback\")]\n#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]\n#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = \"ptr\"))]\n#[cfg_attr(\n not(test),\n cfg(not(any(\n // ---------------\n // Use other ways.\n target_arch = \"avr\",\n target_arch = \"msp430\",\n portable_atomic_unsafe_assume_single_core,\n // --------------------\n // Has 128-bit atomics.\n all(\n target_arch = \"aarch64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n ),\n all(\n target_arch = \"arm64ec\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n ),\n all(\n target_arch = \"x86_64\",\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n ),\n all(\n target_arch = \"riscv64\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"),\n ),\n all(\n target_arch = \"powerpc64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n ),\n ),\n all(\n target_arch = \"s390x\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n ),\n ))\n))]\nmod fallback;\n\n// -----------------------------------------------------------------------------\n// Fallback implementation based on disabling interrupts or critical-section\n\n// On AVR, we always use critical section based fallback implementation.\n// AVR can be safely assumed to be single-core, so this is sound.\n// MSP430 as well.\n// See the module-level comments of interrupt module for more.\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n all(feature = \"critical-section\", any(test, portable_atomic_no_atomic_cas)),\n portable_atomic_unsafe_assume_single_core,\n portable_atomic_unsafe_assume_privileged,\n ))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n all(feature = \"critical-section\", any(test, not(target_has_atomic = \"ptr\"))),\n portable_atomic_unsafe_assume_single_core,\n portable_atomic_unsafe_assume_privileged,\n ))\n)]\n#[cfg(any(\n target_arch = \"arm\",\n target_arch = \"avr\",\n target_arch = \"msp430\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"xtensa\",\n feature = \"critical-section\",\n))]\nmod interrupt;\n\n// -----------------------------------------------------------------------------\n// Atomic float implementations\n\n#[cfg(feature = \"float\")]\npub(crate) mod float;\n\n// -----------------------------------------------------------------------------\n\n// {8,16,32}-bit & ptr-sized atomics\ncfg_sel!({\n // has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(not(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n portable_atomic_no_atomic_load_store,\n all(\n any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ),\n portable_atomic_no_atomic_cas,\n ),\n )))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(not(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n portable_atomic_no_atomic_load_store,\n all(\n any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ),\n not(target_has_atomic = \"ptr\"),\n ),\n )))\n )]\n {\n pub(crate) use self::core_atomic::{\n AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicU32,\n AtomicUsize,\n };\n }\n // no native atomic CAS & (assume single core | critical section) => critical section based fallback\n #[cfg(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ))]\n {\n pub(crate) use self::interrupt::{\n AtomicI8, AtomicI16, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicUsize,\n };\n #[cfg(any(not(target_pointer_width = \"16\"), feature = \"fallback\"))]\n pub(crate) use self::interrupt::{AtomicI32, AtomicU32};\n }\n // RISC-V without A-extension & !(assume single core | critical section)\n #[cfg(any(target_arch = \"riscv32\", target_arch = \"riscv64\"))]\n {\n pub(crate) use self::riscv::{\n AtomicI8, AtomicI16, AtomicI32, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16, AtomicU32,\n AtomicUsize,\n };\n }\n // bpf & !(critical section) => core atomic\n #[cfg(target_arch = \"bpf\")]\n {\n pub(crate) use self::core_atomic::{AtomicIsize, AtomicPtr, AtomicUsize};\n }\n #[cfg(else)]\n {\n // no atomic\n }\n});\n\n// 64-bit atomics\ncfg_sel!({\n // has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(all(\n not(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n portable_atomic_no_atomic_load_store,\n all(\n any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ),\n portable_atomic_no_atomic_cas,\n )\n )),\n any(\n not(portable_atomic_no_atomic_64),\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")),\n ),\n ))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(all(\n not(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n portable_atomic_no_atomic_load_store,\n all(\n any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n ),\n not(target_has_atomic = \"ptr\"),\n )\n )),\n any(\n target_has_atomic = \"64\",\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")),\n ),\n ))\n )]\n {\n pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};\n }\n // pre-v6 Arm Linux\n #[cfg(all(\n feature = \"fallback\",\n target_arch = \"arm\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_os = \"linux\", target_os = \"android\"),\n not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\")),\n not(portable_atomic_no_outline_atomics),\n ))]\n {\n pub(crate) use self::atomic64::arm_linux::{AtomicI64, AtomicU64};\n }\n // riscv32 & (zacas | outline-atomics)\n #[cfg(all(\n target_arch = \"riscv32\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(\n target_feature = \"zacas\",\n portable_atomic_target_feature = \"zacas\",\n all(\n feature = \"fallback\",\n not(portable_atomic_no_outline_atomics),\n any(target_os = \"linux\", target_os = \"android\"),\n ),\n ),\n ))]\n {\n pub(crate) use self::atomic64::riscv32::{AtomicI64, AtomicU64};\n }\n // no native atomic CAS & (assume single core | critical section) => critical section based fallback\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n all(feature = \"critical-section\", portable_atomic_no_atomic_cas),\n portable_atomic_unsafe_assume_single_core,\n ))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n all(feature = \"critical-section\", not(target_has_atomic = \"ptr\")),\n portable_atomic_unsafe_assume_single_core,\n ))\n )]\n {\n #[cfg(any(\n not(any(target_pointer_width = \"16\", target_pointer_width = \"32\")),\n feature = \"fallback\",\n ))]\n pub(crate) use self::interrupt::{AtomicI64, AtomicU64};\n }\n // no core 64-bit atomic & has CAS => use lock-base fallback\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(all(feature = \"fallback\", not(portable_atomic_no_atomic_cas)))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(all(feature = \"fallback\", target_has_atomic = \"ptr\"))\n )]\n {\n pub(crate) use self::fallback::{AtomicI64, AtomicU64};\n }\n // RISC-V without A-extension & !(assume single core | critical section)\n #[cfg(target_arch = \"riscv64\")]\n {\n pub(crate) use self::riscv::{AtomicI64, AtomicU64};\n }\n // bpf & !(critical section) => core atomic\n #[cfg(target_arch = \"bpf\")]\n {\n pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};\n }\n #[cfg(else)]\n {\n // no atomic\n }\n});\n\n// 128-bit atomics\ncfg_sel!({\n // AArch64\n #[cfg(any(\n all(\n target_arch = \"aarch64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n ),\n all(\n target_arch = \"arm64ec\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n ),\n ))]\n {\n pub(crate) use self::atomic128::aarch64::{AtomicI128, AtomicU128};\n }\n // x86_64 & (cmpxchg16b | outline-atomics)\n #[cfg(all(\n target_arch = \"x86_64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n portable_atomic_no_cmpxchg16b_intrinsic,\n )),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(\n target_feature = \"cmpxchg16b\",\n portable_atomic_target_feature = \"cmpxchg16b\",\n all(\n feature = \"fallback\",\n not(portable_atomic_no_outline_atomics),\n not(any(target_env = \"sgx\", miri)),\n ),\n ),\n ))]\n {\n pub(crate) use self::atomic128::x86_64::{AtomicI128, AtomicU128};\n }\n // riscv64 & (zacas | outline-atomics)\n #[cfg(all(\n target_arch = \"riscv64\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(\n target_feature = \"zacas\",\n portable_atomic_target_feature = \"zacas\",\n all(\n feature = \"fallback\",\n not(portable_atomic_no_outline_atomics),\n any(target_os = \"linux\", target_os = \"android\"),\n ),\n ),\n ))]\n {\n pub(crate) use self::atomic128::riscv64::{AtomicI128, AtomicU128};\n }\n // powerpc64 & (pwr8 | outline-atomics)\n #[cfg(all(\n target_arch = \"powerpc64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n all(\n feature = \"fallback\",\n not(portable_atomic_no_outline_atomics),\n any(\n all(\n target_os = \"linux\",\n any(\n all(\n target_env = \"gnu\",\n any(target_endian = \"little\", not(target_feature = \"crt-static\")),\n ),\n all(\n target_env = \"musl\",\n any(not(target_feature = \"crt-static\"), feature = \"std\"),\n ),\n target_env = \"ohos\",\n all(target_env = \"uclibc\", not(target_feature = \"crt-static\")),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"android\",\n all(\n target_os = \"freebsd\",\n any(\n target_endian = \"little\",\n not(target_feature = \"crt-static\"),\n portable_atomic_outline_atomics,\n ),\n ),\n target_os = \"openbsd\",\n all(\n target_os = \"aix\",\n not(portable_atomic_pre_llvm_20),\n portable_atomic_outline_atomics,\n ), // TODO(aix): currently disabled by default\n ),\n not(any(miri, portable_atomic_sanitize_thread)),\n ),\n ),\n ))]\n {\n pub(crate) use self::atomic128::powerpc64::{AtomicI128, AtomicU128};\n }\n // s390x\n #[cfg(all(\n target_arch = \"s390x\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n ))]\n {\n pub(crate) use self::atomic128::s390x::{AtomicI128, AtomicU128};\n }\n // no native atomic CAS & (assume single core | critical section) => critical section based fallback\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n all(feature = \"critical-section\", portable_atomic_no_atomic_cas),\n portable_atomic_unsafe_assume_single_core,\n ))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(any(\n target_arch = \"avr\",\n target_arch = \"msp430\",\n all(feature = \"critical-section\", not(target_has_atomic = \"ptr\")),\n portable_atomic_unsafe_assume_single_core,\n ))\n )]\n {\n #[cfg(feature = \"fallback\")]\n pub(crate) use self::interrupt::{AtomicI128, AtomicU128};\n }\n // no core 128-bit atomic & has CAS => use lock-base fallback\n #[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(all(feature = \"fallback\", not(portable_atomic_no_atomic_cas)))\n )]\n #[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(all(feature = \"fallback\", target_has_atomic = \"ptr\"))\n )]\n {\n pub(crate) use self::fallback::{AtomicI128, AtomicU128};\n }\n #[cfg(else)]\n {\n // no atomic\n }\n});\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/msp430.rs", "sha256": "16635485eeca1c1c4b866253eefc1f39fa6121c84c16ec389b904c03fa9cdb55", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nAtomic implementation on MSP430.\n\nAdapted from https://github.com/pftbest/msp430-atomic.\n\nOperations not supported here are provided by disabling interrupts.\nSee also src/imp/interrupt/msp430.rs.\n\nSee \"Atomic operation overview by architecture\" in atomic-maybe-uninit for a more comprehensive and\ndetailed description of the atomic and synchronize instructions in this architecture:\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#msp430\n\nNote: Ordering is always SeqCst.\n\nRefs:\n- MSP430x5xx and MSP430x6xx Family User's Guide, Rev. Q\n https://www.ti.com/lit/ug/slau208q/slau208q.pdf\n- atomic-maybe-uninit\n https://github.com/taiki-e/atomic-maybe-uninit\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\n#[cfg(not(feature = \"critical-section\"))]\nuse core::cell::UnsafeCell;\nuse core::sync::atomic::Ordering;\n\n/// An atomic fence.\n///\n/// # Panics\n///\n/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).\n#[inline]\n#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\npub fn fence(order: Ordering) {\n match order {\n Ordering::Relaxed => panic!(\"there is no such thing as a relaxed fence\"),\n // MSP430 is single-core and a compiler fence works as an atomic fence.\n _ => compiler_fence(order),\n }\n}\n\n/// A compiler memory fence.\n///\n/// # Panics\n///\n/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).\n#[inline]\n#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\npub fn compiler_fence(order: Ordering) {\n match order {\n Ordering::Relaxed => panic!(\"there is no such thing as a relaxed compiler fence\"),\n _ => {}\n }\n // SAFETY: using an empty asm is safe.\n unsafe {\n // Do not use `nomem` and `readonly` because prevent preceding and subsequent memory accesses from being reordered.\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\"\", options(nostack, preserves_flags));\n #[cfg(portable_atomic_no_asm)]\n llvm_asm!(\"\" ::: \"memory\" : \"volatile\");\n }\n}\n\n#[cfg(not(feature = \"critical-section\"))]\nitems!({\n macro_rules! atomic {\n (load_store, $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $size:tt) => {\n #[repr(transparent)]\n pub(crate) struct $atomic_type $(<$($generics)*>)? {\n v: UnsafeCell<$value_type>,\n }\n\n // Send is implicitly implemented for atomic integers, but not for atomic pointers.\n // SAFETY: any data races are prevented by atomic operations.\n unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}\n // SAFETY: any data races are prevented by atomic operations.\n unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}\n\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n #[inline]\n #[cfg_attr(\n all(debug_assertions, not(portable_atomic_no_track_caller)),\n track_caller\n )]\n pub(crate) fn load(&self, order: Ordering) -> $value_type {\n crate::utils::assert_load_ordering(order);\n let src = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n let out;\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n concat!(\"mov.\", $size, \" @{src}, {out}\"), // atomic { out = *src }\n src = in(reg) src,\n out = lateout(reg) out,\n options(nostack, preserves_flags),\n );\n #[cfg(portable_atomic_no_asm)]\n llvm_asm!(\n concat!(\"mov.\", $size, \" $1, $0\")\n : \"=r\"(out) : \"*m\"(src) : \"memory\" : \"volatile\"\n );\n out\n }\n }\n\n #[inline]\n #[cfg_attr(\n all(debug_assertions, not(portable_atomic_no_track_caller)),\n track_caller\n )]\n pub(crate) fn store(&self, val: $value_type, order: Ordering) {\n crate::utils::assert_store_ordering(order);\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n concat!(\"mov.\", $size, \" {val}, 0({dst})\"), // atomic { *dst = val }\n dst = in(reg) dst,\n val = in(reg) val,\n options(nostack, preserves_flags),\n );\n #[cfg(portable_atomic_no_asm)]\n llvm_asm!(\n concat!(\"mov.\", $size, \" $1, $0\")\n :: \"*m\"(dst), \"ir\"(val) : \"memory\" : \"volatile\"\n );\n }\n }\n }\n };\n ($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $size:tt) => {\n atomic!(load_store, $([$($generics)*])? $atomic_type, $value_type, $size);\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n #[inline]\n pub(crate) fn add(&self, val: $value_type, _order: Ordering) {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n concat!(\"add.\", $size, \" {val}, 0({dst})\"), // atomic { *dst += val }\n dst = in(reg) dst,\n val = in(reg) val,\n // Do not use `preserves_flags` because ADD modifies the V, N, Z, and C bits of the status register.\n options(nostack),\n );\n #[cfg(portable_atomic_no_asm)]\n llvm_asm!(\n concat!(\"add.\", $size, \" $1, $0\")\n :: \"*m\"(dst), \"ir\"(val) : \"memory\" : \"volatile\"\n );\n }\n }\n\n #[inline]\n pub(crate) fn sub(&self, val: $value_type, _order: Ordering) {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n concat!(\"sub.\", $size, \" {val}, 0({dst})\"), // atomic { *dst -= val }\n dst = in(reg) dst,\n val = in(reg) val,\n // Do not use `preserves_flags` because SUB modifies the V, N, Z, and C bits of the status register.\n options(nostack),\n );\n #[cfg(portable_atomic_no_asm)]\n llvm_asm!(\n concat!(\"sub.\", $size, \" $1, $0\")\n :: \"*m\"(dst), \"ir\"(val) : \"memory\" : \"volatile\"\n );\n }\n }\n\n #[inline]\n pub(crate) fn and(&self, val: $value_type, _order: Ordering) {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n concat!(\"and.\", $size, \" {val}, 0({dst})\"), // atomic { *dst &= val }\n dst = in(reg) dst,\n val = in(reg) val,\n // Do not use `preserves_flags` because AND modifies the V, N, Z, and C bits of the status register.\n options(nostack),\n );\n #[cfg(portable_atomic_no_asm)]\n llvm_asm!(\n concat!(\"and.\", $size, \" $1, $0\")\n :: \"*m\"(dst), \"ir\"(val) : \"memory\" : \"volatile\"\n );\n }\n }\n\n #[inline]\n pub(crate) fn or(&self, val: $value_type, _order: Ordering) {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n concat!(\"bis.\", $size, \" {val}, 0({dst})\"), // atomic { *dst |= val }\n dst = in(reg) dst,\n val = in(reg) val,\n options(nostack, preserves_flags),\n );\n #[cfg(portable_atomic_no_asm)]\n llvm_asm!(\n concat!(\"bis.\", $size, \" $1, $0\")\n :: \"*m\"(dst), \"ir\"(val) : \"memory\" : \"volatile\"\n );\n }\n }\n\n #[inline]\n pub(crate) fn xor(&self, val: $value_type, _order: Ordering) {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n concat!(\"xor.\", $size, \" {val}, 0({dst})\"), // atomic { *dst ^= val }\n dst = in(reg) dst,\n val = in(reg) val,\n // Do not use `preserves_flags` because XOR modifies the V, N, Z, and C bits of the status register.\n options(nostack),\n );\n #[cfg(portable_atomic_no_asm)]\n llvm_asm!(\n concat!(\"xor.\", $size, \" $1, $0\")\n :: \"*m\"(dst), \"ir\"(val) : \"memory\" : \"volatile\"\n );\n }\n }\n\n #[inline]\n pub(crate) fn not(&self, _order: Ordering) {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n #[cfg(not(portable_atomic_no_asm))]\n asm!(\n concat!(\"inv.\", $size, \" 0({dst})\"), // atomic { *dst = !*dst }\n dst = in(reg) dst,\n // Do not use `preserves_flags` because INV modifies the V, N, Z, and C bits of the status register.\n options(nostack),\n );\n #[cfg(portable_atomic_no_asm)]\n llvm_asm!(\n concat!(\"inv.\", $size, \" $0\")\n :: \"*m\"(dst) : \"memory\" : \"volatile\"\n );\n }\n }\n }\n };\n }\n\n atomic!(AtomicI8, i8, \"b\");\n atomic!(AtomicU8, u8, \"b\");\n atomic!(AtomicI16, i16, \"w\");\n atomic!(AtomicU16, u16, \"w\");\n atomic!(AtomicIsize, isize, \"w\");\n atomic!(AtomicUsize, usize, \"w\");\n atomic!(load_store, [T] AtomicPtr, *mut T, \"w\");\n});\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/riscv.rs", "sha256": "72c05a835e13b007ff587ec2fd35f619c32b74f18f0aa544fc304456ad5d255a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nAtomic load/store implementation on RISC-V.\n\nThis is for RISC-V targets without A extension. (pre-1.76 rustc doesn't provide atomics\nat all on such targets. https://github.com/rust-lang/rust/pull/114499)\n\nAlso, optionally provides RMW implementation when Zaamo extension or force-amo feature is enabled.\n\nIf A extension is not available but Zalrsc or Zacas extension is available, LLVM can implement all\natomic operations using it (https://github.com/llvm/llvm-project/commit/70f70390bb83a383fcb88ea843adbc4edf6ee34b),\nso this module contains no code for these extensions.\n\nSee \"Atomic operation overview by architecture\" in atomic-maybe-uninit for a more comprehensive and\ndetailed description of the atomic and synchronize instructions in this architecture:\nhttps://github.com/taiki-e/atomic-maybe-uninit/blob/HEAD/src/arch/README.md#risc-v\n\nRefs:\n- RISC-V Instruction Set Manual\n \"Zaamo\" Extension for Atomic Memory Operations\n https://docs.riscv.org/reference/isa/v20260120/unpriv/a-st-ext.html#sec:amo\n \"Zabha\" Extension for Byte and Halfword Atomic Memory Operations\n https://docs.riscv.org/reference/isa/v20260120/unpriv/zabha.html\n- RISC-V Atomics ABI Specification\n https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20250812-301374e92976e298e676e7129a6212926b2299ce/riscv-atomic.adoc\n- atomic-maybe-uninit\n https://github.com/taiki-e/atomic-maybe-uninit\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\nuse core::{cell::UnsafeCell, sync::atomic::Ordering};\n\n#[cfg(any(\n test,\n portable_atomic_force_amo,\n target_feature = \"zaamo\",\n portable_atomic_target_feature = \"zaamo\",\n))]\nitems!({\n macro_rules! atomic_rmw_amo_ext {\n // Use +a also for zaamo because `option arch +zaamo` requires LLVM 19.\n // https://github.com/llvm/llvm-project/commit/8be079cdddfd628d356d9ddb5ab397ea95fb1030\n (\"w\") => {\n \"+a\"\n };\n (\"d\") => {\n \"+a\"\n };\n (\"b\") => {\n \"+a,+zabha\"\n };\n (\"h\") => {\n \"+a,+zabha\"\n };\n }\n macro_rules! atomic_rmw_amo {\n ($op:ident, $dst:ident, $val:ident, $order:ident, $size:tt) => {{\n let out;\n macro_rules! op {\n ($asm_order:tt) => {\n // SAFETY: The user guaranteed that the AMO instruction is available in this\n // system by setting the portable_atomic_force_amo/target_feature and\n // portable_atomic_unsafe_assume_single_core.\n // The caller of this macro must guarantee the validity of the pointer.\n asm!(\n \".option push\",\n // https://github.com/riscv-non-isa/riscv-asm-manual/blob/v0.0.1/src/asm-manual.adoc#arch\n // LLVM supports `.option arch` directive on LLVM 17+.\n // https://github.com/llvm/llvm-project/commit/9e8ed3403c191ab9c4903e8eeb8f732ff8a43cb4\n // Note that `.insn ` directive requires LLVM 19.\n // https://github.com/llvm/llvm-project/commit/2a086dce691e3cc34a2fc27f4fb255bb2cbbfac9\n concat!(\".option arch, \", atomic_rmw_amo_ext!($size)),\n concat!(\"amo\", stringify!($op), \".\", $size, $asm_order, \" {out}, {val}, 0({dst})\"), // atomic { _x = *dst; *dst = op(_x, val); out = _x }\n \".option pop\",\n dst = in(reg) ptr_reg!($dst),\n val = in(reg) $val,\n out = lateout(reg) out,\n options(nostack, preserves_flags),\n )\n };\n }\n match $order {\n Ordering::Relaxed => op!(\"\"),\n Ordering::Acquire => op!(\".aq\"),\n Ordering::Release => op!(\".rl\"),\n // AcqRel and SeqCst RMWs are equivalent.\n Ordering::AcqRel | Ordering::SeqCst => op!(\".aqrl\"),\n _ => unreachable!(),\n }\n out\n }};\n }\n\n #[cfg(not(any(target_feature = \"zabha\", portable_atomic_target_feature = \"zabha\")))]\n items!({\n #[cfg(target_arch = \"riscv32\")]\n macro_rules! w {\n () => {\n \"\"\n };\n }\n #[cfg(target_arch = \"riscv64\")]\n macro_rules! w {\n () => {\n \"w\"\n };\n }\n\n // 32-bit val.wrapping_shl(shift) but no extra `& (u32::BITS - 1)`\n #[inline(always)]\n fn sllw(val: u32, shift: u32) -> u32 {\n // SAFETY: Calling sll{,w} is safe.\n unsafe {\n let out;\n asm!(\n concat!(\"sll\", w!(), \" {out}, {val}, {shift}\"), // out = val << shift & 31\n out = lateout(reg) out,\n val = in(reg) val,\n shift = in(reg) shift,\n options(pure, nomem, nostack, preserves_flags),\n );\n out\n }\n }\n // 32-bit val.wrapping_shr(shift) but no extra `& (u32::BITS - 1)`\n macro_rules! srlw {\n ($val:expr, $shift:expr) => {\n // SAFETY: Calling srl{,w} is safe.\n unsafe {\n let val: u32 = $val;\n let shift: u32 = $shift;\n let out;\n asm!(\n concat!(\"srl\", w!(), \" {out}, {val}, {shift}\"), // out = val >> shift & 31\n out = lateout(reg) out,\n val = in(reg) val,\n shift = in(reg) shift,\n options(pure, nomem, nostack, preserves_flags),\n );\n out\n }\n };\n }\n\n trait ZeroExtend: Copy {\n /// Zero-extends `self` to `u32` if it is smaller than 32-bit.\n fn zero_extend(self) -> u32;\n }\n macro_rules! zero_extend {\n ($int:ident, $uint:ident) => {\n impl ZeroExtend for $uint {\n #[inline(always)]\n fn zero_extend(self) -> u32 {\n self as u32\n }\n }\n impl ZeroExtend for $int {\n #[allow(clippy::cast_sign_loss)]\n #[inline(always)]\n fn zero_extend(self) -> u32 {\n self as $uint as u32\n }\n }\n };\n }\n zero_extend!(i8, u8);\n zero_extend!(i16, u16);\n });\n});\n\nmacro_rules! atomic_load_store {\n ($([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $size:tt) => {\n #[repr(transparent)]\n pub(crate) struct $atomic_type $(<$($generics)*>)? {\n v: UnsafeCell<$value_type>,\n }\n\n // Send is implicitly implemented for atomic integers, but not for atomic pointers.\n // SAFETY: any data races are prevented by atomic operations.\n unsafe impl $(<$($generics)*>)? Send for $atomic_type $(<$($generics)*>)? {}\n // SAFETY: any data races are prevented by atomic operations.\n unsafe impl $(<$($generics)*>)? Sync for $atomic_type $(<$($generics)*>)? {}\n\n #[cfg(any(test, not(portable_atomic_unsafe_assume_single_core)))]\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n #[inline]\n pub(crate) const fn new(v: $value_type) -> Self {\n Self { v: UnsafeCell::new(v) }\n }\n\n #[inline]\n pub(crate) fn is_lock_free() -> bool {\n Self::IS_ALWAYS_LOCK_FREE\n }\n pub(crate) const IS_ALWAYS_LOCK_FREE: bool = true;\n\n #[inline]\n pub(crate) const fn as_ptr(&self) -> *mut $value_type {\n self.v.get()\n }\n }\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n pub(crate) fn load(&self, order: Ordering) -> $value_type {\n crate::utils::assert_load_ordering(order);\n let src = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n let out;\n macro_rules! atomic_load {\n ($acquire:tt, $release:tt) => {\n asm!(\n $release, // fence\n concat!(\"l\", $size, \" {out}, 0({src})\"), // atomic { out = *src }\n $acquire, // fence\n src = in(reg) ptr_reg!(src),\n out = lateout(reg) out,\n options(nostack, preserves_flags),\n )\n };\n }\n match order {\n Ordering::Relaxed => atomic_load!(\"\", \"\"),\n Ordering::Acquire => atomic_load!(\"fence r, rw\", \"\"),\n Ordering::SeqCst => atomic_load!(\"fence r, rw\", \"fence rw, rw\"),\n _ => unreachable!(),\n }\n out\n }\n }\n\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n pub(crate) fn store(&self, val: $value_type, order: Ordering) {\n crate::utils::assert_store_ordering(order);\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe {\n macro_rules! atomic_store {\n ($acquire:tt, $release:tt) => {\n asm!(\n $release, // fence\n concat!(\"s\", $size, \" {val}, 0({dst})\"), // atomic { *dst = val }\n $acquire, // fence\n dst = in(reg) ptr_reg!(dst),\n val = in(reg) val,\n options(nostack, preserves_flags),\n )\n };\n }\n match order {\n Ordering::Relaxed => atomic_store!(\"\", \"\"),\n Ordering::Release => atomic_store!(\"\", \"fence rw, w\"),\n // https://github.com/llvm/llvm-project/commit/3ea8f2526541884e03d5bd4f4e46f4eb190990b6\n Ordering::SeqCst => atomic_store!(\"fence rw, rw\", \"fence rw, w\"),\n _ => unreachable!(),\n }\n }\n }\n }\n };\n}\n\nmacro_rules! atomic_base {\n (\n $([$($generics:tt)*])? $atomic_type:ident, $value_type:ty, $int_type:ty, $size:tt,\n $fetch_add:ident, $fetch_sub:ident\n ) => {\n atomic_load_store!($([$($generics)*])? $atomic_type, $value_type, $size);\n #[cfg(any(\n test,\n portable_atomic_force_amo,\n target_feature = \"zaamo\",\n portable_atomic_target_feature = \"zaamo\",\n ))]\n items!({\n // There is no amo{sub,nand,neg}.\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n #[inline]\n pub(crate) fn swap(&self, val: $value_type, order: Ordering) -> $value_type {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe { atomic_rmw_amo!(swap, dst, val, order, $size) }\n }\n #[inline]\n pub(crate) fn $fetch_add(&self, val: $int_type, order: Ordering) -> $value_type {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe { atomic_rmw_amo!(add, dst, val, order, $size) }\n }\n #[inline]\n pub(crate) fn $fetch_sub(&self, val: $int_type, order: Ordering) -> $value_type {\n self.$fetch_add(val.wrapping_neg(), order)\n }\n #[inline]\n pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $value_type {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe { atomic_rmw_amo!(and, dst, val, order, $size) }\n }\n #[inline]\n pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $value_type {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe { atomic_rmw_amo!(or, dst, val, order, $size) }\n }\n #[inline]\n pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $value_type {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe { atomic_rmw_amo!(xor, dst, val, order, $size) }\n }\n }\n #[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = \"critical-section\")))]\n impl_default_bit_opts!($atomic_type, $int_type);\n });\n };\n}\n\nmacro_rules! atomic_ptr {\n ($size:tt) => {\n atomic_base!([T] AtomicPtr, *mut T, usize, $size, fetch_byte_add, fetch_byte_sub);\n };\n}\n\nmacro_rules! atomic {\n ($atomic_type:ident, $value_type:ty, $size:tt, $max:tt, $min:tt) => {\n atomic_base!($atomic_type, $value_type, $value_type, $size, fetch_add, fetch_sub);\n #[cfg(any(\n test,\n portable_atomic_force_amo,\n target_feature = \"zaamo\",\n portable_atomic_target_feature = \"zaamo\",\n ))]\n items!({\n // There is no amo{sub,nand,neg}.\n impl $atomic_type {\n #[inline]\n pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type {\n let dst = self.v.get();\n let val: crate::utils::RegSize = !0;\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe { atomic_rmw_amo!(xor, dst, val, order, $size) }\n }\n #[inline]\n pub(crate) fn fetch_max(&self, val: $value_type, order: Ordering) -> $value_type {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe { atomic_rmw_amo!($max, dst, val, order, $size) }\n }\n #[inline]\n pub(crate) fn fetch_min(&self, val: $value_type, order: Ordering) -> $value_type {\n let dst = self.v.get();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n unsafe { atomic_rmw_amo!($min, dst, val, order, $size) }\n }\n }\n #[cfg(not(any(\n portable_atomic_unsafe_assume_single_core,\n feature = \"critical-section\",\n )))]\n items!({\n impl_default_no_fetch_ops!($atomic_type, $value_type);\n impl $atomic_type {\n #[inline]\n pub(crate) fn not(&self, order: Ordering) {\n self.fetch_not(order);\n }\n }\n });\n });\n };\n}\n\nmacro_rules! atomic_sub_word {\n ($atomic_type:ident, $value_type:ty, $size:tt, $max:tt, $min:tt) => {\n cfg_sel!({\n #[cfg(any(target_feature = \"zabha\", portable_atomic_target_feature = \"zabha\"))]\n {\n atomic!($atomic_type, $value_type, $size, $max, $min);\n }\n #[cfg(else)]\n {\n atomic_load_store!($atomic_type, $value_type, $size);\n #[cfg(any(\n test,\n portable_atomic_force_amo,\n target_feature = \"zaamo\",\n portable_atomic_target_feature = \"zaamo\",\n ))]\n items!({\n impl $atomic_type {\n #[inline]\n pub(crate) fn fetch_and(\n &self,\n val: $value_type,\n order: Ordering,\n ) -> $value_type {\n let dst = self.v.get();\n let (dst, shift, mut mask) =\n crate::utils::create_sub_word_mask_values(dst);\n mask = !sllw(mask, shift);\n let mut val = sllw(ZeroExtend::zero_extend(val), shift);\n val |= mask;\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n let out: u32 = unsafe { atomic_rmw_amo!(and, dst, val, order, \"w\") };\n srlw!(out, shift)\n }\n #[inline]\n pub(crate) fn fetch_or(\n &self,\n val: $value_type,\n order: Ordering,\n ) -> $value_type {\n let dst = self.v.get();\n let (dst, shift, _mask) =\n crate::utils::create_sub_word_mask_values(dst);\n let val = sllw(ZeroExtend::zero_extend(val), shift);\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n let out: u32 = unsafe { atomic_rmw_amo!(or, dst, val, order, \"w\") };\n srlw!(out, shift)\n }\n #[inline]\n pub(crate) fn fetch_xor(\n &self,\n val: $value_type,\n order: Ordering,\n ) -> $value_type {\n let dst = self.v.get();\n let (dst, shift, _mask) =\n crate::utils::create_sub_word_mask_values(dst);\n let val = sllw(ZeroExtend::zero_extend(val), shift);\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n let out: u32 = unsafe { atomic_rmw_amo!(xor, dst, val, order, \"w\") };\n srlw!(out, shift)\n }\n #[inline]\n pub(crate) fn fetch_not(&self, order: Ordering) -> $value_type {\n self.fetch_xor(!0, order)\n }\n }\n #[cfg(not(any(\n portable_atomic_unsafe_assume_single_core,\n feature = \"critical-section\",\n )))]\n items!({\n impl_default_bit_opts!($atomic_type, $value_type);\n impl $atomic_type {\n #[inline]\n pub(crate) fn and(&self, val: $value_type, order: Ordering) {\n self.fetch_and(val, order);\n }\n #[inline]\n pub(crate) fn or(&self, val: $value_type, order: Ordering) {\n self.fetch_or(val, order);\n }\n #[inline]\n pub(crate) fn xor(&self, val: $value_type, order: Ordering) {\n self.fetch_xor(val, order);\n }\n #[inline]\n pub(crate) fn not(&self, order: Ordering) {\n self.fetch_not(order);\n }\n }\n });\n });\n }\n });\n };\n}\n\natomic_sub_word!(AtomicI8, i8, \"b\", max, min);\natomic_sub_word!(AtomicU8, u8, \"b\", maxu, minu);\natomic_sub_word!(AtomicI16, i16, \"h\", max, min);\natomic_sub_word!(AtomicU16, u16, \"h\", maxu, minu);\natomic!(AtomicI32, i32, \"w\", max, min);\natomic!(AtomicU32, u32, \"w\", maxu, minu);\n#[cfg(target_arch = \"riscv64\")]\natomic!(AtomicI64, i64, \"d\", max, min);\n#[cfg(target_arch = \"riscv64\")]\natomic!(AtomicU64, u64, \"d\", maxu, minu);\n#[cfg(target_pointer_width = \"32\")]\natomic!(AtomicIsize, isize, \"w\", max, min);\n#[cfg(target_pointer_width = \"32\")]\natomic!(AtomicUsize, usize, \"w\", maxu, minu);\n#[cfg(target_pointer_width = \"32\")]\natomic_ptr!(\"w\");\n#[cfg(target_pointer_width = \"64\")]\natomic!(AtomicIsize, isize, \"d\", max, min);\n#[cfg(target_pointer_width = \"64\")]\natomic!(AtomicUsize, usize, \"d\", maxu, minu);\n#[cfg(target_pointer_width = \"64\")]\natomic_ptr!(\"d\");\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n test_atomic_ptr_load_store!();\n test_atomic_int_load_store!(i8);\n test_atomic_int_load_store!(u8);\n test_atomic_int_load_store!(i16);\n test_atomic_int_load_store!(u16);\n test_atomic_int_load_store!(i32);\n test_atomic_int_load_store!(u32);\n #[cfg(target_arch = \"riscv64\")]\n test_atomic_int_load_store!(i64);\n #[cfg(target_arch = \"riscv64\")]\n test_atomic_int_load_store!(u64);\n test_atomic_int_load_store!(isize);\n test_atomic_int_load_store!(usize);\n\n macro_rules! test_atomic_ptr_amo {\n () => {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod test_atomic_ptr_amo {\n use super::*;\n test_atomic_ptr_amo!(AtomicPtr);\n }\n };\n ($atomic_type:ty) => {\n #[allow(unused_imports)]\n use sptr::Strict as _;\n\n use crate::tests::helper; // for old rustc\n\n ::quickcheck::quickcheck! {\n fn quickcheck_swap(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.swap(y, order), x);\n assert_eq!(a.swap(x, order), y);\n }\n true\n }\n fn quickcheck_fetch_byte_add(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut::(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_byte_add(y.addr(), order), x);\n assert_eq!(\n a.load(Ordering::Relaxed).addr(), x.addr().wrapping_add(y.addr())\n );\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_byte_add(x.addr(), order), y);\n assert_eq!(\n a.load(Ordering::Relaxed).addr(), y.addr().wrapping_add(x.addr())\n );\n }\n true\n }\n fn quickcheck_fetch_byte_sub(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut::(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_byte_sub(y.addr(), order), x);\n assert_eq!(\n a.load(Ordering::Relaxed).addr(), x.addr().wrapping_sub(y.addr())\n );\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_byte_sub(x.addr(), order), y);\n assert_eq!(\n a.load(Ordering::Relaxed).addr(), y.addr().wrapping_sub(x.addr())\n );\n }\n true\n }\n fn quickcheck_fetch_and(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut::(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_and(y.addr(), order), x);\n assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() & y.addr());\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_and(x.addr(), order), y);\n assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() & x.addr());\n }\n true\n }\n fn quickcheck_fetch_or(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut::(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_or(y.addr(), order), x);\n assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() | y.addr());\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_or(x.addr(), order), y);\n assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() | x.addr());\n }\n true\n }\n fn quickcheck_fetch_xor(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut::(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_xor(y.addr(), order), x);\n assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() ^ y.addr());\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_xor(x.addr(), order), y);\n assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() ^ x.addr());\n }\n true\n }\n }\n };\n }\n macro_rules! test_atomic_int_amo {\n ($int_type:ident) => {\n paste::paste! {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod [] {\n use super::*;\n test_atomic_int_amo!([], $int_type);\n }\n }\n };\n ($atomic_type:ty, $int_type:ident) => {\n use crate::tests::helper;\n ::quickcheck::quickcheck! {\n fn quickcheck_swap(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.swap(y, order), x);\n assert_eq!(a.swap(x, order), y);\n }\n true\n }\n fn quickcheck_fetch_add(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_add(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x.wrapping_add(y));\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_add(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y.wrapping_add(x));\n }\n true\n }\n fn quickcheck_fetch_sub(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_sub(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x.wrapping_sub(y));\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_sub(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y.wrapping_sub(x));\n }\n true\n }\n fn quickcheck_fetch_and(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_and(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x & y);\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_and(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y & x);\n }\n true\n }\n fn quickcheck_fetch_or(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_or(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x | y);\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_or(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y | x);\n }\n true\n }\n fn quickcheck_fetch_xor(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_xor(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x ^ y);\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_xor(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y ^ x);\n }\n true\n }\n fn quickcheck_fetch_max(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_max(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), core::cmp::max(x, y));\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_max(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), core::cmp::max(y, x));\n }\n true\n }\n fn quickcheck_fetch_min(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_min(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), core::cmp::min(x, y));\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_min(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), core::cmp::min(y, x));\n }\n true\n }\n fn quickcheck_fetch_not(x: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_not(order), x);\n assert_eq!(a.load(Ordering::Relaxed), !x);\n assert_eq!(a.fetch_not(order), !x);\n assert_eq!(a.load(Ordering::Relaxed), x);\n }\n true\n }\n }\n };\n }\n macro_rules! test_atomic_int_amo_sub_word {\n ($int_type:ident) => {\n paste::paste! {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod [] {\n use super::*;\n #[cfg(any(target_feature = \"zabha\", portable_atomic_target_feature = \"zabha\"))]\n test_atomic_int_amo!([], $int_type);\n #[cfg(not(any(target_feature = \"zabha\", portable_atomic_target_feature = \"zabha\")))]\n test_atomic_int_amo_sub_word!([], $int_type);\n }\n }\n };\n ($atomic_type:ty, $int_type:ident) => {\n use crate::tests::helper::{self, *};\n ::quickcheck::quickcheck! {\n fn quickcheck_fetch_and(x: $int_type, y: $int_type) -> bool {\n let mut rng = fastrand::Rng::new();\n for &order in &helper::SWAP_ORDERINGS {\n for base in [0, !0] {\n let mut arr = Align16([\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n ]);\n let a_idx = rng.usize(3..=6);\n arr.0[a_idx] = <$atomic_type>::new(x);\n let a = &arr.0[a_idx];\n assert_eq!(a.fetch_and(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x & y);\n for i in 0..a_idx {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n for i in a_idx + 1..arr.0.len() {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n arr.0[a_idx] = <$atomic_type>::new(y);\n let a = &arr.0[a_idx];\n assert_eq!(a.fetch_and(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y & x);\n for i in 0..a_idx {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n for i in a_idx + 1..arr.0.len() {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n }\n }\n true\n }\n fn quickcheck_fetch_or(x: $int_type, y: $int_type) -> bool {\n let mut rng = fastrand::Rng::new();\n for &order in &helper::SWAP_ORDERINGS {\n for base in [0, !0] {\n let mut arr = Align16([\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n ]);\n let a_idx = rng.usize(3..=6);\n arr.0[a_idx] = <$atomic_type>::new(x);\n let a = &arr.0[a_idx];\n assert_eq!(a.fetch_or(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x | y);\n for i in 0..a_idx {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n for i in a_idx + 1..arr.0.len() {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n arr.0[a_idx] = <$atomic_type>::new(y);\n let a = &arr.0[a_idx];\n assert_eq!(a.fetch_or(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y | x);\n for i in 0..a_idx {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n for i in a_idx + 1..arr.0.len() {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n }\n }\n true\n }\n fn quickcheck_fetch_xor(x: $int_type, y: $int_type) -> bool {\n let mut rng = fastrand::Rng::new();\n for &order in &helper::SWAP_ORDERINGS {\n for base in [0, !0] {\n let mut arr = Align16([\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n ]);\n let a_idx = rng.usize(3..=6);\n arr.0[a_idx] = <$atomic_type>::new(x);\n let a = &arr.0[a_idx];\n assert_eq!(a.fetch_xor(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x ^ y);\n for i in 0..a_idx {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n for i in a_idx + 1..arr.0.len() {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n arr.0[a_idx] = <$atomic_type>::new(y);\n let a = &arr.0[a_idx];\n assert_eq!(a.fetch_xor(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y ^ x);\n for i in 0..a_idx {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n for i in a_idx + 1..arr.0.len() {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n }\n }\n true\n }\n fn quickcheck_fetch_not(x: $int_type) -> bool {\n let mut rng = fastrand::Rng::new();\n for &order in &helper::SWAP_ORDERINGS {\n for base in [0, !0] {\n let mut arr = Align16([\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n <$atomic_type>::new(base),\n ]);\n let a_idx = rng.usize(3..=6);\n arr.0[a_idx] = <$atomic_type>::new(x);\n let a = &arr.0[a_idx];\n assert_eq!(a.fetch_not(order), x);\n assert_eq!(a.load(Ordering::Relaxed), !x);\n assert_eq!(a.fetch_not(order), !x);\n assert_eq!(a.load(Ordering::Relaxed), x);\n for i in 0..a_idx {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n for i in a_idx + 1..arr.0.len() {\n assert_eq!(arr.0[i].load(Ordering::Relaxed), base, \"invalid value written\");\n }\n }\n }\n true\n }\n }\n };\n }\n test_atomic_ptr_amo!();\n test_atomic_int_amo_sub_word!(i8);\n test_atomic_int_amo_sub_word!(u8);\n test_atomic_int_amo_sub_word!(i16);\n test_atomic_int_amo_sub_word!(u16);\n test_atomic_int_amo!(i32);\n test_atomic_int_amo!(u32);\n #[cfg(target_arch = \"riscv64\")]\n test_atomic_int_amo!(i64);\n #[cfg(target_arch = \"riscv64\")]\n test_atomic_int_amo!(u64);\n test_atomic_int_amo!(isize);\n test_atomic_int_amo!(usize);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/imp/x86.rs", "sha256": "4c11976fccadc618d68d6aba4caf744d2859f60f63c6fb08f626f2a2561e6b52", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*\nAtomic operations implementation on x86/x86_64.\n\nThis module provides atomic operations not supported by LLVM or optimizes\ncases where LLVM code generation is not optimal.\n\nNote: On Miri and ThreadSanitizer which do not support inline assembly, we don't use\nthis module and use CAS loop instead.\n\nRefs:\n- x86 and amd64 instruction reference https://www.felixcloutier.com/x86\n\nSee tests/asm-test/asm/portable-atomic for generated assembly.\n*/\n\n#[cfg(not(portable_atomic_no_asm))]\nuse core::arch::asm;\nuse core::sync::atomic::Ordering;\n\n#[cfg(target_pointer_width = \"32\")]\nmacro_rules! ptr_modifier {\n () => {\n \":e\"\n };\n}\n#[cfg(target_pointer_width = \"64\")]\nmacro_rules! ptr_modifier {\n () => {\n \"\"\n };\n}\n\n/// An atomic fence.\n///\n/// # Panics\n///\n/// Panics if `order` is [`Relaxed`](Ordering::Relaxed).\n#[cfg_attr(doc, allow(dead_code))]\n#[inline]\n#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\npub fn fence(order: Ordering) {\n if order == Ordering::SeqCst {\n let p = core::cell::UnsafeCell::new(core::mem::MaybeUninit::::uninit());\n // SAFETY: raw pointer passed in is valid because we got it from a reference.\n unsafe {\n // Equivalent to `mfence`, but is up to 3.1x faster on Coffee Lake and up to 2.4x faster on Raptor Lake-H at least in simple cases.\n // - https://github.com/taiki-e/portable-atomic/pull/156\n // - LLVM uses `lock or` https://godbolt.org/z/vv6rjzfYd\n // - Windows uses `xchg` for x86_32 for MemoryBarrier https://learn.microsoft.com/en-us/windows/win32/api/winnt/nf-winnt-memorybarrier\n // - MSVC STL uses `lock inc` https://github.com/microsoft/STL/pull/740\n // - boost uses `lock or` https://github.com/boostorg/atomic/commit/559eba81af71386cedd99f170dc6101c6ad7bf22\n #[cfg(target_pointer_width = \"64\")]\n asm!(\n concat!(\"xchg qword ptr [{p\", ptr_modifier!(), \"}], {tmp}\"),\n p = in(reg) p.get(),\n tmp = out(reg) _,\n options(nostack, preserves_flags),\n );\n #[cfg(target_pointer_width = \"32\")]\n asm!(\n concat!(\"xchg dword ptr [{p\", ptr_modifier!(), \"}], {tmp:e}\"),\n p = in(reg) p.get(),\n tmp = out(reg) _,\n options(nostack, preserves_flags),\n );\n }\n } else {\n core::sync::atomic::fence(order);\n }\n}\n\ncfg_core_atomic!({\n use super::core_atomic::{\n AtomicI8, AtomicI16, AtomicI32, AtomicI64, AtomicIsize, AtomicPtr, AtomicU8, AtomicU16,\n AtomicU32, AtomicU64, AtomicUsize,\n };\n\n macro_rules! atomic_int {\n ($atomic_type:ident, $ptr_size:tt) => {\n impl $atomic_type {\n #[inline]\n pub(crate) fn not(&self, _order: Ordering) {\n let dst = self.as_ptr();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n //\n // https://www.felixcloutier.com/x86/not\n unsafe {\n // atomic RMW is always SeqCst.\n asm!(\n concat!(\"lock not \", $ptr_size, \" ptr [{dst\", ptr_modifier!(), \"}]\"),\n dst = in(reg) dst,\n options(nostack, preserves_flags),\n );\n }\n }\n #[inline]\n pub(crate) fn neg(&self, _order: Ordering) {\n let dst = self.as_ptr();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n //\n // https://www.felixcloutier.com/x86/neg\n unsafe {\n // atomic RMW is always SeqCst.\n asm!(\n concat!(\"lock neg \", $ptr_size, \" ptr [{dst\", ptr_modifier!(), \"}]\"),\n dst = in(reg) dst,\n // Do not use `preserves_flags` because NEG modifies the CF, OF, SF, ZF, AF, and PF flag.\n options(nostack),\n );\n }\n }\n }\n };\n }\n\n atomic_int!(AtomicI8, \"byte\");\n atomic_int!(AtomicU8, \"byte\");\n atomic_int!(AtomicI16, \"word\");\n atomic_int!(AtomicU16, \"word\");\n atomic_int!(AtomicI32, \"dword\");\n atomic_int!(AtomicU32, \"dword\");\n #[cfg(target_arch = \"x86_64\")]\n atomic_int!(AtomicI64, \"qword\");\n #[cfg(target_arch = \"x86_64\")]\n atomic_int!(AtomicU64, \"qword\");\n #[cfg(target_pointer_width = \"32\")]\n atomic_int!(AtomicIsize, \"dword\");\n #[cfg(target_pointer_width = \"32\")]\n atomic_int!(AtomicUsize, \"dword\");\n #[cfg(target_pointer_width = \"64\")]\n atomic_int!(AtomicIsize, \"qword\");\n #[cfg(target_pointer_width = \"64\")]\n atomic_int!(AtomicUsize, \"qword\");\n\n #[cfg(target_arch = \"x86\")]\n impl AtomicI64 {\n #[inline]\n pub(crate) fn not(&self, order: Ordering) {\n self.fetch_not(order);\n }\n #[inline]\n pub(crate) fn neg(&self, order: Ordering) {\n self.fetch_neg(order);\n }\n }\n #[cfg(target_arch = \"x86\")]\n impl AtomicU64 {\n #[inline]\n pub(crate) fn not(&self, order: Ordering) {\n self.fetch_not(order);\n }\n #[inline]\n pub(crate) fn neg(&self, order: Ordering) {\n self.fetch_neg(order);\n }\n }\n\n macro_rules! atomic_bit_opts {\n (\n $([$($generics:tt)*])? $atomic_type:ident, $int_type:ident, $val_modifier:tt,\n $ptr_size:tt\n ) => {\n // LLVM 14 and older don't support generating `lock bt{s,r,c}`.\n // LLVM 15 only supports generating `lock bt{s,r,c}` for immediate bit offsets.\n // LLVM 16+ can generate `lock bt{s,r,c}` for both immediate and register bit offsets.\n // https://godbolt.org/z/TGhr5z4ds\n // So, use fetch_* based implementations on LLVM 16+, otherwise use asm based implementations.\n #[cfg(not(portable_atomic_pre_llvm_16))]\n impl_default_bit_opts!($atomic_type, $int_type);\n #[cfg(portable_atomic_pre_llvm_16)]\n impl $(<$($generics)*>)? $atomic_type $(<$($generics)*>)? {\n // `::BITS` requires Rust 1.53\n const BITS: u32 = (core::mem::size_of::<$int_type>() * 8) as u32;\n #[inline]\n pub(crate) fn bit_set(&self, bit: u32, _order: Ordering) -> bool {\n let dst = self.as_ptr();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n // the masking by the bit size of the type ensures that we do not shift\n // out of bounds.\n //\n // https://www.felixcloutier.com/x86/bts\n unsafe {\n let r: u8;\n // atomic RMW is always SeqCst.\n asm!(\n concat!(\"lock bts \", $ptr_size, \" ptr [{dst\", ptr_modifier!(), \"}], {bit\", $val_modifier, \"}\"),\n \"setb {r}\",\n dst = in(reg) dst,\n bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,\n r = out(reg_byte) r,\n // Do not use `preserves_flags` because BTS modifies the CF flag.\n options(nostack),\n );\n crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test\n r != 0\n }\n }\n #[inline]\n pub(crate) fn bit_clear(&self, bit: u32, _order: Ordering) -> bool {\n let dst = self.as_ptr();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n // the masking by the bit size of the type ensures that we do not shift\n // out of bounds.\n //\n // https://www.felixcloutier.com/x86/btr\n unsafe {\n let r: u8;\n // atomic RMW is always SeqCst.\n asm!(\n concat!(\"lock btr \", $ptr_size, \" ptr [{dst\", ptr_modifier!(), \"}], {bit\", $val_modifier, \"}\"),\n \"setb {r}\",\n dst = in(reg) dst,\n bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,\n r = out(reg_byte) r,\n // Do not use `preserves_flags` because BTR modifies the CF flag.\n options(nostack),\n );\n crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test\n r != 0\n }\n }\n #[inline]\n pub(crate) fn bit_toggle(&self, bit: u32, _order: Ordering) -> bool {\n let dst = self.as_ptr();\n // SAFETY: any data races are prevented by atomic intrinsics and the raw\n // pointer passed in is valid because we got it from a reference.\n // the masking by the bit size of the type ensures that we do not shift\n // out of bounds.\n //\n // https://www.felixcloutier.com/x86/btc\n unsafe {\n let r: u8;\n // atomic RMW is always SeqCst.\n asm!(\n concat!(\"lock btc \", $ptr_size, \" ptr [{dst\", ptr_modifier!(), \"}], {bit\", $val_modifier, \"}\"),\n \"setb {r}\",\n dst = in(reg) dst,\n bit = in(reg) (bit & (Self::BITS - 1)) as $int_type,\n r = out(reg_byte) r,\n // Do not use `preserves_flags` because BTC modifies the CF flag.\n options(nostack),\n );\n crate::utils::assert_unchecked(r == 0 || r == 1); // may help remove extra test\n r != 0\n }\n }\n }\n };\n }\n\n impl_default_bit_opts!(AtomicI8, i8);\n impl_default_bit_opts!(AtomicU8, u8);\n atomic_bit_opts!(AtomicI16, i16, \":x\", \"word\");\n atomic_bit_opts!(AtomicU16, u16, \":x\", \"word\");\n atomic_bit_opts!(AtomicI32, i32, \":e\", \"dword\");\n atomic_bit_opts!(AtomicU32, u32, \":e\", \"dword\");\n #[cfg(target_arch = \"x86_64\")]\n atomic_bit_opts!(AtomicI64, i64, \"\", \"qword\");\n #[cfg(target_arch = \"x86_64\")]\n atomic_bit_opts!(AtomicU64, u64, \"\", \"qword\");\n #[cfg(target_arch = \"x86\")]\n impl_default_bit_opts!(AtomicI64, i64);\n #[cfg(target_arch = \"x86\")]\n impl_default_bit_opts!(AtomicU64, u64);\n #[cfg(target_pointer_width = \"32\")]\n atomic_bit_opts!(AtomicIsize, isize, \":e\", \"dword\");\n #[cfg(target_pointer_width = \"32\")]\n atomic_bit_opts!(AtomicUsize, usize, \":e\", \"dword\");\n #[cfg(target_pointer_width = \"32\")]\n atomic_bit_opts!([T] AtomicPtr, usize, \":e\", \"dword\");\n #[cfg(target_pointer_width = \"64\")]\n atomic_bit_opts!(AtomicIsize, isize, \"\", \"qword\");\n #[cfg(target_pointer_width = \"64\")]\n atomic_bit_opts!(AtomicUsize, usize, \"\", \"qword\");\n #[cfg(target_pointer_width = \"64\")]\n atomic_bit_opts!([T] AtomicPtr, usize, \"\", \"qword\");\n});\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/lib.rs", "sha256": "8fac4257f626a2669a44dd74a966677b368647fd883e32d8db79c57a06af6895", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*!\n\n\n\nPortable atomic types including support for 128-bit atomics, atomic float, etc.\n\n- Provide all atomic integer types (`Atomic{I,U}{8,16,32,64}`) for all targets that can use atomic CAS. (i.e., all targets that can use `std`, and most no-std targets)\n- Provide `AtomicI128` and `AtomicU128`.\n- Provide `AtomicF32` and `AtomicF64`. ([optional, requires the `float` feature](#optional-features-float))\n- Provide `AtomicF16` and `AtomicF128` for [unstable `f16` and `f128`](https://github.com/rust-lang/rust/issues/116909). ([optional, requires the `float` feature and unstable cfgs](#optional-features-float))\n- Provide atomic load/store for targets where atomic is not available at all in the standard library. (RISC-V without A-extension, MSP430, AVR)\n- Provide atomic CAS for targets where atomic CAS is not available in the standard library. (thumbv6m, pre-v6 Arm, RISC-V without A-extension, MSP430, AVR, Xtensa, etc.) (always enabled for MSP430 and AVR, [optional](#optional-features-critical-section) otherwise)\n- Make features that require newer compilers, such as [`fetch_{max,min}`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.fetch_max), [`fetch_update`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.fetch_update), [`as_ptr`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.as_ptr), [`from_ptr`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicUsize.html#method.from_ptr), [`AtomicBool::fetch_not`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicBool.html#method.fetch_not), [`AtomicPtr::fetch_*`](https://doc.rust-lang.org/std/sync/atomic/struct.AtomicPtr.html#method.fetch_and), and [stronger CAS failure ordering](https://github.com/rust-lang/rust/pull/98383) available on Rust 1.34+.\n- Provide workaround for bugs in the standard library's atomic-related APIs, such as [rust-lang/compiler-builtins#1234], [rust-lang/rust#100650], `fence`/`compiler_fence` on MSP430 that cause LLVM error, etc.\n\n\n\nportable-atomic version of `std::sync::Arc` and `std::task::Wake` is provided by the [portable-atomic-util] crate.\n\n## Usage\n\nAdd this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nportable-atomic = \"1\"\n```\n\nThe default features are mainly for users who use atomics larger than the pointer width.\nIf you don't need them, disabling the default features may reduce code size and compile time slightly.\n\n```toml\n[dependencies]\nportable-atomic = { version = \"1\", default-features = false }\n```\n\nIf your crate supports no-std environment and requires atomic CAS, enabling the `require-cas` feature will allow the portable-atomic to display a [helpful error message](https://github.com/taiki-e/portable-atomic/pull/100) to users on targets requiring additional action on the user side to provide atomic CAS.\n\n```toml\n[dependencies]\nportable-atomic = { version = \"1.3\", default-features = false, features = [\"require-cas\"] }\n```\n\n(Since 1.8, portable-atomic can display a [helpful error message](https://github.com/taiki-e/portable-atomic/pull/181) even without the `require-cas` feature when the rustc version is 1.78+. However, the `require-cas` feature also allows rejecting builds at an earlier stage, we recommend enabling it unless enabling it causes [problems](https://github.com/matklad/once_cell/pull/267).)\n\n## 128-bit atomics support\n\nNative 128-bit atomic operations are available on x86_64 (Rust 1.59+), AArch64 (Rust 1.59+), riscv64 (Rust 1.59+), Arm64EC (Rust 1.84+), s390x (Rust 1.84+), and powerpc64 (Rust 1.95+), otherwise the fallback implementation is used.\n\nOn x86_64, even if `cmpxchg16b` is not available at compile-time (Note: `cmpxchg16b` target feature is enabled by default only on Apple, Windows (except Windows 7), and Fuchsia targets), run-time detection checks whether `cmpxchg16b` is available. If `cmpxchg16b` is not available at either compile-time or run-time detection, the fallback implementation is used. See also [`portable_atomic_no_outline_atomics`](#optional-cfg-no-outline-atomics) cfg.\n\nThey are usually implemented using inline assembly, and when using Miri or ThreadSanitizer that do not support inline assembly, core intrinsics are used instead of inline assembly if possible.\n\nSee the [`atomic128` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/atomic128/README.md) for details.\n\n## Optional features/cfgs\n\nportable-atomic provides features and cfgs to allow enabling specific APIs and customizing its behavior.\n\nSome options have both a feature and a cfg. When both exist, it indicates that the feature does not follow Cargo's recommendation that [features should be additive](https://doc.rust-lang.org/nightly/cargo/reference/features.html#feature-unification). Therefore, the maintainer's recommendation is to use cfg instead of feature. However, in the embedded ecosystem, it is very common to use features in such places, so these options provide both so you can choose based on your preference.\n\n
\nHow to enable cfg (click to show)\n\nOne of the ways to enable cfg is to set [rustflags in the cargo config](https://doc.rust-lang.org/cargo/reference/config.html#targettriplerustflags):\n\n```toml\n# .cargo/config.toml\n[target.]\nrustflags = [\"--cfg\", \"portable_atomic_unsafe_assume_single_core\"]\n```\n\nOr set environment variable:\n\n```sh\nRUSTFLAGS=\"--cfg portable_atomic_unsafe_assume_single_core\" cargo ...\n```\n\n
\n\n- **`fallback` feature** *(enabled by default)*
\n Enable fallback implementations.\n\n This enables atomic types with larger than the width supported by atomic instructions available on the current target. If the current target [supports 128-bit atomics](#128-bit-atomics-support), this is no-op.\n\n This uses fallback implementation that using global locks by default. The following features/cfgs change this behavior:\n - [`unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg](#optional-features-unsafe-assume-single-core): Use fallback implementations that disabling interrupts instead of using global locks.\n - If your target is single-core and calling interrupt disable instructions is safe, this is a safer and more efficient option.\n - [`unsafe-assume-privileged` feature / `portable_atomic_unsafe_assume_privileged` cfg](#optional-features-unsafe-assume-privileged): Use fallback implementations that using global locks with disabling interrupts.\n - If your target is multi-core and calling interrupt disable instructions is safe, this is a safer option.\n\n- **`float` feature**
\n Provide `AtomicF{32,64}`.\n\n If you want atomic types for unstable float types ([`f16` and `f128`](https://github.com/rust-lang/rust/issues/116909)), enable unstable cfg (`portable_atomic_unstable_f16` cfg for `AtomicF16`, `portable_atomic_unstable_f128` cfg for `AtomicF128`, [there is no possibility that both feature and cfg will be provided for unstable options.](https://github.com/taiki-e/portable-atomic/pull/200#issuecomment-2682252991)).\n\n
\n\n> **ⓘ Note**\n>\n> - Atomic float's `fetch_{add,sub,min,max}` are usually implemented using CAS loops, which can be slower than equivalent operations of atomic integers. As an exception, AArch64 with FEAT_LSFE and GPU targets have atomic float instructions and we use them on AArch64 when `lsfe` target feature is available at compile-time. We [plan to use atomic float instructions for GPU targets as well in the future.](https://github.com/taiki-e/portable-atomic/issues/34)\n> - Unstable cfgs are outside of the normal semver guarantees and minor or patch versions of portable-atomic may make breaking changes to them at any time.\n\n
\n\n- **`std` feature**
\n Use `std`.\n\n- **`require-cas` feature**
\n Emit compile error if atomic CAS is not available. See [Usage](#usage) section for usage of this feature.\n\n- **`serde` feature**
\n Implement `serde::{Serialize,Deserialize}` for atomic types.\n\n Note:\n - The MSRV when this feature is enabled depends on the MSRV of [serde].\n\n- **`critical-section` feature**
\n Use [critical-section] to provide atomic CAS for targets where atomic CAS is not available in the standard library.\n\n `critical-section` support is useful to get atomic CAS when the [`unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)](#optional-features-unsafe-assume-single-core) can't be used,\n such as multi-core targets, unprivileged code running under some RTOS, or environments where disabling interrupts\n needs extra care due to e.g. real-time requirements.\n\n
\n\n> **ⓘ Note**\n>\n> - When enabling this feature, you should provide a suitable critical section implementation for the current target, see the [critical-section] documentation for details on how to do so.\n> - With this feature, critical sections are taken for all atomic operations, while with `unsafe-assume-single-core` feature [some operations](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/interrupt/README.md#no-disable-interrupts) don't require disabling interrupts. Therefore, for better performance, if all the `critical-section` implementation for your target does is disable interrupts, prefer using `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg) instead.\n> - It is usually **discouraged** to always enable this feature in libraries that depend on `portable-atomic`.\n>\n> Enabling this feature will prevent the end user from having the chance to take advantage of other (potentially) efficient implementations (implementations provided by `unsafe-assume-single-core` feature mentioned above, implementation proposed in [#60], etc.). Also, targets that are currently unsupported may be supported in the future.\n>\n> The recommended approach for libraries is to leave it up to the end user whether or not to enable this feature. (However, it may make sense to enable this feature by default for libraries specific to a platform where other implementations are known not to work.)\n>\n> See also [this comment](https://github.com/matklad/once_cell/issues/264#issuecomment-2352654806).\n>\n> As an example, the end-user's `Cargo.toml` that uses a crate that provides a critical-section implementation and a crate that depends on portable-atomic as an option would be expected to look like this:\n>\n> ```toml\n> [dependencies]\n> portable-atomic = { version = \"1\", default-features = false, features = [\"critical-section\"] }\n> crate-provides-critical-section-impl = \"...\"\n> crate-uses-portable-atomic-as-feature = { version = \"...\", features = [\"portable-atomic\"] }\n> ```\n>\n> - Enabling both this feature and `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg) will result in a compile error.\n> - Enabling both this feature and `unsafe-assume-privileged` feature (or `portable_atomic_unsafe_assume_privileged` cfg) will result in a compile error.\n> - The MSRV when this feature is enabled depends on the MSRV of [critical-section].\n\n
\n\n- **`unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg**
\n Assume that the target is single-core and privileged instructions required to disable interrupts are available.\n\n - When this feature/cfg is enabled, this crate provides atomic CAS for targets where atomic CAS is not available in the standard library by disabling interrupts.\n - When both this feature/cfg and enabled-by-default `fallback` feature is enabled, this crate provides atomic types with larger than the width supported by native instructions by disabling interrupts.\n\n
\n\n> **⚠ Warning**\n>\n> This feature/cfg is `unsafe`, and note the following safety requirements:\n> - Enabling this feature/cfg for multi-core systems is always **unsound**.\n>\n> - This uses privileged instructions to disable interrupts, so it usually doesn't work on unprivileged mode.\n>\n> Enabling this feature/cfg in an environment where privileged instructions are not available, or if the instructions used are not sufficient to disable interrupts in the system, it is also usually considered **unsound**, although the details are system-dependent.\n>\n> The following are known cases:\n> - On Arm (except for M-Profile architectures), this disables only IRQs by default. For many systems (e.g., GBA) this is enough. If the system need to disable both IRQs and FIQs, you need to enable the `disable-fiq` feature (or `portable_atomic_disable_fiq` cfg) together.\n> - On RISC-V, this generates code for machine-mode (M-mode) by default. If you enable the `s-mode` feature (or `portable_atomic_s_mode` cfg) together, this generates code for supervisor-mode (S-mode). In particular, `qemu-system-riscv*` uses [OpenSBI](https://github.com/riscv-software-src/opensbi) as the default firmware.\n\n
\n\nConsider using the [`unsafe-assume-privileged` feature (or `portable_atomic_unsafe_assume_privileged` cfg)](#optional-features-unsafe-assume-privileged) for multi-core systems with atomic CAS.\n\nConsider using the [`critical-section` feature](#optional-features-critical-section) for systems that cannot use this feature/cfg.\n\nSee also the [`interrupt` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/interrupt/README.md).\n\n
\n\n> **ⓘ Note**\n>\n> - It is **very strongly discouraged** to enable this feature/cfg in libraries that depend on `portable-atomic`.\n>\n> The recommended approach for libraries is to leave it up to the end user whether or not to enable this feature/cfg. (However, it may make sense to enable this feature/cfg by default for libraries specific to a platform where it is guaranteed to always be sound, for example in a hardware abstraction layer targeting a single-core chip.)\n> - Enabling this feature/cfg for unsupported architectures will result in a compile error.\n> - Arm, RISC-V, and Xtensa are currently supported. (Since all MSP430 and AVR are single-core, we always provide atomic CAS for them without this feature/cfg.)\n> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target is not supported yet.\n> - Enabling this feature/cfg for targets where privileged instructions are obviously unavailable (e.g., Linux) will result in a compile error.\n> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target supports privileged instructions but the build rejected.\n> - Enabling both this feature/cfg and `critical-section` feature will result in a compile error.\n> - When both this feature/cfg and `unsafe-assume-privileged` feature (or `portable_atomic_unsafe_assume_privileged` cfg) are enabled, this feature/cfg is preferred.\n\n
\n\n- **`unsafe-assume-privileged` feature / `portable_atomic_unsafe_assume_privileged` cfg**
\n Similar to `unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg, but only assumes about availability of privileged instructions required to disable interrupts.\n\n - When both this feature/cfg and enabled-by-default `fallback` feature is enabled, this crate provides atomic types with larger than the width supported by native instructions by using global locks with disabling interrupts.\n\n
\n\n> **⚠ Warning**\n>\n> This feature/cfg is `unsafe`, and except for being sound in multi-core systems, this has the same safety requirements as [`unsafe-assume-single-core` feature / `portable_atomic_unsafe_assume_single_core` cfg](#optional-features-unsafe-assume-single-core).\n\n
\n\n
\n\n> **ⓘ Note**\n>\n> - It is **very strongly discouraged** to enable this feature/cfg in libraries that depend on `portable-atomic`.\n>\n> The recommended approach for libraries is to leave it up to the end user whether or not to enable this feature/cfg. (However, it may make sense to enable this feature/cfg by default for libraries specific to a platform where it is guaranteed to always be sound, for example in a hardware abstraction layer.)\n> - Enabling this feature/cfg for unsupported targets will result in a compile error.\n> - This requires atomic CAS (`cfg(target_has_atomic = \"ptr\")` or `cfg_no_atomic_cas!`).\n> - Arm, RISC-V, and Xtensa are currently supported.\n> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target is not supported yet.\n> - Enabling this feature/cfg for targets where privileged instructions are obviously unavailable (e.g., Linux) will result in a compile error.\n> - Feel free to [submit an issue](https://github.com/taiki-e/portable-atomic/issues/new) if your target supports privileged instructions but the build rejected.\n> - Enabling both this feature/cfg and `critical-section` feature will result in a compile error.\n> - When both this feature/cfg and `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg) are enabled, `unsafe-assume-single-core` is preferred.\n\n
\n\n- **`portable_atomic_no_outline_atomics` cfg**
\n Disable dynamic dispatching by run-time CPU feature detection.\n\n Dynamic dispatching by run-time CPU feature detection allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE/FEAT_LSE2 (AArch64).\n\n See also the [`atomic128` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/atomic128/README.md).\n\n
\n\n> **ⓘ Note**\n>\n> - If the required target features are enabled at compile-time, dynamic dispatching is automatically disabled and the atomic operations are inlined.\n> - This is compatible with no-std (as with all features except `std`).\n> - On some targets, run-time detection is disabled by default mainly for compatibility with incomplete build environments or support for it is experimental, and can be enabled by `portable_atomic_outline_atomics` cfg. (When both cfg are enabled, `*_no_*` cfg is preferred.)\n> - Some AArch64 targets enable LLVM's `outline-atomics` target feature by default, so if you set this cfg, you may want to disable that as well. (However, portable-atomic's outline-atomics does not depend on the compiler-rt symbols, so even if you need to disable LLVM's outline-atomics, you may not need to disable portable-atomic's outline-atomics.)\n> - Dynamic detection is currently only supported in x86_64, AArch64, Arm, RISC-V, Arm64EC, and powerpc64. Enabling this cfg for unsupported architectures will be ignored.\n\n
\n\n## Related Projects\n\n- [portable-atomic-util]: Synchronization primitives built with portable-atomic. This provides portable-atomic version of `std::sync::Arc` and `std::task::Wake`.\n- [atomic-maybe-uninit]: Atomic operations on potentially uninitialized integers.\n- [atomic-memcpy]: Byte-wise atomic memcpy.\n- [asmtest]: A library for tracking generated assemblies.\n\n[#60]: https://github.com/taiki-e/portable-atomic/issues/60\n[asmtest]: https://github.com/taiki-e/asmtest\n[atomic-maybe-uninit]: https://github.com/taiki-e/atomic-maybe-uninit\n[atomic-memcpy]: https://github.com/taiki-e/atomic-memcpy\n[critical-section]: https://github.com/rust-embedded/critical-section\n[portable-atomic-util]: https://github.com/taiki-e/portable-atomic-util\n[rust-lang/compiler-builtins#1234]: https://github.com/rust-lang/compiler-builtins/pull/1234\n[rust-lang/rust#100650]: https://github.com/rust-lang/rust/issues/100650\n[serde]: https://github.com/serde-rs/serde\n\n\n*/\n\n#![no_std]\n#![doc(test(\n no_crate_inject,\n attr(\n deny(warnings, rust_2018_idioms, single_use_lifetimes),\n allow(dead_code, unused_variables)\n )\n))]\n#![cfg_attr(not(portable_atomic_no_unsafe_op_in_unsafe_fn), warn(unsafe_op_in_unsafe_fn))] // unsafe_op_in_unsafe_fn requires Rust 1.52\n#![cfg_attr(portable_atomic_no_unsafe_op_in_unsafe_fn, allow(unused_unsafe))]\n#![warn(\n // Lints that may help when writing public library.\n missing_debug_implementations,\n // missing_docs,\n clippy::alloc_instead_of_core,\n clippy::exhaustive_enums,\n clippy::exhaustive_structs,\n clippy::impl_trait_in_params,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::missing_inline_in_public_items,\n // Code outside of cfg(feature = \"float\") shouldn't use float.\n clippy::float_arithmetic,\n)]\n#![cfg_attr(not(portable_atomic_no_asm), warn(missing_docs))] // module-level #![allow(missing_docs)] doesn't work for macros on old rustc\n#![cfg_attr(portable_atomic_no_strict_provenance, allow(unstable_name_collisions))]\n#![allow(\n clippy::inline_always,\n clippy::manual_assert_eq,\n clippy::unreadable_literal,\n clippy::used_underscore_items\n)]\n// asm_experimental_arch\n// AVR, MSP430, and Xtensa are tier 3 platforms and require nightly anyway.\n// On tier 2 platforms (currently N/A), we use cfg set by build script to\n// determine whether this feature is available or not.\n#![cfg_attr(\n all(\n not(portable_atomic_no_asm),\n any(\n all(target_arch = \"avr\", not(feature = \"critical-section\")),\n target_arch = \"msp430\",\n all(\n target_arch = \"xtensa\",\n any(\n portable_atomic_unsafe_assume_single_core,\n portable_atomic_unsafe_assume_privileged,\n ),\n ),\n ),\n ),\n feature(asm_experimental_arch)\n)]\n// f16/f128\n// cfg is unstable and explicitly enabled by the user\n#![cfg_attr(\n any(portable_atomic_unstable_f16, portable_atomic_unstable_f128),\n allow(unused_features)\n)]\n#![cfg_attr(all(portable_atomic_unstable_f16, feature = \"float\"), feature(f16))]\n#![cfg_attr(all(portable_atomic_unstable_f128, feature = \"float\"), feature(f128))]\n// Old nightly only\n// These features are already stabilized or have already been removed from compilers,\n// and can safely be enabled for old nightly as long as version detection works.\n// - cfg(target_has_atomic)\n// - asm! on AArch64, Arm, RISC-V, x86, x86_64, Arm64EC, s390x, PowerPC64\n// - llvm_asm! on AVR (tier 3) and MSP430 (tier 3)\n// - #[instruction_set] on non-Linux/Android pre-v6 Arm (tier 3)\n// This also helps us test that our assembly code works with the minimum external\n// LLVM version of the first rustc version that inline assembly stabilized.\n#![cfg_attr(portable_atomic_unstable_cfg_target_has_atomic, feature(cfg_target_has_atomic))]\n#![cfg_attr(\n all(\n portable_atomic_unstable_asm,\n any(\n target_arch = \"aarch64\",\n target_arch = \"arm\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"x86\",\n target_arch = \"x86_64\",\n ),\n ),\n feature(asm)\n)]\n#![cfg_attr(\n all(\n portable_atomic_unstable_asm_experimental_arch,\n any(target_arch = \"arm64ec\", target_arch = \"s390x\", target_arch = \"powerpc64\"),\n ),\n feature(asm_experimental_arch)\n)]\n#![cfg_attr(\n all(any(target_arch = \"avr\", target_arch = \"msp430\"), portable_atomic_no_asm),\n feature(llvm_asm)\n)]\n#![cfg_attr(\n all(\n target_arch = \"arm\",\n portable_atomic_unstable_isa_attribute,\n any(portable_atomic_unsafe_assume_single_core, portable_atomic_unsafe_assume_privileged),\n not(any(target_feature = \"v7\", portable_atomic_target_feature = \"v7\")),\n not(any(target_feature = \"mclass\", portable_atomic_target_feature = \"mclass\")),\n ),\n feature(isa_attribute)\n)]\n// Miri and/or ThreadSanitizer only\n// They do not support inline assembly, so we need to use unstable features instead.\n// Since they require nightly compilers anyway, we can use the unstable features.\n// This is not an ideal situation, but it is still better than always using lock-based\n// fallback and causing memory ordering problems to be missed by these checkers.\n#![cfg_attr(\n all(\n any(\n target_arch = \"aarch64\",\n target_arch = \"arm64ec\",\n target_arch = \"powerpc64\",\n target_arch = \"s390x\",\n ),\n any(miri, portable_atomic_sanitize_thread),\n ),\n allow(internal_features)\n)]\n#![cfg_attr(\n all(\n any(\n target_arch = \"aarch64\",\n target_arch = \"arm64ec\",\n target_arch = \"powerpc64\",\n target_arch = \"s390x\",\n ),\n portable_atomic_atomic_intrinsics,\n any(miri, portable_atomic_sanitize_thread),\n ),\n feature(core_intrinsics)\n)]\n// docs.rs only (cfg is enabled by docs.rs, not build script)\n#![cfg_attr(docsrs, feature(doc_cfg))]\n#![cfg_attr(docsrs, doc(auto_cfg = false))]\n#![cfg_attr(\n all(\n portable_atomic_no_atomic_load_store,\n not(any(\n target_arch = \"avr\",\n target_arch = \"bpf\",\n target_arch = \"msp430\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )),\n ),\n allow(unused_imports, unused_macros, clippy::unused_trait_names)\n)]\n\n#[cfg(any(test, feature = \"std\"))]\nextern crate std;\n\n#[macro_use]\nmod cfgs;\n#[cfg(target_pointer_width = \"16\")]\npub use self::{cfg_has_atomic_16 as cfg_has_atomic_ptr, cfg_no_atomic_16 as cfg_no_atomic_ptr};\n#[cfg(target_pointer_width = \"32\")]\npub use self::{cfg_has_atomic_32 as cfg_has_atomic_ptr, cfg_no_atomic_32 as cfg_no_atomic_ptr};\n#[cfg(target_pointer_width = \"64\")]\npub use self::{cfg_has_atomic_64 as cfg_has_atomic_ptr, cfg_no_atomic_64 as cfg_no_atomic_ptr};\n#[cfg(target_pointer_width = \"128\")]\npub use self::{cfg_has_atomic_128 as cfg_has_atomic_ptr, cfg_no_atomic_128 as cfg_no_atomic_ptr};\n\n// There are currently no 128-bit or higher builtin targets.\n// (Although some of our generic code is written with the future\n// addition of 128-bit targets in mind.)\n// Note that Rust (and C99) pointers must be at least 16-bit (i.e., 8-bit targets are impossible): https://github.com/rust-lang/rust/pull/49305\n#[cfg(not(any(\n target_pointer_width = \"16\",\n target_pointer_width = \"32\",\n target_pointer_width = \"64\",\n)))]\ncompile_error!(\n \"portable-atomic currently only supports targets with {16,32,64}-bit pointer width; \\\n if you need support for others, \\\n please submit an issue at \"\n);\n\n// Reject unsupported architectures.\n#[cfg(portable_atomic_unsafe_assume_single_core)]\n#[cfg(not(any(\n target_arch = \"arm\",\n target_arch = \"avr\",\n target_arch = \"msp430\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"xtensa\",\n)))]\ncompile_error!(\n \"`portable_atomic_unsafe_assume_single_core` cfg (`unsafe-assume-single-core` feature) \\\n is not supported yet on this architecture;\\n\\\n if you need unsafe-assume-{single-core,privileged} support for this target,\\n\\\n please submit an issue at \"\n);\n// unsafe-assume-single-core is accepted on AVR/MSP430, but\n// unsafe-assume-privileged on them is really useless on them since they are\n// always single-core, so rejected here.\n#[cfg(portable_atomic_unsafe_assume_privileged)]\n#[cfg(not(any(\n target_arch = \"arm\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"xtensa\",\n)))]\ncompile_error!(\n \"`portable_atomic_unsafe_assume_privileged` cfg (`unsafe-assume-privileged` feature) \\\n is not supported yet on this architecture;\\n\\\n if you need unsafe-assume-{single-core,privileged} support for this target,\\n\\\n please submit an issue at \"\n);\n// unsafe-assume-privileged requires CAS.\n#[cfg(portable_atomic_unsafe_assume_privileged)]\ncfg_no_atomic_cas! {\n compile_error!(\n \"`portable_atomic_unsafe_assume_privileged` cfg (`unsafe-assume-privileged` feature) \\\n requires atomic CAS\"\n );\n}\n// Reject targets where privileged instructions are obviously unavailable.\n// TODO: Some embedded OSes should probably be accepted here.\n#[cfg(any(portable_atomic_unsafe_assume_single_core, portable_atomic_unsafe_assume_privileged))]\n#[cfg(any(\n target_arch = \"arm\",\n target_arch = \"avr\",\n target_arch = \"msp430\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"xtensa\",\n))]\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(all(not(portable_atomic_no_atomic_cas), not(target_os = \"none\")))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(all(target_has_atomic = \"ptr\", not(target_os = \"none\")))\n)]\ncompile_error!(\n \"`portable_atomic_unsafe_assume_{single_core,privileged}` cfg (`unsafe-assume-{single-core,privileged}` feature) \\\n is not compatible with target where privileged instructions are obviously unavailable;\\n\\\n if you need unsafe-assume-{single-core,privileged} support for this target,\\n\\\n please submit an issue at \\n\\\n see also for troubleshooting\"\n);\n\n#[cfg(portable_atomic_outline_atomics)]\n#[cfg(not(any(\n target_arch = \"aarch64\",\n target_arch = \"powerpc64\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n)))]\ncompile_error!(\"`portable_atomic_outline_atomics` cfg does not compatible with this target\");\n\n#[cfg(portable_atomic_disable_fiq)]\n#[cfg(not(all(\n target_arch = \"arm\",\n not(any(target_feature = \"mclass\", portable_atomic_target_feature = \"mclass\")),\n)))]\ncompile_error!(\n \"`portable_atomic_disable_fiq` cfg (`disable-fiq` feature) is only available on Arm (except for M-Profile architectures)\"\n);\n#[cfg(portable_atomic_s_mode)]\n#[cfg(not(any(target_arch = \"riscv32\", target_arch = \"riscv64\")))]\ncompile_error!(\"`portable_atomic_s_mode` cfg (`s-mode` feature) is only available on RISC-V\");\n#[cfg(portable_atomic_force_amo)]\n#[cfg(not(any(target_arch = \"riscv32\", target_arch = \"riscv64\")))]\ncompile_error!(\"`portable_atomic_force_amo` cfg (`force-amo` feature) is only available on RISC-V\");\n\n#[cfg(portable_atomic_disable_fiq)]\n#[cfg(not(any(\n portable_atomic_unsafe_assume_single_core,\n portable_atomic_unsafe_assume_privileged,\n)))]\ncompile_error!(\n \"`portable_atomic_disable_fiq` cfg (`disable-fiq` feature) may only be used together with `portable_atomic_unsafe_assume_{single_core,privileged}` cfg (`unsafe-assume-{single-core,privileged}` feature)\"\n);\n#[cfg(portable_atomic_s_mode)]\n#[cfg(not(any(\n portable_atomic_unsafe_assume_single_core,\n portable_atomic_unsafe_assume_privileged,\n)))]\ncompile_error!(\n \"`portable_atomic_s_mode` cfg (`s-mode` feature) may only be used together with `portable_atomic_unsafe_assume_{single_core,privileged}` cfg (`unsafe-assume-{single-core,privileged}` feature)\"\n);\n#[cfg(portable_atomic_force_amo)]\n#[cfg(not(portable_atomic_unsafe_assume_single_core))]\ncompile_error!(\n \"`portable_atomic_force_amo` cfg (`force-amo` feature) may only be used together with `portable_atomic_unsafe_assume_single_core` cfg (`unsafe-assume-single-core` feature)\"\n);\n#[cfg(portable_atomic_unsafe_assume_privileged)]\n#[cfg(not(feature = \"fallback\"))]\ncompile_error!(\n \"`portable_atomic_unsafe_assume_privileged` cfg (`unsafe-assume-privileged` feature) may only be used together with `fallback` feature\"\n);\n\n#[cfg(all(\n any(portable_atomic_unsafe_assume_single_core, portable_atomic_unsafe_assume_privileged),\n feature = \"critical-section\"\n))]\ncompile_error!(\n \"you may not enable `critical-section` feature and `portable_atomic_unsafe_assume_{single_core,privileged}` cfg (`unsafe-assume-{single-core,privileged}` feature) at the same time\"\n);\n\n#[cfg(feature = \"require-cas\")]\n#[cfg_attr(\n portable_atomic_no_cfg_target_has_atomic,\n cfg(not(any(\n not(portable_atomic_no_atomic_cas),\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )))\n)]\n#[cfg_attr(\n not(portable_atomic_no_cfg_target_has_atomic),\n cfg(not(any(\n target_has_atomic = \"ptr\",\n target_arch = \"avr\",\n target_arch = \"msp430\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )))\n)]\ncompile_error!(\n \"dependents require atomic CAS but not available on this target by default;\\n\\\n consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg).\\n\\\n see for more.\"\n);\n\n#[macro_use]\nmod utils;\n\n#[cfg(test)]\n#[macro_use]\nmod tests;\n\n#[doc(no_inline)]\npub use core::sync::atomic::Ordering;\n\ncfg_sel!({\n // LLVM doesn't support fence/compiler_fence for MSP430.\n #[cfg(target_arch = \"msp430\")]\n {\n pub use self::imp::msp430::compiler_fence;\n }\n #[cfg(else)]\n {\n pub use core::sync::atomic::compiler_fence;\n }\n});\ncfg_sel!({\n // LLVM doesn't support fence/compiler_fence for MSP430.\n #[cfg(target_arch = \"msp430\")]\n {\n pub use self::imp::msp430::fence;\n }\n // We have optimized fence for x86.\n // Miri and Sanitizer do not support inline assembly.\n #[cfg(all(\n not(doc),\n any(target_arch = \"x86\", target_arch = \"x86_64\"),\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n ))]\n {\n pub use self::imp::x86::fence;\n }\n // We have optimized fence for pre-v6 ARM Linux/Android.\n // Miri and Sanitizer do not go through __sync_synchronize.\n #[cfg(all(\n not(doc),\n target_arch = \"arm\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(target_os = \"linux\", target_os = \"android\"),\n not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\")),\n ))]\n {\n pub use self::imp::arm_linux::fence;\n }\n #[cfg(else)]\n {\n pub use core::sync::atomic::fence;\n }\n});\n\nmod imp;\n\npub mod hint {\n //! Re-export of the [`core::hint`] module.\n //!\n //! The only difference from the [`core::hint`] module is that [`spin_loop`]\n //! is available in all rust versions that this crate supports.\n //!\n //! ```\n //! use portable_atomic::hint;\n //!\n //! hint::spin_loop();\n //! ```\n\n #[doc(no_inline)]\n pub use core::hint::*;\n\n /// Emits a machine instruction to signal the processor that it is running in\n /// a busy-wait spin-loop (\"spin lock\").\n ///\n /// Upon receiving the spin-loop signal the processor can optimize its behavior by,\n /// for example, saving power or switching hyper-threads.\n ///\n /// This function is different from [`thread::yield_now`] which directly\n /// yields to the system's scheduler, whereas `spin_loop` does not interact\n /// with the operating system.\n ///\n /// A common use case for `spin_loop` is implementing bounded optimistic\n /// spinning in a CAS loop in synchronization primitives. To avoid problems\n /// like priority inversion, it is strongly recommended that the spin loop is\n /// terminated after a finite amount of iterations and an appropriate blocking\n /// syscall is made.\n ///\n /// **Note:** On platforms that do not support receiving spin-loop hints this\n /// function does not do anything at all.\n ///\n /// [`thread::yield_now`]: https://doc.rust-lang.org/std/thread/fn.yield_now.html\n #[inline]\n pub fn spin_loop() {\n #[allow(deprecated)]\n core::sync::atomic::spin_loop_hint();\n }\n}\n\n#[cfg(doc)]\nuse core::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release, SeqCst};\nuse core::{fmt, ptr};\n\ncfg_has_atomic_8! {\n/// A boolean type which can be safely shared between threads.\n///\n/// This type has the same in-memory representation as a [`bool`].\n///\n/// If the compiler and the platform support atomic loads and stores of `u8`,\n/// this type is a wrapper for the standard library's\n/// [`AtomicBool`](core::sync::atomic::AtomicBool). If the platform supports it\n/// but the compiler does not, atomic operations are implemented using inline\n/// assembly.\n#[repr(C, align(1))]\npub struct AtomicBool {\n v: core::cell::UnsafeCell,\n}\n\nimpl Default for AtomicBool {\n /// Creates an `AtomicBool` initialized to `false`.\n #[inline]\n fn default() -> Self {\n Self::new(false)\n }\n}\n\nimpl From for AtomicBool {\n /// Converts a `bool` into an `AtomicBool`.\n #[inline]\n fn from(b: bool) -> Self {\n Self::new(b)\n }\n}\n\n// Send is implicitly implemented.\n// SAFETY: any data races are prevented by disabling interrupts or\n// atomic intrinsics (see module-level comments).\nunsafe impl Sync for AtomicBool {}\n\n// UnwindSafe is implicitly implemented.\n#[cfg(not(portable_atomic_no_core_unwind_safe))]\nimpl core::panic::RefUnwindSafe for AtomicBool {}\n#[cfg(all(portable_atomic_no_core_unwind_safe, feature = \"std\"))]\nimpl std::panic::RefUnwindSafe for AtomicBool {}\n\nimpl_debug_and_serde!(AtomicBool);\n\nimpl AtomicBool {\n /// Creates a new `AtomicBool`.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::AtomicBool;\n ///\n /// let atomic_true = AtomicBool::new(true);\n /// let atomic_false = AtomicBool::new(false);\n /// ```\n #[inline]\n #[must_use]\n pub const fn new(v: bool) -> Self {\n static_assert_layout!(AtomicBool, bool);\n Self { v: core::cell::UnsafeCell::new(v as u8) }\n }\n\n // TODO: update docs based on https://github.com/rust-lang/rust/pull/116762\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];\n /// Creates a new `AtomicBool` from a pointer.\n ///\n /// This is `const fn` on Rust 1.58+.\n ///\n /// # Safety\n ///\n /// * `ptr` must be aligned to `align_of::()` (note that on some platforms this can\n /// be bigger than `align_of::()`).\n /// * `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`.\n /// * If this atomic type is [lock-free](Self::is_lock_free), non-atomic accesses to the value\n /// behind `ptr` must have a happens-before relationship with atomic accesses via the returned\n /// value (or vice-versa).\n /// * In other words, time periods where the value is accessed atomically may not overlap\n /// with periods where the value is accessed non-atomically.\n /// * This requirement is trivially satisfied if `ptr` is never used non-atomically for the\n /// duration of lifetime `'a`. Most use cases should be able to follow this guideline.\n /// * This requirement is also trivially satisfied if all accesses (atomic or not) are done\n /// from the same thread.\n /// * If this atomic type is *not* lock-free:\n /// * Any accesses to the value behind `ptr` must have a happens-before relationship\n /// with accesses via the returned value (or vice-versa).\n /// * Any concurrent accesses to the value behind `ptr` for the duration of lifetime `'a` must\n /// be compatible with operations performed by this atomic type.\n /// * This method must not be used to create overlapping or mixed-size atomic accesses, as\n /// these are not supported by the memory model.\n ///\n /// [valid]: core::ptr#safety\n #[inline]\n #[must_use]\n pub const unsafe fn from_ptr<'a>(ptr: *mut bool) -> &'a Self {\n #[allow(clippy::cast_ptr_alignment)]\n // SAFETY: guaranteed by the caller\n unsafe { &*(ptr as *mut Self as *const Self) }\n }\n }\n\n /// Returns `true` if operations on values of this type are lock-free.\n ///\n /// If the compiler or the platform doesn't support the necessary\n /// atomic instructions, global locks for every potentially\n /// concurrent atomic operation will be used.\n ///\n /// This function is guaranteed to always return the same result.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::AtomicBool;\n ///\n /// let is_lock_free = AtomicBool::is_lock_free();\n /// ```\n #[inline]\n #[must_use]\n pub fn is_lock_free() -> bool {\n imp::AtomicU8::is_lock_free()\n }\n\n /// Returns `true` if operations on values of this type are lock-free.\n ///\n /// If the compiler or the platform doesn't support the necessary\n /// atomic instructions, global locks for every potentially\n /// concurrent atomic operation will be used.\n ///\n /// **Note:** If the atomic operation relies on dynamic CPU feature detection,\n /// this type may be lock-free even if the function returns false.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::AtomicBool;\n ///\n /// const IS_ALWAYS_LOCK_FREE: bool = AtomicBool::is_always_lock_free();\n /// ```\n #[inline]\n #[must_use]\n pub const fn is_always_lock_free() -> bool {\n imp::AtomicU8::IS_ALWAYS_LOCK_FREE\n }\n #[cfg(test)]\n const IS_ALWAYS_LOCK_FREE: bool = Self::is_always_lock_free();\n\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_mut_refs))];\n /// Returns a mutable reference to the underlying [`bool`].\n ///\n /// This is safe because the mutable reference guarantees that no other threads are\n /// concurrently accessing the atomic data.\n ///\n /// This is `const fn` on Rust 1.83+.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let mut some_bool = AtomicBool::new(true);\n /// assert_eq!(*some_bool.get_mut(), true);\n /// *some_bool.get_mut() = false;\n /// assert_eq!(some_bool.load(Ordering::SeqCst), false);\n /// ```\n #[inline]\n pub const fn get_mut(&mut self) -> &mut bool {\n // SAFETY: the mutable reference guarantees unique ownership.\n unsafe { &mut *self.as_ptr() }\n }\n }\n\n // TODO: Add from_mut/get_mut_slice/from_mut_slice once it is stable on std atomic types.\n // https://github.com/rust-lang/rust/issues/76314\n\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_transmute))];\n /// Consumes the atomic and returns the contained value.\n ///\n /// This is safe because passing `self` by value guarantees that no other threads are\n /// concurrently accessing the atomic data.\n ///\n /// This is `const fn` on Rust 1.56+.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::AtomicBool;\n ///\n /// let some_bool = AtomicBool::new(true);\n /// assert_eq!(some_bool.into_inner(), true);\n /// ```\n #[inline]\n pub const fn into_inner(self) -> bool {\n // SAFETY: AtomicBool and u8 have the same size and in-memory representations,\n // so they can be safely transmuted.\n // (const UnsafeCell::into_inner is unstable)\n unsafe { core::mem::transmute::(self) != 0 }\n }\n }\n\n /// Loads a value from the bool.\n ///\n /// `load` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `order` is [`Release`] or [`AcqRel`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let some_bool = AtomicBool::new(true);\n ///\n /// assert_eq!(some_bool.load(Ordering::Relaxed), true);\n /// ```\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn load(&self, order: Ordering) -> bool {\n self.as_atomic_u8().load(order) != 0\n }\n\n /// Stores a value into the bool.\n ///\n /// `store` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `order` is [`Acquire`] or [`AcqRel`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let some_bool = AtomicBool::new(true);\n ///\n /// some_bool.store(false, Ordering::Relaxed);\n /// assert_eq!(some_bool.load(Ordering::Relaxed), false);\n /// ```\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn store(&self, val: bool, order: Ordering) {\n self.as_atomic_u8().store(val as u8, order);\n }\n\n cfg_has_atomic_cas_or_amo32! {\n /// Stores a value into the bool, returning the previous value.\n ///\n /// `swap` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let some_bool = AtomicBool::new(true);\n ///\n /// assert_eq!(some_bool.swap(false, Ordering::Relaxed), true);\n /// assert_eq!(some_bool.load(Ordering::Relaxed), false);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn swap(&self, val: bool, order: Ordering) -> bool {\n #[cfg(any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"loongarch32\",\n target_arch = \"loongarch64\",\n ))]\n {\n // See https://github.com/rust-lang/rust/pull/114034 for details.\n // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L249\n // https://godbolt.org/z/ofbGGdx44\n if val { self.fetch_or(true, order) } else { self.fetch_and(false, order) }\n }\n #[cfg(not(any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"loongarch32\",\n target_arch = \"loongarch64\",\n )))]\n {\n self.as_atomic_u8().swap(val as u8, order) != 0\n }\n }\n\n /// Stores a value into the [`bool`] if the current value is the same as the `current` value.\n ///\n /// The return value is a result indicating whether the new value was written and containing\n /// the previous value. On success this value is guaranteed to be equal to `current`.\n ///\n /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory\n /// ordering of this operation. `success` describes the required ordering for the\n /// read-modify-write operation that takes place if the comparison with `current` succeeds.\n /// `failure` describes the required ordering for the load operation that takes place when\n /// the comparison fails. Using [`Acquire`] as success ordering makes the store part\n /// of this operation [`Relaxed`], and using [`Release`] makes the successful load\n /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `failure` is [`Release`], [`AcqRel`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let some_bool = AtomicBool::new(true);\n ///\n /// assert_eq!(\n /// some_bool.compare_exchange(true, false, Ordering::Acquire, Ordering::Relaxed),\n /// Ok(true)\n /// );\n /// assert_eq!(some_bool.load(Ordering::Relaxed), false);\n ///\n /// assert_eq!(\n /// some_bool.compare_exchange(true, true, Ordering::SeqCst, Ordering::Acquire),\n /// Err(false)\n /// );\n /// assert_eq!(some_bool.load(Ordering::Relaxed), false);\n /// ```\n #[cfg_attr(docsrs, doc(alias = \"compare_and_swap\"))]\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn compare_exchange(\n &self,\n current: bool,\n new: bool,\n success: Ordering,\n failure: Ordering,\n ) -> Result {\n #[cfg(any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"loongarch32\",\n target_arch = \"loongarch64\",\n ))]\n {\n // See https://github.com/rust-lang/rust/pull/114034 for details.\n // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L249\n // https://godbolt.org/z/ofbGGdx44\n crate::utils::assert_compare_exchange_ordering(success, failure);\n let order = crate::utils::upgrade_success_ordering(success, failure);\n let old = if current == new {\n // This is a no-op, but we still need to perform the operation\n // for memory ordering reasons.\n self.fetch_or(false, order)\n } else {\n // This sets the value to the new one and returns the old one.\n self.swap(new, order)\n };\n if old == current { Ok(old) } else { Err(old) }\n }\n #[cfg(not(any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"loongarch32\",\n target_arch = \"loongarch64\",\n )))]\n {\n match self.as_atomic_u8().compare_exchange(current as u8, new as u8, success, failure) {\n Ok(x) => Ok(x != 0),\n Err(x) => Err(x != 0),\n }\n }\n }\n\n /// Stores a value into the [`bool`] if the current value is the same as the `current` value.\n ///\n /// Unlike [`AtomicBool::compare_exchange`], this function is allowed to spuriously fail even when the\n /// comparison succeeds, which can result in more efficient code on some platforms. The\n /// return value is a result indicating whether the new value was written and containing the\n /// previous value.\n ///\n /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory\n /// ordering of this operation. `success` describes the required ordering for the\n /// read-modify-write operation that takes place if the comparison with `current` succeeds.\n /// `failure` describes the required ordering for the load operation that takes place when\n /// the comparison fails. Using [`Acquire`] as success ordering makes the store part\n /// of this operation [`Relaxed`], and using [`Release`] makes the successful load\n /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `failure` is [`Release`], [`AcqRel`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let val = AtomicBool::new(false);\n ///\n /// let new = true;\n /// let mut old = val.load(Ordering::Relaxed);\n /// loop {\n /// match val.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) {\n /// Ok(_) => break,\n /// Err(x) => old = x,\n /// }\n /// }\n /// ```\n #[cfg_attr(docsrs, doc(alias = \"compare_and_swap\"))]\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn compare_exchange_weak(\n &self,\n current: bool,\n new: bool,\n success: Ordering,\n failure: Ordering,\n ) -> Result {\n #[cfg(any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"loongarch32\",\n target_arch = \"loongarch64\",\n ))]\n {\n // See https://github.com/rust-lang/rust/pull/114034 for details.\n // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L249\n // https://godbolt.org/z/ofbGGdx44\n self.compare_exchange(current, new, success, failure)\n }\n #[cfg(not(any(\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"loongarch32\",\n target_arch = \"loongarch64\",\n )))]\n {\n match self\n .as_atomic_u8()\n .compare_exchange_weak(current as u8, new as u8, success, failure)\n {\n Ok(x) => Ok(x != 0),\n Err(x) => Err(x != 0),\n }\n }\n }\n\n /// Logical \"and\" with a boolean value.\n ///\n /// Performs a logical \"and\" operation on the current value and the argument `val`, and sets\n /// the new value to the result.\n ///\n /// Returns the previous value.\n ///\n /// `fetch_and` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let foo = AtomicBool::new(true);\n /// assert_eq!(foo.fetch_and(false, Ordering::SeqCst), true);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n ///\n /// let foo = AtomicBool::new(true);\n /// assert_eq!(foo.fetch_and(true, Ordering::SeqCst), true);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n ///\n /// let foo = AtomicBool::new(false);\n /// assert_eq!(foo.fetch_and(false, Ordering::SeqCst), false);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_and(&self, val: bool, order: Ordering) -> bool {\n self.as_atomic_u8().fetch_and(val as u8, order) != 0\n }\n\n /// Logical \"and\" with a boolean value.\n ///\n /// Performs a logical \"and\" operation on the current value and the argument `val`, and sets\n /// the new value to the result.\n ///\n /// Unlike `fetch_and`, this does not return the previous value.\n ///\n /// `and` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// This function may generate more efficient code than `fetch_and` on some platforms.\n ///\n /// - x86/x86_64: `lock and` instead of `cmpxchg` loop\n /// - MSP430: `and` instead of disabling interrupts\n ///\n /// Note: On x86/x86_64, the use of either function should not usually\n /// affect the generated code, because LLVM can properly optimize the case\n /// where the result is unused.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let foo = AtomicBool::new(true);\n /// foo.and(false, Ordering::SeqCst);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n ///\n /// let foo = AtomicBool::new(true);\n /// foo.and(true, Ordering::SeqCst);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n ///\n /// let foo = AtomicBool::new(false);\n /// foo.and(false, Ordering::SeqCst);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn and(&self, val: bool, order: Ordering) {\n self.as_atomic_u8().and(val as u8, order);\n }\n\n /// Logical \"nand\" with a boolean value.\n ///\n /// Performs a logical \"nand\" operation on the current value and the argument `val`, and sets\n /// the new value to the result.\n ///\n /// Returns the previous value.\n ///\n /// `fetch_nand` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let foo = AtomicBool::new(true);\n /// assert_eq!(foo.fetch_nand(false, Ordering::SeqCst), true);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n ///\n /// let foo = AtomicBool::new(true);\n /// assert_eq!(foo.fetch_nand(true, Ordering::SeqCst), true);\n /// assert_eq!(foo.load(Ordering::SeqCst) as usize, 0);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n ///\n /// let foo = AtomicBool::new(false);\n /// assert_eq!(foo.fetch_nand(false, Ordering::SeqCst), false);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_nand(&self, val: bool, order: Ordering) -> bool {\n // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L973-L985\n if val {\n // !(x & true) == !x\n // We must invert the bool.\n self.fetch_xor(true, order)\n } else {\n // !(x & false) == true\n // We must set the bool to true.\n self.swap(true, order)\n }\n }\n\n /// Logical \"or\" with a boolean value.\n ///\n /// Performs a logical \"or\" operation on the current value and the argument `val`, and sets the\n /// new value to the result.\n ///\n /// Returns the previous value.\n ///\n /// `fetch_or` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let foo = AtomicBool::new(true);\n /// assert_eq!(foo.fetch_or(false, Ordering::SeqCst), true);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n ///\n /// let foo = AtomicBool::new(true);\n /// assert_eq!(foo.fetch_or(true, Ordering::SeqCst), true);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n ///\n /// let foo = AtomicBool::new(false);\n /// assert_eq!(foo.fetch_or(false, Ordering::SeqCst), false);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_or(&self, val: bool, order: Ordering) -> bool {\n self.as_atomic_u8().fetch_or(val as u8, order) != 0\n }\n\n /// Logical \"or\" with a boolean value.\n ///\n /// Performs a logical \"or\" operation on the current value and the argument `val`, and sets the\n /// new value to the result.\n ///\n /// Unlike `fetch_or`, this does not return the previous value.\n ///\n /// `or` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// This function may generate more efficient code than `fetch_or` on some platforms.\n ///\n /// - x86/x86_64: `lock or` instead of `cmpxchg` loop\n /// - MSP430: `bis` instead of disabling interrupts\n ///\n /// Note: On x86/x86_64, the use of either function should not usually\n /// affect the generated code, because LLVM can properly optimize the case\n /// where the result is unused.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let foo = AtomicBool::new(true);\n /// foo.or(false, Ordering::SeqCst);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n ///\n /// let foo = AtomicBool::new(true);\n /// foo.or(true, Ordering::SeqCst);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n ///\n /// let foo = AtomicBool::new(false);\n /// foo.or(false, Ordering::SeqCst);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn or(&self, val: bool, order: Ordering) {\n self.as_atomic_u8().or(val as u8, order);\n }\n\n /// Logical \"xor\" with a boolean value.\n ///\n /// Performs a logical \"xor\" operation on the current value and the argument `val`, and sets\n /// the new value to the result.\n ///\n /// Returns the previous value.\n ///\n /// `fetch_xor` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let foo = AtomicBool::new(true);\n /// assert_eq!(foo.fetch_xor(false, Ordering::SeqCst), true);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n ///\n /// let foo = AtomicBool::new(true);\n /// assert_eq!(foo.fetch_xor(true, Ordering::SeqCst), true);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n ///\n /// let foo = AtomicBool::new(false);\n /// assert_eq!(foo.fetch_xor(false, Ordering::SeqCst), false);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_xor(&self, val: bool, order: Ordering) -> bool {\n self.as_atomic_u8().fetch_xor(val as u8, order) != 0\n }\n\n /// Logical \"xor\" with a boolean value.\n ///\n /// Performs a logical \"xor\" operation on the current value and the argument `val`, and sets\n /// the new value to the result.\n ///\n /// Unlike `fetch_xor`, this does not return the previous value.\n ///\n /// `xor` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// This function may generate more efficient code than `fetch_xor` on some platforms.\n ///\n /// - x86/x86_64: `lock xor` instead of `cmpxchg` loop\n /// - MSP430: `xor` instead of disabling interrupts\n ///\n /// Note: On x86/x86_64, the use of either function should not usually\n /// affect the generated code, because LLVM can properly optimize the case\n /// where the result is unused.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let foo = AtomicBool::new(true);\n /// foo.xor(false, Ordering::SeqCst);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n ///\n /// let foo = AtomicBool::new(true);\n /// foo.xor(true, Ordering::SeqCst);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n ///\n /// let foo = AtomicBool::new(false);\n /// foo.xor(false, Ordering::SeqCst);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn xor(&self, val: bool, order: Ordering) {\n self.as_atomic_u8().xor(val as u8, order);\n }\n\n /// Logical \"not\" with a boolean value.\n ///\n /// Performs a logical \"not\" operation on the current value, and sets\n /// the new value to the result.\n ///\n /// Returns the previous value.\n ///\n /// `fetch_not` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let foo = AtomicBool::new(true);\n /// assert_eq!(foo.fetch_not(Ordering::SeqCst), true);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n ///\n /// let foo = AtomicBool::new(false);\n /// assert_eq!(foo.fetch_not(Ordering::SeqCst), false);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_not(&self, order: Ordering) -> bool {\n self.fetch_xor(true, order)\n }\n\n /// Logical \"not\" with a boolean value.\n ///\n /// Performs a logical \"not\" operation on the current value, and sets\n /// the new value to the result.\n ///\n /// Unlike `fetch_not`, this does not return the previous value.\n ///\n /// `not` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// This function may generate more efficient code than `fetch_not` on some platforms.\n ///\n /// - x86/x86_64: `lock xor` instead of `cmpxchg` loop\n /// - MSP430: `xor` instead of disabling interrupts\n ///\n /// Note: On x86/x86_64, the use of either function should not usually\n /// affect the generated code, because LLVM can properly optimize the case\n /// where the result is unused.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let foo = AtomicBool::new(true);\n /// foo.not(Ordering::SeqCst);\n /// assert_eq!(foo.load(Ordering::SeqCst), false);\n ///\n /// let foo = AtomicBool::new(false);\n /// foo.not(Ordering::SeqCst);\n /// assert_eq!(foo.load(Ordering::SeqCst), true);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn not(&self, order: Ordering) {\n self.xor(true, order);\n }\n\n /// Fetches the value, and applies a function to it that returns an optional\n /// new value. Returns a `Result` of `Ok(previous_value)` if the function\n /// returned `Some(_)`, else `Err(previous_value)`.\n ///\n /// Note: This may call the function multiple times if the value has been\n /// changed from other threads in the meantime, as long as the function\n /// returns `Some(_)`, but the function will have been applied only once to\n /// the stored value.\n ///\n /// `fetch_update` takes two [`Ordering`] arguments to describe the memory\n /// ordering of this operation. The first describes the required ordering for\n /// when the operation finally succeeds while the second describes the\n /// required ordering for loads. These correspond to the success and failure\n /// orderings of [`compare_exchange`](Self::compare_exchange) respectively.\n ///\n /// Using [`Acquire`] as success ordering makes the store part of this\n /// operation [`Relaxed`], and using [`Release`] makes the final successful\n /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`],\n /// [`Acquire`] or [`Relaxed`].\n ///\n /// # Considerations\n ///\n /// This method is not magic; it is not provided by the hardware.\n /// It is implemented in terms of [`compare_exchange_weak`](Self::compare_exchange_weak),\n /// and suffers from the same drawbacks.\n /// In particular, this method will not circumvent the [ABA Problem].\n ///\n /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem\n ///\n /// # Panics\n ///\n /// Panics if `fetch_order` is [`Release`], [`AcqRel`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicBool, Ordering};\n ///\n /// let x = AtomicBool::new(false);\n /// assert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(false));\n /// assert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(false));\n /// assert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(!x)), Ok(true));\n /// assert_eq!(x.load(Ordering::SeqCst), false);\n /// ```\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn fetch_update(\n &self,\n set_order: Ordering,\n fetch_order: Ordering,\n mut f: F,\n ) -> Result\n where\n F: FnMut(bool) -> Option,\n {\n let mut prev = self.load(fetch_order);\n while let Some(next) = f(prev) {\n match self.compare_exchange_weak(prev, next, set_order, fetch_order) {\n x @ Ok(_) => return x,\n Err(next_prev) => prev = next_prev,\n }\n }\n Err(prev)\n }\n } // cfg_has_atomic_cas_or_amo32!\n\n const_fn! {\n // This function is actually `const fn`-compatible on Rust 1.32+,\n // but makes `const fn` only on Rust 1.58+ to match other atomic types.\n const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];\n /// Returns a mutable pointer to the underlying [`bool`].\n ///\n /// Returning an `*mut` pointer from a shared reference to this atomic is\n /// safe because the atomic types work with interior mutability. Any use of\n /// the returned raw pointer requires an `unsafe` block and has to uphold\n /// the safety requirements. If there is concurrent access, note the following\n /// additional safety requirements:\n ///\n /// - If this atomic type is [lock-free](Self::is_lock_free), any concurrent\n /// operations on it must be atomic.\n /// - Otherwise, any concurrent operations on it must be compatible with\n /// operations performed by this atomic type.\n ///\n /// This is `const fn` on Rust 1.58+.\n #[inline]\n pub const fn as_ptr(&self) -> *mut bool {\n self.v.get() as *mut bool\n }\n }\n\n #[inline(always)]\n fn as_atomic_u8(&self) -> &imp::AtomicU8 {\n // SAFETY: AtomicBool and imp::AtomicU8 have the same layout,\n // and both access data in the same way.\n unsafe { &*(self as *const Self as *const imp::AtomicU8) }\n }\n}\n// See https://github.com/taiki-e/portable-atomic/issues/180\n#[cfg(not(feature = \"require-cas\"))]\ncfg_no_atomic_cas! {\n#[doc(hidden)]\n#[allow(unused_variables, clippy::unused_self, clippy::extra_unused_lifetimes)]\nimpl<'a> AtomicBool {\n cfg_no_atomic_cas_or_amo32! {\n #[inline]\n pub fn swap(&self, val: bool, order: Ordering) -> bool\n where\n &'a Self: HasSwap,\n {\n unimplemented!()\n }\n #[inline]\n pub fn compare_exchange(\n &self,\n current: bool,\n new: bool,\n success: Ordering,\n failure: Ordering,\n ) -> Result\n where\n &'a Self: HasCompareExchange,\n {\n unimplemented!()\n }\n #[inline]\n pub fn compare_exchange_weak(\n &self,\n current: bool,\n new: bool,\n success: Ordering,\n failure: Ordering,\n ) -> Result\n where\n &'a Self: HasCompareExchangeWeak,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_and(&self, val: bool, order: Ordering) -> bool\n where\n &'a Self: HasFetchAnd,\n {\n unimplemented!()\n }\n #[inline]\n pub fn and(&self, val: bool, order: Ordering)\n where\n &'a Self: HasAnd,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_nand(&self, val: bool, order: Ordering) -> bool\n where\n &'a Self: HasFetchNand,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_or(&self, val: bool, order: Ordering) -> bool\n where\n &'a Self: HasFetchOr,\n {\n unimplemented!()\n }\n #[inline]\n pub fn or(&self, val: bool, order: Ordering)\n where\n &'a Self: HasOr,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_xor(&self, val: bool, order: Ordering) -> bool\n where\n &'a Self: HasFetchXor,\n {\n unimplemented!()\n }\n #[inline]\n pub fn xor(&self, val: bool, order: Ordering)\n where\n &'a Self: HasXor,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_not(&self, order: Ordering) -> bool\n where\n &'a Self: HasFetchNot,\n {\n unimplemented!()\n }\n #[inline]\n pub fn not(&self, order: Ordering)\n where\n &'a Self: HasNot,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_update(\n &self,\n set_order: Ordering,\n fetch_order: Ordering,\n f: F,\n ) -> Result\n where\n F: FnMut(bool) -> Option,\n &'a Self: HasFetchUpdate,\n {\n unimplemented!()\n }\n } // cfg_no_atomic_cas_or_amo32!\n}\n} // cfg_no_atomic_cas!\n} // cfg_has_atomic_8!\n\ncfg_has_atomic_ptr! {\n/// A raw pointer type which can be safely shared between threads.\n///\n/// This type has the same in-memory representation as a `*mut T`.\n///\n/// If the compiler and the platform support atomic loads and stores of pointers,\n/// this type is a wrapper for the standard library's\n/// [`AtomicPtr`](core::sync::atomic::AtomicPtr). If the platform supports it\n/// but the compiler does not, atomic operations are implemented using inline\n/// assembly.\n// We can use #[repr(transparent)] here, but #[repr(C, align(N))]\n// will show clearer docs.\n#[cfg_attr(target_pointer_width = \"16\", repr(C, align(2)))]\n#[cfg_attr(target_pointer_width = \"32\", repr(C, align(4)))]\n#[cfg_attr(target_pointer_width = \"64\", repr(C, align(8)))]\n#[cfg_attr(target_pointer_width = \"128\", repr(C, align(16)))]\npub struct AtomicPtr {\n inner: imp::AtomicPtr,\n}\n\nimpl Default for AtomicPtr {\n /// Creates a null `AtomicPtr`.\n #[inline]\n fn default() -> Self {\n Self::new(ptr::null_mut())\n }\n}\n\nimpl From<*mut T> for AtomicPtr {\n #[inline]\n fn from(p: *mut T) -> Self {\n Self::new(p)\n }\n}\n\nimpl fmt::Debug for AtomicPtr {\n #[inline] // fmt is not hot path, but #[inline] on fmt seems to still be useful: https://github.com/rust-lang/rust/pull/117727\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n // std atomic types use Relaxed in Debug::fmt: https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L2188\n fmt::Debug::fmt(&self.load(Ordering::Relaxed), f)\n }\n}\n\nimpl fmt::Pointer for AtomicPtr {\n #[inline] // fmt is not hot path, but #[inline] on fmt seems to still be useful: https://github.com/rust-lang/rust/pull/117727\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n // std atomic types use Relaxed in Debug::fmt: https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L2188\n fmt::Pointer::fmt(&self.load(Ordering::Relaxed), f)\n }\n}\n\n// UnwindSafe is implicitly implemented.\n#[cfg(not(portable_atomic_no_core_unwind_safe))]\nimpl core::panic::RefUnwindSafe for AtomicPtr {}\n#[cfg(all(portable_atomic_no_core_unwind_safe, feature = \"std\"))]\nimpl std::panic::RefUnwindSafe for AtomicPtr {}\n\nimpl AtomicPtr {\n /// Creates a new `AtomicPtr`.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::AtomicPtr;\n ///\n /// let ptr = &mut 5;\n /// let atomic_ptr = AtomicPtr::new(ptr);\n /// ```\n #[inline]\n #[must_use]\n pub const fn new(p: *mut T) -> Self {\n static_assert_layout!(AtomicPtr<()>, *mut ());\n Self { inner: imp::AtomicPtr::new(p) }\n }\n\n // TODO: update docs based on https://github.com/rust-lang/rust/pull/116762\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];\n /// Creates a new `AtomicPtr` from a pointer.\n ///\n /// This is `const fn` on Rust 1.58+.\n ///\n /// # Safety\n ///\n /// * `ptr` must be aligned to `align_of::>()` (note that on some platforms this\n /// can be bigger than `align_of::<*mut T>()`).\n /// * `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`.\n /// * If this atomic type is [lock-free](Self::is_lock_free), non-atomic accesses to the value\n /// behind `ptr` must have a happens-before relationship with atomic accesses via the returned\n /// value (or vice-versa).\n /// * In other words, time periods where the value is accessed atomically may not overlap\n /// with periods where the value is accessed non-atomically.\n /// * This requirement is trivially satisfied if `ptr` is never used non-atomically for the\n /// duration of lifetime `'a`. Most use cases should be able to follow this guideline.\n /// * This requirement is also trivially satisfied if all accesses (atomic or not) are done\n /// from the same thread.\n /// * If this atomic type is *not* lock-free:\n /// * Any accesses to the value behind `ptr` must have a happens-before relationship\n /// with accesses via the returned value (or vice-versa).\n /// * Any concurrent accesses to the value behind `ptr` for the duration of lifetime `'a` must\n /// be compatible with operations performed by this atomic type.\n /// * This method must not be used to create overlapping or mixed-size atomic accesses, as\n /// these are not supported by the memory model.\n ///\n /// [valid]: core::ptr#safety\n #[inline]\n #[must_use]\n pub const unsafe fn from_ptr<'a>(ptr: *mut *mut T) -> &'a Self {\n #[allow(clippy::cast_ptr_alignment)]\n // SAFETY: guaranteed by the caller\n unsafe { &*(ptr as *mut Self as *const Self) }\n }\n }\n\n /// Returns `true` if operations on values of this type are lock-free.\n ///\n /// If the compiler or the platform doesn't support the necessary\n /// atomic instructions, global locks for every potentially\n /// concurrent atomic operation will be used.\n ///\n /// This function is guaranteed to always return the same result.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::AtomicPtr;\n ///\n /// let is_lock_free = AtomicPtr::<()>::is_lock_free();\n /// ```\n #[inline]\n #[must_use]\n pub fn is_lock_free() -> bool {\n >::is_lock_free()\n }\n\n /// Returns `true` if operations on values of this type are lock-free.\n ///\n /// If the compiler or the platform doesn't support the necessary\n /// atomic instructions, global locks for every potentially\n /// concurrent atomic operation will be used.\n ///\n /// **Note:** If the atomic operation relies on dynamic CPU feature detection,\n /// this type may be lock-free even if the function returns false.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::AtomicPtr;\n ///\n /// const IS_ALWAYS_LOCK_FREE: bool = AtomicPtr::<()>::is_always_lock_free();\n /// ```\n #[inline]\n #[must_use]\n pub const fn is_always_lock_free() -> bool {\n >::IS_ALWAYS_LOCK_FREE\n }\n #[cfg(test)]\n const IS_ALWAYS_LOCK_FREE: bool = Self::is_always_lock_free();\n\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_mut_refs))];\n /// Returns a mutable reference to the underlying pointer.\n ///\n /// This is safe because the mutable reference guarantees that no other threads are\n /// concurrently accessing the atomic data.\n ///\n /// This is `const fn` on Rust 1.83+.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let mut data = 10;\n /// let mut atomic_ptr = AtomicPtr::new(&mut data);\n /// let mut other_data = 5;\n /// *atomic_ptr.get_mut() = &mut other_data;\n /// assert_eq!(unsafe { *atomic_ptr.load(Ordering::SeqCst) }, 5);\n /// ```\n #[inline]\n pub const fn get_mut(&mut self) -> &mut *mut T {\n // SAFETY: the mutable reference guarantees unique ownership.\n // (core::sync::atomic::Atomic*::get_mut is not const yet)\n unsafe { &mut *self.as_ptr() }\n }\n }\n\n // TODO: Add from_mut/get_mut_slice/from_mut_slice once it is stable on std atomic types.\n // https://github.com/rust-lang/rust/issues/76314\n\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_transmute))];\n /// Consumes the atomic and returns the contained value.\n ///\n /// This is safe because passing `self` by value guarantees that no other threads are\n /// concurrently accessing the atomic data.\n ///\n /// This is `const fn` on Rust 1.56+.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::AtomicPtr;\n ///\n /// let mut data = 5;\n /// let atomic_ptr = AtomicPtr::new(&mut data);\n /// assert_eq!(unsafe { *atomic_ptr.into_inner() }, 5);\n /// ```\n #[inline]\n pub const fn into_inner(self) -> *mut T {\n // SAFETY: AtomicPtr and *mut T have the same size and in-memory representations,\n // so they can be safely transmuted.\n // (const UnsafeCell::into_inner is unstable)\n unsafe { core::mem::transmute(self) }\n }\n }\n\n /// Loads a value from the pointer.\n ///\n /// `load` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `order` is [`Release`] or [`AcqRel`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let ptr = &mut 5;\n /// let some_ptr = AtomicPtr::new(ptr);\n ///\n /// let value = some_ptr.load(Ordering::Relaxed);\n /// ```\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn load(&self, order: Ordering) -> *mut T {\n self.inner.load(order)\n }\n\n /// Stores a value into the pointer.\n ///\n /// `store` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. Possible values are [`SeqCst`], [`Release`] and [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `order` is [`Acquire`] or [`AcqRel`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let ptr = &mut 5;\n /// let some_ptr = AtomicPtr::new(ptr);\n ///\n /// let other_ptr = &mut 10;\n ///\n /// some_ptr.store(other_ptr, Ordering::Relaxed);\n /// ```\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn store(&self, ptr: *mut T, order: Ordering) {\n self.inner.store(ptr, order);\n }\n\n cfg_has_atomic_cas_or_amo32! {\n /// Stores a value into the pointer, returning the previous value.\n ///\n /// `swap` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let ptr = &mut 5;\n /// let some_ptr = AtomicPtr::new(ptr);\n ///\n /// let other_ptr = &mut 10;\n ///\n /// let value = some_ptr.swap(other_ptr, Ordering::Relaxed);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T {\n self.inner.swap(ptr, order)\n }\n\n cfg_has_atomic_cas! {\n /// Stores a value into the pointer if the current value is the same as the `current` value.\n ///\n /// The return value is a result indicating whether the new value was written and containing\n /// the previous value. On success this value is guaranteed to be equal to `current`.\n ///\n /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory\n /// ordering of this operation. `success` describes the required ordering for the\n /// read-modify-write operation that takes place if the comparison with `current` succeeds.\n /// `failure` describes the required ordering for the load operation that takes place when\n /// the comparison fails. Using [`Acquire`] as success ordering makes the store part\n /// of this operation [`Relaxed`], and using [`Release`] makes the successful load\n /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `failure` is [`Release`], [`AcqRel`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let ptr = &mut 5;\n /// let some_ptr = AtomicPtr::new(ptr);\n ///\n /// let other_ptr = &mut 10;\n ///\n /// let value = some_ptr.compare_exchange(ptr, other_ptr, Ordering::SeqCst, Ordering::Relaxed);\n /// ```\n #[cfg_attr(docsrs, doc(alias = \"compare_and_swap\"))]\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn compare_exchange(\n &self,\n current: *mut T,\n new: *mut T,\n success: Ordering,\n failure: Ordering,\n ) -> Result<*mut T, *mut T> {\n self.inner.compare_exchange(current, new, success, failure)\n }\n\n /// Stores a value into the pointer if the current value is the same as the `current` value.\n ///\n /// Unlike [`AtomicPtr::compare_exchange`], this function is allowed to spuriously fail even when the\n /// comparison succeeds, which can result in more efficient code on some platforms. The\n /// return value is a result indicating whether the new value was written and containing the\n /// previous value.\n ///\n /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory\n /// ordering of this operation. `success` describes the required ordering for the\n /// read-modify-write operation that takes place if the comparison with `current` succeeds.\n /// `failure` describes the required ordering for the load operation that takes place when\n /// the comparison fails. Using [`Acquire`] as success ordering makes the store part\n /// of this operation [`Relaxed`], and using [`Release`] makes the successful load\n /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `failure` is [`Release`], [`AcqRel`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let some_ptr = AtomicPtr::new(&mut 5);\n ///\n /// let new = &mut 10;\n /// let mut old = some_ptr.load(Ordering::Relaxed);\n /// loop {\n /// match some_ptr.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) {\n /// Ok(_) => break,\n /// Err(x) => old = x,\n /// }\n /// }\n /// ```\n #[cfg_attr(docsrs, doc(alias = \"compare_and_swap\"))]\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn compare_exchange_weak(\n &self,\n current: *mut T,\n new: *mut T,\n success: Ordering,\n failure: Ordering,\n ) -> Result<*mut T, *mut T> {\n self.inner.compare_exchange_weak(current, new, success, failure)\n }\n\n /// Fetches the value, and applies a function to it that returns an optional\n /// new value. Returns a `Result` of `Ok(previous_value)` if the function\n /// returned `Some(_)`, else `Err(previous_value)`.\n ///\n /// Note: This may call the function multiple times if the value has been\n /// changed from other threads in the meantime, as long as the function\n /// returns `Some(_)`, but the function will have been applied only once to\n /// the stored value.\n ///\n /// `fetch_update` takes two [`Ordering`] arguments to describe the memory\n /// ordering of this operation. The first describes the required ordering for\n /// when the operation finally succeeds while the second describes the\n /// required ordering for loads. These correspond to the success and failure\n /// orderings of [`compare_exchange`](Self::compare_exchange) respectively.\n ///\n /// Using [`Acquire`] as success ordering makes the store part of this\n /// operation [`Relaxed`], and using [`Release`] makes the final successful\n /// load [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`],\n /// [`Acquire`] or [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `fetch_order` is [`Release`], [`AcqRel`].\n ///\n /// # Considerations\n ///\n /// This method is not magic; it is not provided by the hardware.\n /// It is implemented in terms of [`compare_exchange_weak`](Self::compare_exchange_weak),\n /// and suffers from the same drawbacks.\n /// In particular, this method will not circumvent the [ABA Problem].\n ///\n /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let ptr: *mut _ = &mut 5;\n /// let some_ptr = AtomicPtr::new(ptr);\n ///\n /// let new: *mut _ = &mut 10;\n /// assert_eq!(some_ptr.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(ptr));\n /// let result = some_ptr.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |x| {\n /// if x == ptr {\n /// Some(new)\n /// } else {\n /// None\n /// }\n /// });\n /// assert_eq!(result, Ok(ptr));\n /// assert_eq!(some_ptr.load(Ordering::SeqCst), new);\n /// ```\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn fetch_update(\n &self,\n set_order: Ordering,\n fetch_order: Ordering,\n mut f: F,\n ) -> Result<*mut T, *mut T>\n where\n F: FnMut(*mut T) -> Option<*mut T>,\n {\n let mut prev = self.load(fetch_order);\n while let Some(next) = f(prev) {\n match self.compare_exchange_weak(prev, next, set_order, fetch_order) {\n x @ Ok(_) => return x,\n Err(next_prev) => prev = next_prev,\n }\n }\n Err(prev)\n }\n } // cfg_has_atomic_cas!\n\n /// Offsets the pointer's address by adding `val` (in units of `T`),\n /// returning the previous pointer.\n ///\n /// This is equivalent to using [`wrapping_add`] to atomically perform the\n /// equivalent of `ptr = ptr.wrapping_add(val);`.\n ///\n /// This method operates in units of `T`, which means that it cannot be used\n /// to offset the pointer by an amount which is not a multiple of\n /// `size_of::()`. This can sometimes be inconvenient, as you may want to\n /// work with a deliberately misaligned pointer. In such cases, you may use\n /// the [`fetch_byte_add`](Self::fetch_byte_add) method instead.\n ///\n /// `fetch_ptr_add` takes an [`Ordering`] argument which describes the\n /// memory ordering of this operation. All ordering modes are possible. Note\n /// that using [`Acquire`] makes the store part of this operation\n /// [`Relaxed`], and using [`Release`] makes the load part [`Relaxed`].\n ///\n /// [`wrapping_add`]: https://doc.rust-lang.org/std/primitive.pointer.html#method.wrapping_add\n ///\n /// # Examples\n ///\n /// ```\n /// # #![allow(unstable_name_collisions)]\n /// # #[allow(unused_imports)] use sptr::Strict as _; // strict provenance polyfill for old rustc\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let atom = AtomicPtr::::new(core::ptr::null_mut());\n /// assert_eq!(atom.fetch_ptr_add(1, Ordering::Relaxed).addr(), 0);\n /// // Note: units of `size_of::()`.\n /// assert_eq!(atom.load(Ordering::Relaxed).addr(), 8);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_ptr_add(&self, val: usize, order: Ordering) -> *mut T {\n self.fetch_byte_add(val.wrapping_mul(core::mem::size_of::()), order)\n }\n\n /// Offsets the pointer's address by subtracting `val` (in units of `T`),\n /// returning the previous pointer.\n ///\n /// This is equivalent to using [`wrapping_sub`] to atomically perform the\n /// equivalent of `ptr = ptr.wrapping_sub(val);`.\n ///\n /// This method operates in units of `T`, which means that it cannot be used\n /// to offset the pointer by an amount which is not a multiple of\n /// `size_of::()`. This can sometimes be inconvenient, as you may want to\n /// work with a deliberately misaligned pointer. In such cases, you may use\n /// the [`fetch_byte_sub`](Self::fetch_byte_sub) method instead.\n ///\n /// `fetch_ptr_sub` takes an [`Ordering`] argument which describes the memory\n /// ordering of this operation. All ordering modes are possible. Note that\n /// using [`Acquire`] makes the store part of this operation [`Relaxed`],\n /// and using [`Release`] makes the load part [`Relaxed`].\n ///\n /// [`wrapping_sub`]: https://doc.rust-lang.org/std/primitive.pointer.html#method.wrapping_sub\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let array = [1i32, 2i32];\n /// let atom = AtomicPtr::new(array.as_ptr().wrapping_add(1) as *mut _);\n ///\n /// assert!(core::ptr::eq(atom.fetch_ptr_sub(1, Ordering::Relaxed), &array[1]));\n /// assert!(core::ptr::eq(atom.load(Ordering::Relaxed), &array[0]));\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_ptr_sub(&self, val: usize, order: Ordering) -> *mut T {\n self.fetch_byte_sub(val.wrapping_mul(core::mem::size_of::()), order)\n }\n\n /// Offsets the pointer's address by adding `val` *bytes*, returning the\n /// previous pointer.\n ///\n /// This is equivalent to using [`wrapping_add`] and [`cast`] to atomically\n /// perform `ptr = ptr.cast::().wrapping_add(val).cast::()`.\n ///\n /// `fetch_byte_add` takes an [`Ordering`] argument which describes the\n /// memory ordering of this operation. All ordering modes are possible. Note\n /// that using [`Acquire`] makes the store part of this operation\n /// [`Relaxed`], and using [`Release`] makes the load part [`Relaxed`].\n ///\n /// [`wrapping_add`]: https://doc.rust-lang.org/std/primitive.pointer.html#method.wrapping_add\n /// [`cast`]: https://doc.rust-lang.org/std/primitive.pointer.html#method.cast\n ///\n /// # Examples\n ///\n /// ```\n /// # #![allow(unstable_name_collisions)]\n /// # #[allow(unused_imports)] use sptr::Strict as _; // strict provenance polyfill for old rustc\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let atom = AtomicPtr::::new(core::ptr::null_mut());\n /// assert_eq!(atom.fetch_byte_add(1, Ordering::Relaxed).addr(), 0);\n /// // Note: in units of bytes, not `size_of::()`.\n /// assert_eq!(atom.load(Ordering::Relaxed).addr(), 1);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_byte_add(&self, val: usize, order: Ordering) -> *mut T {\n self.inner.fetch_byte_add(val, order)\n }\n\n /// Offsets the pointer's address by subtracting `val` *bytes*, returning the\n /// previous pointer.\n ///\n /// This is equivalent to using [`wrapping_sub`] and [`cast`] to atomically\n /// perform `ptr = ptr.cast::().wrapping_sub(val).cast::()`.\n ///\n /// `fetch_byte_sub` takes an [`Ordering`] argument which describes the\n /// memory ordering of this operation. All ordering modes are possible. Note\n /// that using [`Acquire`] makes the store part of this operation\n /// [`Relaxed`], and using [`Release`] makes the load part [`Relaxed`].\n ///\n /// [`wrapping_sub`]: https://doc.rust-lang.org/std/primitive.pointer.html#method.wrapping_sub\n /// [`cast`]: https://doc.rust-lang.org/std/primitive.pointer.html#method.cast\n ///\n /// # Examples\n ///\n /// ```\n /// # #![allow(unstable_name_collisions)]\n /// # #[allow(unused_imports)] use sptr::Strict as _; // strict provenance polyfill for old rustc\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let atom = AtomicPtr::::new(sptr::invalid_mut(1));\n /// assert_eq!(atom.fetch_byte_sub(1, Ordering::Relaxed).addr(), 1);\n /// assert_eq!(atom.load(Ordering::Relaxed).addr(), 0);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_byte_sub(&self, val: usize, order: Ordering) -> *mut T {\n self.inner.fetch_byte_sub(val, order)\n }\n\n /// Performs a bitwise \"or\" operation on the address of the current pointer,\n /// and the argument `val`, and stores a pointer with provenance of the\n /// current pointer and the resulting address.\n ///\n /// This is equivalent to using [`map_addr`] to atomically perform\n /// `ptr = ptr.map_addr(|a| a | val)`. This can be used in tagged\n /// pointer schemes to atomically set tag bits.\n ///\n /// **Caveat**: This operation returns the previous value. To compute the\n /// stored value without losing provenance, you may use [`map_addr`]. For\n /// example: `a.fetch_or(val).map_addr(|a| a | val)`.\n ///\n /// `fetch_or` takes an [`Ordering`] argument which describes the memory\n /// ordering of this operation. All ordering modes are possible. Note that\n /// using [`Acquire`] makes the store part of this operation [`Relaxed`],\n /// and using [`Release`] makes the load part [`Relaxed`].\n ///\n /// This API and its claimed semantics are part of the Strict Provenance\n /// experiment, see the [module documentation for `ptr`][core::ptr] for\n /// details.\n ///\n /// [`map_addr`]: https://doc.rust-lang.org/std/primitive.pointer.html#method.map_addr\n ///\n /// # Examples\n ///\n /// ```\n /// # #![allow(unstable_name_collisions)]\n /// # #[allow(unused_imports)] use sptr::Strict as _; // strict provenance polyfill for old rustc\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let pointer = &mut 3i64 as *mut i64;\n ///\n /// let atom = AtomicPtr::::new(pointer);\n /// // Tag the bottom bit of the pointer.\n /// assert_eq!(atom.fetch_or(1, Ordering::Relaxed).addr() & 1, 0);\n /// // Extract and untag.\n /// let tagged = atom.load(Ordering::Relaxed);\n /// assert_eq!(tagged.addr() & 1, 1);\n /// assert_eq!(tagged.map_addr(|p| p & !1), pointer);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_or(&self, val: usize, order: Ordering) -> *mut T {\n self.inner.fetch_or(val, order)\n }\n\n /// Performs a bitwise \"and\" operation on the address of the current\n /// pointer, and the argument `val`, and stores a pointer with provenance of\n /// the current pointer and the resulting address.\n ///\n /// This is equivalent to using [`map_addr`] to atomically perform\n /// `ptr = ptr.map_addr(|a| a & val)`. This can be used in tagged\n /// pointer schemes to atomically unset tag bits.\n ///\n /// **Caveat**: This operation returns the previous value. To compute the\n /// stored value without losing provenance, you may use [`map_addr`]. For\n /// example: `a.fetch_and(val).map_addr(|a| a & val)`.\n ///\n /// `fetch_and` takes an [`Ordering`] argument which describes the memory\n /// ordering of this operation. All ordering modes are possible. Note that\n /// using [`Acquire`] makes the store part of this operation [`Relaxed`],\n /// and using [`Release`] makes the load part [`Relaxed`].\n ///\n /// This API and its claimed semantics are part of the Strict Provenance\n /// experiment, see the [module documentation for `ptr`][core::ptr] for\n /// details.\n ///\n /// [`map_addr`]: https://doc.rust-lang.org/std/primitive.pointer.html#method.map_addr\n ///\n /// # Examples\n ///\n /// ```\n /// # #![allow(unstable_name_collisions)]\n /// # #[allow(unused_imports)] use sptr::Strict as _; // strict provenance polyfill for old rustc\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let pointer = &mut 3i64 as *mut i64;\n /// // A tagged pointer\n /// let atom = AtomicPtr::::new(pointer.map_addr(|a| a | 1));\n /// assert_eq!(atom.fetch_or(1, Ordering::Relaxed).addr() & 1, 1);\n /// // Untag, and extract the previously tagged pointer.\n /// let untagged = atom.fetch_and(!1, Ordering::Relaxed).map_addr(|a| a & !1);\n /// assert_eq!(untagged, pointer);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_and(&self, val: usize, order: Ordering) -> *mut T {\n self.inner.fetch_and(val, order)\n }\n\n /// Performs a bitwise \"xor\" operation on the address of the current\n /// pointer, and the argument `val`, and stores a pointer with provenance of\n /// the current pointer and the resulting address.\n ///\n /// This is equivalent to using [`map_addr`] to atomically perform\n /// `ptr = ptr.map_addr(|a| a ^ val)`. This can be used in tagged\n /// pointer schemes to atomically toggle tag bits.\n ///\n /// **Caveat**: This operation returns the previous value. To compute the\n /// stored value without losing provenance, you may use [`map_addr`]. For\n /// example: `a.fetch_xor(val).map_addr(|a| a ^ val)`.\n ///\n /// `fetch_xor` takes an [`Ordering`] argument which describes the memory\n /// ordering of this operation. All ordering modes are possible. Note that\n /// using [`Acquire`] makes the store part of this operation [`Relaxed`],\n /// and using [`Release`] makes the load part [`Relaxed`].\n ///\n /// This API and its claimed semantics are part of the Strict Provenance\n /// experiment, see the [module documentation for `ptr`][core::ptr] for\n /// details.\n ///\n /// [`map_addr`]: https://doc.rust-lang.org/std/primitive.pointer.html#method.map_addr\n ///\n /// # Examples\n ///\n /// ```\n /// # #![allow(unstable_name_collisions)]\n /// # #[allow(unused_imports)] use sptr::Strict as _; // strict provenance polyfill for old rustc\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let pointer = &mut 3i64 as *mut i64;\n /// let atom = AtomicPtr::::new(pointer);\n ///\n /// // Toggle a tag bit on the pointer.\n /// atom.fetch_xor(1, Ordering::Relaxed);\n /// assert_eq!(atom.load(Ordering::Relaxed).addr() & 1, 1);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_xor(&self, val: usize, order: Ordering) -> *mut T {\n self.inner.fetch_xor(val, order)\n }\n\n /// Sets the bit at the specified bit-position to 1.\n ///\n /// Returns `true` if the specified bit was previously set to 1.\n ///\n /// `bit_set` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// This corresponds to x86's `lock bts`, and the implementation calls them on x86/x86_64.\n ///\n /// # Examples\n ///\n /// ```\n /// # #![allow(unstable_name_collisions)]\n /// # #[allow(unused_imports)] use sptr::Strict as _; // strict provenance polyfill for old rustc\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let pointer = &mut 3i64 as *mut i64;\n ///\n /// let atom = AtomicPtr::::new(pointer);\n /// // Tag the bottom bit of the pointer.\n /// assert!(!atom.bit_set(0, Ordering::Relaxed));\n /// // Extract and untag.\n /// let tagged = atom.load(Ordering::Relaxed);\n /// assert_eq!(tagged.addr() & 1, 1);\n /// assert_eq!(tagged.map_addr(|p| p & !1), pointer);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn bit_set(&self, bit: u32, order: Ordering) -> bool {\n self.inner.bit_set(bit, order)\n }\n\n /// Clears the bit at the specified bit-position to 0.\n ///\n /// Returns `true` if the specified bit was previously set to 1.\n ///\n /// `bit_clear` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// This corresponds to x86's `lock btr`, and the implementation calls them on x86/x86_64.\n ///\n /// # Examples\n ///\n /// ```\n /// # #![allow(unstable_name_collisions)]\n /// # #[allow(unused_imports)] use sptr::Strict as _; // strict provenance polyfill for old rustc\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let pointer = &mut 3i64 as *mut i64;\n /// // A tagged pointer\n /// let atom = AtomicPtr::::new(pointer.map_addr(|a| a | 1));\n /// assert!(atom.bit_set(0, Ordering::Relaxed));\n /// // Untag\n /// assert!(atom.bit_clear(0, Ordering::Relaxed));\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn bit_clear(&self, bit: u32, order: Ordering) -> bool {\n self.inner.bit_clear(bit, order)\n }\n\n /// Toggles the bit at the specified bit-position.\n ///\n /// Returns `true` if the specified bit was previously set to 1.\n ///\n /// `bit_toggle` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n ///\n /// This corresponds to x86's `lock btc`, and the implementation calls them on x86/x86_64.\n ///\n /// # Examples\n ///\n /// ```\n /// # #![allow(unstable_name_collisions)]\n /// # #[allow(unused_imports)] use sptr::Strict as _; // strict provenance polyfill for old rustc\n /// use portable_atomic::{AtomicPtr, Ordering};\n ///\n /// let pointer = &mut 3i64 as *mut i64;\n /// let atom = AtomicPtr::::new(pointer);\n ///\n /// // Toggle a tag bit on the pointer.\n /// atom.bit_toggle(0, Ordering::Relaxed);\n /// assert_eq!(atom.load(Ordering::Relaxed).addr() & 1, 1);\n /// ```\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn bit_toggle(&self, bit: u32, order: Ordering) -> bool {\n self.inner.bit_toggle(bit, order)\n }\n } // cfg_has_atomic_cas_or_amo32!\n\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];\n /// Returns a mutable pointer to the underlying pointer.\n ///\n /// Returning an `*mut` pointer from a shared reference to this atomic is\n /// safe because the atomic types work with interior mutability. Any use of\n /// the returned raw pointer requires an `unsafe` block and has to uphold\n /// the safety requirements. If there is concurrent access, note the following\n /// additional safety requirements:\n ///\n /// - If this atomic type is [lock-free](Self::is_lock_free), any concurrent\n /// operations on it must be atomic.\n /// - Otherwise, any concurrent operations on it must be compatible with\n /// operations performed by this atomic type.\n ///\n /// This is `const fn` on Rust 1.58+.\n #[inline]\n pub const fn as_ptr(&self) -> *mut *mut T {\n self.inner.as_ptr()\n }\n }\n}\n// See https://github.com/taiki-e/portable-atomic/issues/180\n#[cfg(not(feature = \"require-cas\"))]\ncfg_no_atomic_cas! {\n#[doc(hidden)]\n#[allow(unused_variables, clippy::unused_self, clippy::extra_unused_lifetimes)]\nimpl<'a, T: 'a> AtomicPtr {\n cfg_no_atomic_cas_or_amo32! {\n #[inline]\n pub fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T\n where\n &'a Self: HasSwap,\n {\n unimplemented!()\n }\n } // cfg_no_atomic_cas_or_amo32!\n #[inline]\n pub fn compare_exchange(\n &self,\n current: *mut T,\n new: *mut T,\n success: Ordering,\n failure: Ordering,\n ) -> Result<*mut T, *mut T>\n where\n &'a Self: HasCompareExchange,\n {\n unimplemented!()\n }\n #[inline]\n pub fn compare_exchange_weak(\n &self,\n current: *mut T,\n new: *mut T,\n success: Ordering,\n failure: Ordering,\n ) -> Result<*mut T, *mut T>\n where\n &'a Self: HasCompareExchangeWeak,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_update(\n &self,\n set_order: Ordering,\n fetch_order: Ordering,\n f: F,\n ) -> Result<*mut T, *mut T>\n where\n F: FnMut(*mut T) -> Option<*mut T>,\n &'a Self: HasFetchUpdate,\n {\n unimplemented!()\n }\n cfg_no_atomic_cas_or_amo32! {\n #[inline]\n pub fn fetch_ptr_add(&self, val: usize, order: Ordering) -> *mut T\n where\n &'a Self: HasFetchPtrAdd,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_ptr_sub(&self, val: usize, order: Ordering) -> *mut T\n where\n &'a Self: HasFetchPtrSub,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_byte_add(&self, val: usize, order: Ordering) -> *mut T\n where\n &'a Self: HasFetchByteAdd,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_byte_sub(&self, val: usize, order: Ordering) -> *mut T\n where\n &'a Self: HasFetchByteSub,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_or(&self, val: usize, order: Ordering) -> *mut T\n where\n &'a Self: HasFetchOr,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_and(&self, val: usize, order: Ordering) -> *mut T\n where\n &'a Self: HasFetchAnd,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_xor(&self, val: usize, order: Ordering) -> *mut T\n where\n &'a Self: HasFetchXor,\n {\n unimplemented!()\n }\n #[inline]\n pub fn bit_set(&self, bit: u32, order: Ordering) -> bool\n where\n &'a Self: HasBitSet,\n {\n unimplemented!()\n }\n #[inline]\n pub fn bit_clear(&self, bit: u32, order: Ordering) -> bool\n where\n &'a Self: HasBitClear,\n {\n unimplemented!()\n }\n #[inline]\n pub fn bit_toggle(&self, bit: u32, order: Ordering) -> bool\n where\n &'a Self: HasBitToggle,\n {\n unimplemented!()\n }\n } // cfg_no_atomic_cas_or_amo32!\n}\n} // cfg_no_atomic_cas!\n} // cfg_has_atomic_ptr!\n\nmacro_rules! atomic_int {\n // Atomic{I,U}* impls\n ($atomic_type:ident, $int_type:ident, $align:literal,\n $cfg_has_atomic_cas_or_amo32_or_8:ident, $cfg_no_atomic_cas_or_amo32_or_8:ident\n $(, #[$cfg_float:meta] $atomic_float_type:ident, $float_type:ident)?\n ) => {\n doc_comment! {\n concat!(\"An integer type which can be safely shared between threads.\n\nThis type has the same in-memory representation as the underlying integer type,\n[`\", stringify!($int_type), \"`].\n\nIf the compiler and the platform support atomic loads and stores of [`\", stringify!($int_type),\n\"`], this type is a wrapper for the standard library's `\", stringify!($atomic_type),\n\"`. If the platform supports it but the compiler does not, atomic operations are implemented using\ninline assembly. Otherwise synchronizes using global locks.\nYou can call [`\", stringify!($atomic_type), \"::is_lock_free()`] to check whether\natomic instructions or locks will be used.\n\"\n ),\n // We can use #[repr(transparent)] here, but #[repr(C, align(N))]\n // will show clearer docs.\n #[repr(C, align($align))]\n pub struct $atomic_type {\n inner: imp::$atomic_type,\n }\n }\n\n impl Default for $atomic_type {\n #[inline]\n fn default() -> Self {\n Self::new($int_type::default())\n }\n }\n\n impl From<$int_type> for $atomic_type {\n #[inline]\n fn from(v: $int_type) -> Self {\n Self::new(v)\n }\n }\n\n // UnwindSafe is implicitly implemented.\n #[cfg(not(portable_atomic_no_core_unwind_safe))]\n impl core::panic::RefUnwindSafe for $atomic_type {}\n #[cfg(all(portable_atomic_no_core_unwind_safe, feature = \"std\"))]\n impl std::panic::RefUnwindSafe for $atomic_type {}\n\n impl_debug_and_serde!($atomic_type);\n\n impl $atomic_type {\n doc_comment! {\n concat!(\n \"Creates a new atomic integer.\n\n# Examples\n\n```\nuse portable_atomic::\", stringify!($atomic_type), \";\n\nlet atomic_forty_two = \", stringify!($atomic_type), \"::new(42);\n```\"\n ),\n #[inline]\n #[must_use]\n pub const fn new(v: $int_type) -> Self {\n static_assert_layout!($atomic_type, $int_type);\n Self { inner: imp::$atomic_type::new(v) }\n }\n }\n\n // TODO: update docs based on https://github.com/rust-lang/rust/pull/116762\n #[cfg(not(portable_atomic_no_const_raw_ptr_deref))]\n doc_comment! {\n concat!(\"Creates a new reference to an atomic integer from a pointer.\n\nThis is `const fn` on Rust 1.58+.\n\n# Safety\n\n* `ptr` must be aligned to `align_of::<\", stringify!($atomic_type), \">()` (note that on some platforms this\n can be bigger than `align_of::<\", stringify!($int_type), \">()`).\n* `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`.\n* If this atomic type is [lock-free](Self::is_lock_free), non-atomic accesses to the value\n behind `ptr` must have a happens-before relationship with atomic accesses via\n the returned value (or vice-versa).\n * In other words, time periods where the value is accessed atomically may not\n overlap with periods where the value is accessed non-atomically.\n * This requirement is trivially satisfied if `ptr` is never used non-atomically\n for the duration of lifetime `'a`. Most use cases should be able to follow\n this guideline.\n * This requirement is also trivially satisfied if all accesses (atomic or not) are\n done from the same thread.\n* If this atomic type is *not* lock-free:\n * Any accesses to the value behind `ptr` must have a happens-before relationship\n with accesses via the returned value (or vice-versa).\n * Any concurrent accesses to the value behind `ptr` for the duration of lifetime `'a` must\n be compatible with operations performed by this atomic type.\n* This method must not be used to create overlapping or mixed-size atomic\n accesses, as these are not supported by the memory model.\n\n[valid]: core::ptr#safety\"),\n #[inline]\n #[must_use]\n pub const unsafe fn from_ptr<'a>(ptr: *mut $int_type) -> &'a Self {\n #[allow(clippy::cast_ptr_alignment)]\n // SAFETY: guaranteed by the caller\n unsafe { &*(ptr as *mut Self as *const Self) }\n }\n }\n #[cfg(portable_atomic_no_const_raw_ptr_deref)]\n doc_comment! {\n concat!(\"Creates a new reference to an atomic integer from a pointer.\n\nThis is `const fn` on Rust 1.58+.\n\n# Safety\n\n* `ptr` must be aligned to `align_of::<\", stringify!($atomic_type), \">()` (note that on some platforms this\n can be bigger than `align_of::<\", stringify!($int_type), \">()`).\n* `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`.\n* If this atomic type is [lock-free](Self::is_lock_free), non-atomic accesses to the value\n behind `ptr` must have a happens-before relationship with atomic accesses via\n the returned value (or vice-versa).\n * In other words, time periods where the value is accessed atomically may not\n overlap with periods where the value is accessed non-atomically.\n * This requirement is trivially satisfied if `ptr` is never used non-atomically\n for the duration of lifetime `'a`. Most use cases should be able to follow\n this guideline.\n * This requirement is also trivially satisfied if all accesses (atomic or not) are\n done from the same thread.\n* If this atomic type is *not* lock-free:\n * Any accesses to the value behind `ptr` must have a happens-before relationship\n with accesses via the returned value (or vice-versa).\n * Any concurrent accesses to the value behind `ptr` for the duration of lifetime `'a` must\n be compatible with operations performed by this atomic type.\n* This method must not be used to create overlapping or mixed-size atomic\n accesses, as these are not supported by the memory model.\n\n[valid]: core::ptr#safety\"),\n #[inline]\n #[must_use]\n pub unsafe fn from_ptr<'a>(ptr: *mut $int_type) -> &'a Self {\n #[allow(clippy::cast_ptr_alignment)]\n // SAFETY: guaranteed by the caller\n unsafe { &*(ptr as *mut Self as *const Self) }\n }\n }\n\n doc_comment! {\n concat!(\"Returns `true` if operations on values of this type are lock-free.\n\nIf the compiler or the platform doesn't support the necessary\natomic instructions, global locks for every potentially\nconcurrent atomic operation will be used.\n\nThis function is guaranteed to always return the same result.\n\n# Examples\n\n```\nuse portable_atomic::\", stringify!($atomic_type), \";\n\nlet is_lock_free = \", stringify!($atomic_type), \"::is_lock_free();\n```\"),\n #[inline]\n #[must_use]\n pub fn is_lock_free() -> bool {\n ::is_lock_free()\n }\n }\n\n doc_comment! {\n concat!(\"Returns `true` if operations on values of this type are lock-free.\n\nIf the compiler or the platform doesn't support the necessary\natomic instructions, global locks for every potentially\nconcurrent atomic operation will be used.\n\n**Note:** If the atomic operation relies on dynamic CPU feature detection,\nthis type may be lock-free even if the function returns false.\n\n# Examples\n\n```\nuse portable_atomic::\", stringify!($atomic_type), \";\n\nconst IS_ALWAYS_LOCK_FREE: bool = \", stringify!($atomic_type), \"::is_always_lock_free();\n```\"),\n #[inline]\n #[must_use]\n pub const fn is_always_lock_free() -> bool {\n ::IS_ALWAYS_LOCK_FREE\n }\n }\n #[cfg(test)]\n #[cfg_attr(all(valgrind, target_arch = \"powerpc64\"), allow(dead_code))] // TODO(powerpc64): Hang (as of Valgrind 3.26)\n const IS_ALWAYS_LOCK_FREE: bool = Self::is_always_lock_free();\n\n #[cfg(not(portable_atomic_no_const_mut_refs))]\n doc_comment! {\n concat!(\"Returns a mutable reference to the underlying integer.\\n\nThis is safe because the mutable reference guarantees that no other threads are\nconcurrently accessing the atomic data.\n\nThis is `const fn` on Rust 1.83+.\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet mut some_var = \", stringify!($atomic_type), \"::new(10);\nassert_eq!(*some_var.get_mut(), 10);\n*some_var.get_mut() = 5;\nassert_eq!(some_var.load(Ordering::SeqCst), 5);\n```\"),\n #[inline]\n pub const fn get_mut(&mut self) -> &mut $int_type {\n // SAFETY: the mutable reference guarantees unique ownership.\n // (core::sync::atomic::Atomic*::get_mut is not const yet)\n unsafe { &mut *self.as_ptr() }\n }\n }\n #[cfg(portable_atomic_no_const_mut_refs)]\n doc_comment! {\n concat!(\"Returns a mutable reference to the underlying integer.\\n\nThis is safe because the mutable reference guarantees that no other threads are\nconcurrently accessing the atomic data.\n\nThis is `const fn` on Rust 1.83+.\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet mut some_var = \", stringify!($atomic_type), \"::new(10);\nassert_eq!(*some_var.get_mut(), 10);\n*some_var.get_mut() = 5;\nassert_eq!(some_var.load(Ordering::SeqCst), 5);\n```\"),\n #[inline]\n pub fn get_mut(&mut self) -> &mut $int_type {\n // SAFETY: the mutable reference guarantees unique ownership.\n unsafe { &mut *self.as_ptr() }\n }\n }\n\n // TODO: Add from_mut/get_mut_slice/from_mut_slice once it is stable on std atomic types.\n // https://github.com/rust-lang/rust/issues/76314\n\n #[cfg(not(portable_atomic_no_const_transmute))]\n doc_comment! {\n concat!(\"Consumes the atomic and returns the contained value.\n\nThis is safe because passing `self` by value guarantees that no other threads are\nconcurrently accessing the atomic data.\n\nThis is `const fn` on Rust 1.56+.\n\n# Examples\n\n```\nuse portable_atomic::\", stringify!($atomic_type), \";\n\nlet some_var = \", stringify!($atomic_type), \"::new(5);\nassert_eq!(some_var.into_inner(), 5);\n```\"),\n #[inline]\n pub const fn into_inner(self) -> $int_type {\n // SAFETY: $atomic_type and $int_type have the same size and in-memory representations,\n // so they can be safely transmuted.\n // (const UnsafeCell::into_inner is unstable)\n unsafe { core::mem::transmute(self) }\n }\n }\n #[cfg(portable_atomic_no_const_transmute)]\n doc_comment! {\n concat!(\"Consumes the atomic and returns the contained value.\n\nThis is safe because passing `self` by value guarantees that no other threads are\nconcurrently accessing the atomic data.\n\nThis is `const fn` on Rust 1.56+.\n\n# Examples\n\n```\nuse portable_atomic::\", stringify!($atomic_type), \";\n\nlet some_var = \", stringify!($atomic_type), \"::new(5);\nassert_eq!(some_var.into_inner(), 5);\n```\"),\n #[inline]\n pub fn into_inner(self) -> $int_type {\n // SAFETY: $atomic_type and $int_type have the same size and in-memory representations,\n // so they can be safely transmuted.\n // (const UnsafeCell::into_inner is unstable)\n unsafe { core::mem::transmute(self) }\n }\n }\n\n doc_comment! {\n concat!(\"Loads a value from the atomic integer.\n\n`load` takes an [`Ordering`] argument which describes the memory ordering of this operation.\nPossible values are [`SeqCst`], [`Acquire`] and [`Relaxed`].\n\n# Panics\n\nPanics if `order` is [`Release`] or [`AcqRel`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet some_var = \", stringify!($atomic_type), \"::new(5);\n\nassert_eq!(some_var.load(Ordering::Relaxed), 5);\n```\"),\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn load(&self, order: Ordering) -> $int_type {\n self.inner.load(order)\n }\n }\n\n doc_comment! {\n concat!(\"Stores a value into the atomic integer.\n\n`store` takes an [`Ordering`] argument which describes the memory ordering of this operation.\nPossible values are [`SeqCst`], [`Release`] and [`Relaxed`].\n\n# Panics\n\nPanics if `order` is [`Acquire`] or [`AcqRel`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet some_var = \", stringify!($atomic_type), \"::new(5);\n\nsome_var.store(10, Ordering::Relaxed);\nassert_eq!(some_var.load(Ordering::Relaxed), 10);\n```\"),\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn store(&self, val: $int_type, order: Ordering) {\n self.inner.store(val, order)\n }\n }\n\n cfg_has_atomic_cas_or_amo32! {\n $cfg_has_atomic_cas_or_amo32_or_8! {\n doc_comment! {\n concat!(\"Stores a value into the atomic integer, returning the previous value.\n\n`swap` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet some_var = \", stringify!($atomic_type), \"::new(5);\n\nassert_eq!(some_var.swap(10, Ordering::Relaxed), 5);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn swap(&self, val: $int_type, order: Ordering) -> $int_type {\n self.inner.swap(val, order)\n }\n }\n } // $cfg_has_atomic_cas_or_amo32_or_8!\n\n cfg_has_atomic_cas! {\n doc_comment! {\n concat!(\"Stores a value into the atomic integer if the current value is the same as\nthe `current` value.\n\nThe return value is a result indicating whether the new value was written and\ncontaining the previous value. On success this value is guaranteed to be equal to\n`current`.\n\n`compare_exchange` takes two [`Ordering`] arguments to describe the memory\nordering of this operation. `success` describes the required ordering for the\nread-modify-write operation that takes place if the comparison with `current` succeeds.\n`failure` describes the required ordering for the load operation that takes place when\nthe comparison fails. Using [`Acquire`] as success ordering makes the store part\nof this operation [`Relaxed`], and using [`Release`] makes the successful load\n[`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].\n\n# Panics\n\nPanics if `failure` is [`Release`], [`AcqRel`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet some_var = \", stringify!($atomic_type), \"::new(5);\n\nassert_eq!(\n some_var.compare_exchange(5, 10, Ordering::Acquire, Ordering::Relaxed),\n Ok(5),\n);\nassert_eq!(some_var.load(Ordering::Relaxed), 10);\n\nassert_eq!(\n some_var.compare_exchange(6, 12, Ordering::SeqCst, Ordering::Acquire),\n Err(10),\n);\nassert_eq!(some_var.load(Ordering::Relaxed), 10);\n```\"),\n #[cfg_attr(docsrs, doc(alias = \"compare_and_swap\"))]\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn compare_exchange(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n self.inner.compare_exchange(current, new, success, failure)\n }\n }\n\n doc_comment! {\n concat!(\"Stores a value into the atomic integer if the current value is the same as\nthe `current` value.\nUnlike [`compare_exchange`](Self::compare_exchange)\nthis function is allowed to spuriously fail even\nwhen the comparison succeeds, which can result in more efficient code on some\nplatforms. The return value is a result indicating whether the new value was\nwritten and containing the previous value.\n\n`compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory\nordering of this operation. `success` describes the required ordering for the\nread-modify-write operation that takes place if the comparison with `current` succeeds.\n`failure` describes the required ordering for the load operation that takes place when\nthe comparison fails. Using [`Acquire`] as success ordering makes the store part\nof this operation [`Relaxed`], and using [`Release`] makes the successful load\n[`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].\n\n# Panics\n\nPanics if `failure` is [`Release`], [`AcqRel`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet val = \", stringify!($atomic_type), \"::new(4);\n\nlet mut old = val.load(Ordering::Relaxed);\nloop {\n let new = old * 2;\n match val.compare_exchange_weak(old, new, Ordering::SeqCst, Ordering::Relaxed) {\n Ok(_) => break,\n Err(x) => old = x,\n }\n}\n```\"),\n #[cfg_attr(docsrs, doc(alias = \"compare_and_swap\"))]\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn compare_exchange_weak(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type> {\n self.inner.compare_exchange_weak(current, new, success, failure)\n }\n }\n } // cfg_has_atomic_cas!\n\n $cfg_has_atomic_cas_or_amo32_or_8! {\n doc_comment! {\n concat!(\"Adds to the current value, returning the previous value.\n\nThis operation wraps around on overflow.\n\n`fetch_add` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0);\nassert_eq!(foo.fetch_add(10, Ordering::SeqCst), 0);\nassert_eq!(foo.load(Ordering::SeqCst), 10);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type {\n self.inner.fetch_add(val, order)\n }\n }\n\n doc_comment! {\n concat!(\"Adds to the current value.\n\nThis operation wraps around on overflow.\n\nUnlike `fetch_add`, this does not return the previous value.\n\n`add` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\nThis function may generate more efficient code than `fetch_add` on some platforms.\n\n- MSP430: `add` instead of disabling interrupts ({8,16}-bit atomics)\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0);\nfoo.add(10, Ordering::SeqCst);\nassert_eq!(foo.load(Ordering::SeqCst), 10);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn add(&self, val: $int_type, order: Ordering) {\n self.inner.add(val, order);\n }\n }\n\n doc_comment! {\n concat!(\"Subtracts from the current value, returning the previous value.\n\nThis operation wraps around on overflow.\n\n`fetch_sub` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(20);\nassert_eq!(foo.fetch_sub(10, Ordering::SeqCst), 20);\nassert_eq!(foo.load(Ordering::SeqCst), 10);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type {\n self.inner.fetch_sub(val, order)\n }\n }\n\n doc_comment! {\n concat!(\"Subtracts from the current value.\n\nThis operation wraps around on overflow.\n\nUnlike `fetch_sub`, this does not return the previous value.\n\n`sub` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\nThis function may generate more efficient code than `fetch_sub` on some platforms.\n\n- MSP430: `sub` instead of disabling interrupts ({8,16}-bit atomics)\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(20);\nfoo.sub(10, Ordering::SeqCst);\nassert_eq!(foo.load(Ordering::SeqCst), 10);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn sub(&self, val: $int_type, order: Ordering) {\n self.inner.sub(val, order);\n }\n }\n } // $cfg_has_atomic_cas_or_amo32_or_8!\n\n doc_comment! {\n concat!(\"Bitwise \\\"and\\\" with the current value.\n\nPerforms a bitwise \\\"and\\\" operation on the current value and the argument `val`, and\nsets the new value to the result.\n\nReturns the previous value.\n\n`fetch_and` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0b101101);\nassert_eq!(foo.fetch_and(0b110011, Ordering::SeqCst), 0b101101);\nassert_eq!(foo.load(Ordering::SeqCst), 0b100001);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {\n self.inner.fetch_and(val, order)\n }\n }\n\n doc_comment! {\n concat!(\"Bitwise \\\"and\\\" with the current value.\n\nPerforms a bitwise \\\"and\\\" operation on the current value and the argument `val`, and\nsets the new value to the result.\n\nUnlike `fetch_and`, this does not return the previous value.\n\n`and` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\nThis function may generate more efficient code than `fetch_and` on some platforms.\n\n- x86/x86_64: `lock and` instead of `cmpxchg` loop ({8,16,32}-bit atomics on x86, but additionally 64-bit atomics on x86_64)\n- MSP430: `and` instead of disabling interrupts ({8,16}-bit atomics)\n\nNote: On x86/x86_64, the use of either function should not usually\naffect the generated code, because LLVM can properly optimize the case\nwhere the result is unused.\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0b101101);\nassert_eq!(foo.fetch_and(0b110011, Ordering::SeqCst), 0b101101);\nassert_eq!(foo.load(Ordering::SeqCst), 0b100001);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn and(&self, val: $int_type, order: Ordering) {\n self.inner.and(val, order);\n }\n }\n\n cfg_has_atomic_cas! {\n doc_comment! {\n concat!(\"Bitwise \\\"nand\\\" with the current value.\n\nPerforms a bitwise \\\"nand\\\" operation on the current value and the argument `val`, and\nsets the new value to the result.\n\nReturns the previous value.\n\n`fetch_nand` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0x13);\nassert_eq!(foo.fetch_nand(0x31, Ordering::SeqCst), 0x13);\nassert_eq!(foo.load(Ordering::SeqCst), !(0x13 & 0x31));\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type {\n self.inner.fetch_nand(val, order)\n }\n }\n } // cfg_has_atomic_cas!\n\n doc_comment! {\n concat!(\"Bitwise \\\"or\\\" with the current value.\n\nPerforms a bitwise \\\"or\\\" operation on the current value and the argument `val`, and\nsets the new value to the result.\n\nReturns the previous value.\n\n`fetch_or` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0b101101);\nassert_eq!(foo.fetch_or(0b110011, Ordering::SeqCst), 0b101101);\nassert_eq!(foo.load(Ordering::SeqCst), 0b111111);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {\n self.inner.fetch_or(val, order)\n }\n }\n\n doc_comment! {\n concat!(\"Bitwise \\\"or\\\" with the current value.\n\nPerforms a bitwise \\\"or\\\" operation on the current value and the argument `val`, and\nsets the new value to the result.\n\nUnlike `fetch_or`, this does not return the previous value.\n\n`or` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\nThis function may generate more efficient code than `fetch_or` on some platforms.\n\n- x86/x86_64: `lock or` instead of `cmpxchg` loop ({8,16,32}-bit atomics on x86, but additionally 64-bit atomics on x86_64)\n- MSP430: `or` instead of disabling interrupts ({8,16}-bit atomics)\n\nNote: On x86/x86_64, the use of either function should not usually\naffect the generated code, because LLVM can properly optimize the case\nwhere the result is unused.\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0b101101);\nassert_eq!(foo.fetch_or(0b110011, Ordering::SeqCst), 0b101101);\nassert_eq!(foo.load(Ordering::SeqCst), 0b111111);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn or(&self, val: $int_type, order: Ordering) {\n self.inner.or(val, order);\n }\n }\n\n doc_comment! {\n concat!(\"Bitwise \\\"xor\\\" with the current value.\n\nPerforms a bitwise \\\"xor\\\" operation on the current value and the argument `val`, and\nsets the new value to the result.\n\nReturns the previous value.\n\n`fetch_xor` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0b101101);\nassert_eq!(foo.fetch_xor(0b110011, Ordering::SeqCst), 0b101101);\nassert_eq!(foo.load(Ordering::SeqCst), 0b011110);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {\n self.inner.fetch_xor(val, order)\n }\n }\n\n doc_comment! {\n concat!(\"Bitwise \\\"xor\\\" with the current value.\n\nPerforms a bitwise \\\"xor\\\" operation on the current value and the argument `val`, and\nsets the new value to the result.\n\nUnlike `fetch_xor`, this does not return the previous value.\n\n`xor` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\nThis function may generate more efficient code than `fetch_xor` on some platforms.\n\n- x86/x86_64: `lock xor` instead of `cmpxchg` loop ({8,16,32}-bit atomics on x86, but additionally 64-bit atomics on x86_64)\n- MSP430: `xor` instead of disabling interrupts ({8,16}-bit atomics)\n\nNote: On x86/x86_64, the use of either function should not usually\naffect the generated code, because LLVM can properly optimize the case\nwhere the result is unused.\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0b101101);\nfoo.xor(0b110011, Ordering::SeqCst);\nassert_eq!(foo.load(Ordering::SeqCst), 0b011110);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn xor(&self, val: $int_type, order: Ordering) {\n self.inner.xor(val, order);\n }\n }\n\n cfg_has_atomic_cas! {\n doc_comment! {\n concat!(\"Fetches the value, and applies a function to it that returns an optional\nnew value. Returns a `Result` of `Ok(previous_value)` if the function returned `Some(_)`, else\n`Err(previous_value)`.\n\nNote: This may call the function multiple times if the value has been changed from other threads in\nthe meantime, as long as the function returns `Some(_)`, but the function will have been applied\nonly once to the stored value.\n\n`fetch_update` takes two [`Ordering`] arguments to describe the memory ordering of this operation.\nThe first describes the required ordering for when the operation finally succeeds while the second\ndescribes the required ordering for loads. These correspond to the success and failure orderings of\n[`compare_exchange`](Self::compare_exchange) respectively.\n\nUsing [`Acquire`] as success ordering makes the store part\nof this operation [`Relaxed`], and using [`Release`] makes the final successful load\n[`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].\n\n# Panics\n\nPanics if `fetch_order` is [`Release`], [`AcqRel`].\n\n# Considerations\n\nThis method is not magic; it is not provided by the hardware.\nIt is implemented in terms of [`compare_exchange_weak`](Self::compare_exchange_weak),\nand suffers from the same drawbacks.\nIn particular, this method will not circumvent the [ABA Problem].\n\n[ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet x = \", stringify!($atomic_type), \"::new(7);\nassert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |_| None), Err(7));\nassert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(x + 1)), Ok(7));\nassert_eq!(x.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |x| Some(x + 1)), Ok(8));\nassert_eq!(x.load(Ordering::SeqCst), 9);\n```\"),\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn fetch_update(\n &self,\n set_order: Ordering,\n fetch_order: Ordering,\n mut f: F,\n ) -> Result<$int_type, $int_type>\n where\n F: FnMut($int_type) -> Option<$int_type>,\n {\n let mut prev = self.load(fetch_order);\n while let Some(next) = f(prev) {\n match self.compare_exchange_weak(prev, next, set_order, fetch_order) {\n x @ Ok(_) => return x,\n Err(next_prev) => prev = next_prev,\n }\n }\n Err(prev)\n }\n }\n } // cfg_has_atomic_cas!\n\n $cfg_has_atomic_cas_or_amo32_or_8! {\n doc_comment! {\n concat!(\"Maximum with the current value.\n\nFinds the maximum of the current value and the argument `val`, and\nsets the new value to the result.\n\nReturns the previous value.\n\n`fetch_max` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(23);\nassert_eq!(foo.fetch_max(42, Ordering::SeqCst), 23);\nassert_eq!(foo.load(Ordering::SeqCst), 42);\n```\n\nIf you want to obtain the maximum value in one step, you can use the following:\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(23);\nlet bar = 42;\nlet max_foo = foo.fetch_max(bar, Ordering::SeqCst).max(bar);\nassert!(max_foo == 42);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {\n self.inner.fetch_max(val, order)\n }\n }\n\n doc_comment! {\n concat!(\"Minimum with the current value.\n\nFinds the minimum of the current value and the argument `val`, and\nsets the new value to the result.\n\nReturns the previous value.\n\n`fetch_min` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(23);\nassert_eq!(foo.fetch_min(42, Ordering::Relaxed), 23);\nassert_eq!(foo.load(Ordering::Relaxed), 23);\nassert_eq!(foo.fetch_min(22, Ordering::Relaxed), 23);\nassert_eq!(foo.load(Ordering::Relaxed), 22);\n```\n\nIf you want to obtain the minimum value in one step, you can use the following:\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(23);\nlet bar = 12;\nlet min_foo = foo.fetch_min(bar, Ordering::SeqCst).min(bar);\nassert_eq!(min_foo, 12);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {\n self.inner.fetch_min(val, order)\n }\n }\n } // $cfg_has_atomic_cas_or_amo32_or_8!\n\n doc_comment! {\n concat!(\"Sets the bit at the specified bit-position to 1.\n\nReturns `true` if the specified bit was previously set to 1.\n\n`bit_set` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\nThis corresponds to x86's `lock bts`, and the implementation calls them on x86/x86_64.\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0b0000);\nassert!(!foo.bit_set(0, Ordering::Relaxed));\nassert_eq!(foo.load(Ordering::Relaxed), 0b0001);\nassert!(foo.bit_set(0, Ordering::Relaxed));\nassert_eq!(foo.load(Ordering::Relaxed), 0b0001);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn bit_set(&self, bit: u32, order: Ordering) -> bool {\n self.inner.bit_set(bit, order)\n }\n }\n\n doc_comment! {\n concat!(\"Clears the bit at the specified bit-position to 0.\n\nReturns `true` if the specified bit was previously set to 1.\n\n`bit_clear` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\nThis corresponds to x86's `lock btr`, and the implementation calls them on x86/x86_64.\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0b0001);\nassert!(foo.bit_clear(0, Ordering::Relaxed));\nassert_eq!(foo.load(Ordering::Relaxed), 0b0000);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn bit_clear(&self, bit: u32, order: Ordering) -> bool {\n self.inner.bit_clear(bit, order)\n }\n }\n\n doc_comment! {\n concat!(\"Toggles the bit at the specified bit-position.\n\nReturns `true` if the specified bit was previously set to 1.\n\n`bit_toggle` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\nThis corresponds to x86's `lock btc`, and the implementation calls them on x86/x86_64.\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0b0000);\nassert!(!foo.bit_toggle(0, Ordering::Relaxed));\nassert_eq!(foo.load(Ordering::Relaxed), 0b0001);\nassert!(foo.bit_toggle(0, Ordering::Relaxed));\nassert_eq!(foo.load(Ordering::Relaxed), 0b0000);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn bit_toggle(&self, bit: u32, order: Ordering) -> bool {\n self.inner.bit_toggle(bit, order)\n }\n }\n\n doc_comment! {\n concat!(\"Logical negates the current value, and sets the new value to the result.\n\nReturns the previous value.\n\n`fetch_not` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0);\nassert_eq!(foo.fetch_not(Ordering::Relaxed), 0);\nassert_eq!(foo.load(Ordering::Relaxed), !0);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_not(&self, order: Ordering) -> $int_type {\n self.inner.fetch_not(order)\n }\n }\n\n doc_comment! {\n concat!(\"Logical negates the current value, and sets the new value to the result.\n\nUnlike `fetch_not`, this does not return the previous value.\n\n`not` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\nThis function may generate more efficient code than `fetch_not` on some platforms.\n\n- x86/x86_64: `lock not` instead of `cmpxchg` loop ({8,16,32}-bit atomics on x86, but additionally 64-bit atomics on x86_64)\n- MSP430: `inv` instead of disabling interrupts ({8,16}-bit atomics)\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(0);\nfoo.not(Ordering::Relaxed);\nassert_eq!(foo.load(Ordering::Relaxed), !0);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn not(&self, order: Ordering) {\n self.inner.not(order);\n }\n }\n\n cfg_has_atomic_cas! {\n doc_comment! {\n concat!(\"Negates the current value, and sets the new value to the result.\n\nReturns the previous value.\n\n`fetch_neg` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(5);\nassert_eq!(foo.fetch_neg(Ordering::Relaxed), 5);\nassert_eq!(foo.load(Ordering::Relaxed), 5_\", stringify!($int_type), \".wrapping_neg());\nassert_eq!(foo.fetch_neg(Ordering::Relaxed), 5_\", stringify!($int_type), \".wrapping_neg());\nassert_eq!(foo.load(Ordering::Relaxed), 5);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_neg(&self, order: Ordering) -> $int_type {\n self.inner.fetch_neg(order)\n }\n }\n\n doc_comment! {\n concat!(\"Negates the current value, and sets the new value to the result.\n\nUnlike `fetch_neg`, this does not return the previous value.\n\n`neg` takes an [`Ordering`] argument which describes the memory ordering\nof this operation. All ordering modes are possible. Note that using\n[`Acquire`] makes the store part of this operation [`Relaxed`], and\nusing [`Release`] makes the load part [`Relaxed`].\n\nThis function may generate more efficient code than `fetch_neg` on some platforms.\n\n- x86/x86_64: `lock neg` instead of `cmpxchg` loop ({8,16,32}-bit atomics on x86, but additionally 64-bit atomics on x86_64)\n\n# Examples\n\n```\nuse portable_atomic::{\", stringify!($atomic_type), \", Ordering};\n\nlet foo = \", stringify!($atomic_type), \"::new(5);\nfoo.neg(Ordering::Relaxed);\nassert_eq!(foo.load(Ordering::Relaxed), 5_\", stringify!($int_type), \".wrapping_neg());\nfoo.neg(Ordering::Relaxed);\nassert_eq!(foo.load(Ordering::Relaxed), 5);\n```\"),\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn neg(&self, order: Ordering) {\n self.inner.neg(order);\n }\n }\n } // cfg_has_atomic_cas!\n } // cfg_has_atomic_cas_or_amo32!\n\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];\n /// Returns a mutable pointer to the underlying integer.\n ///\n /// Returning an `*mut` pointer from a shared reference to this atomic is\n /// safe because the atomic types work with interior mutability. Any use of\n /// the returned raw pointer requires an `unsafe` block and has to uphold\n /// the safety requirements. If there is concurrent access, note the following\n /// additional safety requirements:\n ///\n /// - If this atomic type is [lock-free](Self::is_lock_free), any concurrent\n /// operations on it must be atomic.\n /// - Otherwise, any concurrent operations on it must be compatible with\n /// operations performed by this atomic type.\n ///\n /// This is `const fn` on Rust 1.58+.\n #[inline]\n pub const fn as_ptr(&self) -> *mut $int_type {\n self.inner.as_ptr()\n }\n }\n }\n // See https://github.com/taiki-e/portable-atomic/issues/180\n #[cfg(not(feature = \"require-cas\"))]\n cfg_no_atomic_cas! {\n #[doc(hidden)]\n #[allow(unused_variables, clippy::unused_self, clippy::extra_unused_lifetimes)]\n impl<'a> $atomic_type {\n $cfg_no_atomic_cas_or_amo32_or_8! {\n #[inline]\n pub fn swap(&self, val: $int_type, order: Ordering) -> $int_type\n where\n &'a Self: HasSwap,\n {\n unimplemented!()\n }\n } // $cfg_no_atomic_cas_or_amo32_or_8!\n #[inline]\n pub fn compare_exchange(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type>\n where\n &'a Self: HasCompareExchange,\n {\n unimplemented!()\n }\n #[inline]\n pub fn compare_exchange_weak(\n &self,\n current: $int_type,\n new: $int_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$int_type, $int_type>\n where\n &'a Self: HasCompareExchangeWeak,\n {\n unimplemented!()\n }\n $cfg_no_atomic_cas_or_amo32_or_8! {\n #[inline]\n pub fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type\n where\n &'a Self: HasFetchAdd,\n {\n unimplemented!()\n }\n #[inline]\n pub fn add(&self, val: $int_type, order: Ordering)\n where\n &'a Self: HasAdd,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type\n where\n &'a Self: HasFetchSub,\n {\n unimplemented!()\n }\n #[inline]\n pub fn sub(&self, val: $int_type, order: Ordering)\n where\n &'a Self: HasSub,\n {\n unimplemented!()\n }\n } // $cfg_no_atomic_cas_or_amo32_or_8!\n cfg_no_atomic_cas_or_amo32! {\n #[inline]\n pub fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type\n where\n &'a Self: HasFetchAnd,\n {\n unimplemented!()\n }\n #[inline]\n pub fn and(&self, val: $int_type, order: Ordering)\n where\n &'a Self: HasAnd,\n {\n unimplemented!()\n }\n } // cfg_no_atomic_cas_or_amo32!\n #[inline]\n pub fn fetch_nand(&self, val: $int_type, order: Ordering) -> $int_type\n where\n &'a Self: HasFetchNand,\n {\n unimplemented!()\n }\n cfg_no_atomic_cas_or_amo32! {\n #[inline]\n pub fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type\n where\n &'a Self: HasFetchOr,\n {\n unimplemented!()\n }\n #[inline]\n pub fn or(&self, val: $int_type, order: Ordering)\n where\n &'a Self: HasOr,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type\n where\n &'a Self: HasFetchXor,\n {\n unimplemented!()\n }\n #[inline]\n pub fn xor(&self, val: $int_type, order: Ordering)\n where\n &'a Self: HasXor,\n {\n unimplemented!()\n }\n } // cfg_no_atomic_cas_or_amo32!\n #[inline]\n pub fn fetch_update(\n &self,\n set_order: Ordering,\n fetch_order: Ordering,\n f: F,\n ) -> Result<$int_type, $int_type>\n where\n F: FnMut($int_type) -> Option<$int_type>,\n &'a Self: HasFetchUpdate,\n {\n unimplemented!()\n }\n $cfg_no_atomic_cas_or_amo32_or_8! {\n #[inline]\n pub fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type\n where\n &'a Self: HasFetchMax,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type\n where\n &'a Self: HasFetchMin,\n {\n unimplemented!()\n }\n } // $cfg_no_atomic_cas_or_amo32_or_8!\n cfg_no_atomic_cas_or_amo32! {\n #[inline]\n pub fn bit_set(&self, bit: u32, order: Ordering) -> bool\n where\n &'a Self: HasBitSet,\n {\n unimplemented!()\n }\n #[inline]\n pub fn bit_clear(&self, bit: u32, order: Ordering) -> bool\n where\n &'a Self: HasBitClear,\n {\n unimplemented!()\n }\n #[inline]\n pub fn bit_toggle(&self, bit: u32, order: Ordering) -> bool\n where\n &'a Self: HasBitToggle,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_not(&self, order: Ordering) -> $int_type\n where\n &'a Self: HasFetchNot,\n {\n unimplemented!()\n }\n #[inline]\n pub fn not(&self, order: Ordering)\n where\n &'a Self: HasNot,\n {\n unimplemented!()\n }\n } // cfg_no_atomic_cas_or_amo32!\n #[inline]\n pub fn fetch_neg(&self, order: Ordering) -> $int_type\n where\n &'a Self: HasFetchNeg,\n {\n unimplemented!()\n }\n #[inline]\n pub fn neg(&self, order: Ordering)\n where\n &'a Self: HasNeg,\n {\n unimplemented!()\n }\n }\n } // cfg_no_atomic_cas!\n $(\n #[$cfg_float]\n atomic_int!(float,\n #[$cfg_float] $atomic_float_type, $float_type, $atomic_type, $int_type, $align\n );\n )?\n };\n\n // AtomicF* impls\n (float,\n #[$cfg_float:meta]\n $atomic_type:ident,\n $float_type:ident,\n $atomic_int_type:ident,\n $int_type:ident,\n $align:literal\n ) => {\n doc_comment! {\n concat!(\"A floating point type which can be safely shared between threads.\n\nThis type has the same in-memory representation as the underlying floating point type,\n[`\", stringify!($float_type), \"`].\n\"\n ),\n #[cfg_attr(docsrs, doc($cfg_float))]\n // We can use #[repr(transparent)] here, but #[repr(C, align(N))]\n // will show clearer docs.\n #[repr(C, align($align))]\n pub struct $atomic_type {\n inner: imp::float::$atomic_type,\n }\n }\n\n impl Default for $atomic_type {\n #[inline]\n fn default() -> Self {\n Self::new($float_type::default())\n }\n }\n\n impl From<$float_type> for $atomic_type {\n #[inline]\n fn from(v: $float_type) -> Self {\n Self::new(v)\n }\n }\n\n // UnwindSafe is implicitly implemented.\n #[cfg(not(portable_atomic_no_core_unwind_safe))]\n impl core::panic::RefUnwindSafe for $atomic_type {}\n #[cfg(all(portable_atomic_no_core_unwind_safe, feature = \"std\"))]\n impl std::panic::RefUnwindSafe for $atomic_type {}\n\n impl_debug_and_serde!($atomic_type);\n\n impl $atomic_type {\n /// Creates a new atomic float.\n #[inline]\n #[must_use]\n pub const fn new(v: $float_type) -> Self {\n static_assert_layout!($atomic_type, $float_type);\n Self { inner: imp::float::$atomic_type::new(v) }\n }\n\n // TODO: update docs based on https://github.com/rust-lang/rust/pull/116762\n #[cfg(not(portable_atomic_no_const_raw_ptr_deref))]\n doc_comment! {\n concat!(\"Creates a new reference to an atomic float from a pointer.\n\nThis is `const fn` on Rust 1.58+.\n\n# Safety\n\n* `ptr` must be aligned to `align_of::<\", stringify!($atomic_type), \">()` (note that on some platforms this\n can be bigger than `align_of::<\", stringify!($float_type), \">()`).\n* `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`.\n* If this atomic type is [lock-free](Self::is_lock_free), non-atomic accesses to the value\n behind `ptr` must have a happens-before relationship with atomic accesses via\n the returned value (or vice-versa).\n * In other words, time periods where the value is accessed atomically may not\n overlap with periods where the value is accessed non-atomically.\n * This requirement is trivially satisfied if `ptr` is never used non-atomically\n for the duration of lifetime `'a`. Most use cases should be able to follow\n this guideline.\n * This requirement is also trivially satisfied if all accesses (atomic or not) are\n done from the same thread.\n* If this atomic type is *not* lock-free:\n * Any accesses to the value behind `ptr` must have a happens-before relationship\n with accesses via the returned value (or vice-versa).\n * Any concurrent accesses to the value behind `ptr` for the duration of lifetime `'a` must\n be compatible with operations performed by this atomic type.\n* This method must not be used to create overlapping or mixed-size atomic\n accesses, as these are not supported by the memory model.\n\n[valid]: core::ptr#safety\"),\n #[inline]\n #[must_use]\n pub const unsafe fn from_ptr<'a>(ptr: *mut $float_type) -> &'a Self {\n #[allow(clippy::cast_ptr_alignment)]\n // SAFETY: guaranteed by the caller\n unsafe { &*(ptr as *mut Self as *const Self) }\n }\n }\n #[cfg(portable_atomic_no_const_raw_ptr_deref)]\n doc_comment! {\n concat!(\"Creates a new reference to an atomic float from a pointer.\n\nThis is `const fn` on Rust 1.58+.\n\n# Safety\n\n* `ptr` must be aligned to `align_of::<\", stringify!($atomic_type), \">()` (note that on some platforms this\n can be bigger than `align_of::<\", stringify!($float_type), \">()`).\n* `ptr` must be [valid] for both reads and writes for the whole lifetime `'a`.\n* If this atomic type is [lock-free](Self::is_lock_free), non-atomic accesses to the value\n behind `ptr` must have a happens-before relationship with atomic accesses via\n the returned value (or vice-versa).\n * In other words, time periods where the value is accessed atomically may not\n overlap with periods where the value is accessed non-atomically.\n * This requirement is trivially satisfied if `ptr` is never used non-atomically\n for the duration of lifetime `'a`. Most use cases should be able to follow\n this guideline.\n * This requirement is also trivially satisfied if all accesses (atomic or not) are\n done from the same thread.\n* If this atomic type is *not* lock-free:\n * Any accesses to the value behind `ptr` must have a happens-before relationship\n with accesses via the returned value (or vice-versa).\n * Any concurrent accesses to the value behind `ptr` for the duration of lifetime `'a` must\n be compatible with operations performed by this atomic type.\n* This method must not be used to create overlapping or mixed-size atomic\n accesses, as these are not supported by the memory model.\n\n[valid]: core::ptr#safety\"),\n #[inline]\n #[must_use]\n pub unsafe fn from_ptr<'a>(ptr: *mut $float_type) -> &'a Self {\n #[allow(clippy::cast_ptr_alignment)]\n // SAFETY: guaranteed by the caller\n unsafe { &*(ptr as *mut Self as *const Self) }\n }\n }\n\n /// Returns `true` if operations on values of this type are lock-free.\n ///\n /// If the compiler or the platform doesn't support the necessary\n /// atomic instructions, global locks for every potentially\n /// concurrent atomic operation will be used.\n ///\n /// This function is guaranteed to always return the same result.\n #[inline]\n #[must_use]\n pub fn is_lock_free() -> bool {\n ::is_lock_free()\n }\n\n /// Returns `true` if operations on values of this type are lock-free.\n ///\n /// If the compiler or the platform doesn't support the necessary\n /// atomic instructions, global locks for every potentially\n /// concurrent atomic operation will be used.\n ///\n /// **Note:** If the atomic operation relies on dynamic CPU feature detection,\n /// this type may be lock-free even if the function returns false.\n #[inline]\n #[must_use]\n pub const fn is_always_lock_free() -> bool {\n ::IS_ALWAYS_LOCK_FREE\n }\n #[cfg(test)]\n #[cfg_attr(all(not(debug_assertions), target_arch = \"x86\", not(target_feature = \"sse2\")), allow(dead_code))]\n const IS_ALWAYS_LOCK_FREE: bool = Self::is_always_lock_free();\n\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_mut_refs))];\n /// Returns a mutable reference to the underlying float.\n ///\n /// This is safe because the mutable reference guarantees that no other threads are\n /// concurrently accessing the atomic data.\n ///\n /// This is `const fn` on Rust 1.83+.\n #[inline]\n pub const fn get_mut(&mut self) -> &mut $float_type {\n // SAFETY: the mutable reference guarantees unique ownership.\n unsafe { &mut *self.as_ptr() }\n }\n }\n\n // TODO: Add from_mut/get_mut_slice/from_mut_slice once it is stable on std atomic types.\n // https://github.com/rust-lang/rust/issues/76314\n\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_transmute))];\n /// Consumes the atomic and returns the contained value.\n ///\n /// This is safe because passing `self` by value guarantees that no other threads are\n /// concurrently accessing the atomic data.\n ///\n /// This is `const fn` on Rust 1.56+.\n #[inline]\n pub const fn into_inner(self) -> $float_type {\n // SAFETY: $atomic_type and $float_type have the same size and in-memory representations,\n // so they can be safely transmuted.\n // (const UnsafeCell::into_inner is unstable)\n unsafe { core::mem::transmute(self) }\n }\n }\n\n /// Loads a value from the atomic float.\n ///\n /// `load` takes an [`Ordering`] argument which describes the memory ordering of this operation.\n /// Possible values are [`SeqCst`], [`Acquire`] and [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `order` is [`Release`] or [`AcqRel`].\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn load(&self, order: Ordering) -> $float_type {\n self.inner.load(order)\n }\n\n /// Stores a value into the atomic float.\n ///\n /// `store` takes an [`Ordering`] argument which describes the memory ordering of this operation.\n /// Possible values are [`SeqCst`], [`Release`] and [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `order` is [`Acquire`] or [`AcqRel`].\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn store(&self, val: $float_type, order: Ordering) {\n self.inner.store(val, order)\n }\n\n cfg_has_atomic_cas_or_amo32! {\n /// Stores a value into the atomic float, returning the previous value.\n ///\n /// `swap` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn swap(&self, val: $float_type, order: Ordering) -> $float_type {\n self.inner.swap(val, order)\n }\n\n cfg_has_atomic_cas! {\n /// Stores a value into the atomic float if the current value is the same as\n /// the `current` value.\n ///\n /// The return value is a result indicating whether the new value was written and\n /// containing the previous value. On success this value is guaranteed to be equal to\n /// `current`.\n ///\n /// `compare_exchange` takes two [`Ordering`] arguments to describe the memory\n /// ordering of this operation. `success` describes the required ordering for the\n /// read-modify-write operation that takes place if the comparison with `current` succeeds.\n /// `failure` describes the required ordering for the load operation that takes place when\n /// the comparison fails. Using [`Acquire`] as success ordering makes the store part\n /// of this operation [`Relaxed`], and using [`Release`] makes the successful load\n /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `failure` is [`Release`], [`AcqRel`].\n #[cfg_attr(docsrs, doc(alias = \"compare_and_swap\"))]\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn compare_exchange(\n &self,\n current: $float_type,\n new: $float_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$float_type, $float_type> {\n self.inner.compare_exchange(current, new, success, failure)\n }\n\n /// Stores a value into the atomic float if the current value is the same as\n /// the `current` value.\n /// Unlike [`compare_exchange`](Self::compare_exchange)\n /// this function is allowed to spuriously fail even\n /// when the comparison succeeds, which can result in more efficient code on some\n /// platforms. The return value is a result indicating whether the new value was\n /// written and containing the previous value.\n ///\n /// `compare_exchange_weak` takes two [`Ordering`] arguments to describe the memory\n /// ordering of this operation. `success` describes the required ordering for the\n /// read-modify-write operation that takes place if the comparison with `current` succeeds.\n /// `failure` describes the required ordering for the load operation that takes place when\n /// the comparison fails. Using [`Acquire`] as success ordering makes the store part\n /// of this operation [`Relaxed`], and using [`Release`] makes the successful load\n /// [`Relaxed`]. The failure ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `failure` is [`Release`], [`AcqRel`].\n #[cfg_attr(docsrs, doc(alias = \"compare_and_swap\"))]\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn compare_exchange_weak(\n &self,\n current: $float_type,\n new: $float_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$float_type, $float_type> {\n self.inner.compare_exchange_weak(current, new, success, failure)\n }\n\n /// Adds to the current value, returning the previous value.\n ///\n /// `fetch_add` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_add(&self, val: $float_type, order: Ordering) -> $float_type {\n self.inner.fetch_add(val, order)\n }\n\n /// Subtracts from the current value, returning the previous value.\n ///\n /// `fetch_sub` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_sub(&self, val: $float_type, order: Ordering) -> $float_type {\n self.inner.fetch_sub(val, order)\n }\n\n /// Fetches the value, and applies a function to it that returns an optional\n /// new value. Returns a `Result` of `Ok(previous_value)` if the function returned `Some(_)`, else\n /// `Err(previous_value)`.\n ///\n /// Note: This may call the function multiple times if the value has been changed from other threads in\n /// the meantime, as long as the function returns `Some(_)`, but the function will have been applied\n /// only once to the stored value.\n ///\n /// `fetch_update` takes two [`Ordering`] arguments to describe the memory ordering of this operation.\n /// The first describes the required ordering for when the operation finally succeeds while the second\n /// describes the required ordering for loads. These correspond to the success and failure orderings of\n /// [`compare_exchange`](Self::compare_exchange) respectively.\n ///\n /// Using [`Acquire`] as success ordering makes the store part\n /// of this operation [`Relaxed`], and using [`Release`] makes the final successful load\n /// [`Relaxed`]. The (failed) load ordering can only be [`SeqCst`], [`Acquire`] or [`Relaxed`].\n ///\n /// # Panics\n ///\n /// Panics if `fetch_order` is [`Release`], [`AcqRel`].\n ///\n /// # Considerations\n ///\n /// This method is not magic; it is not provided by the hardware.\n /// It is implemented in terms of [`compare_exchange_weak`](Self::compare_exchange_weak),\n /// and suffers from the same drawbacks.\n /// In particular, this method will not circumvent the [ABA Problem].\n ///\n /// [ABA Problem]: https://en.wikipedia.org/wiki/ABA_problem\n #[inline]\n #[cfg_attr(\n any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri),\n track_caller\n )]\n pub fn fetch_update(\n &self,\n set_order: Ordering,\n fetch_order: Ordering,\n mut f: F,\n ) -> Result<$float_type, $float_type>\n where\n F: FnMut($float_type) -> Option<$float_type>,\n {\n let mut prev = self.load(fetch_order);\n while let Some(next) = f(prev) {\n match self.compare_exchange_weak(prev, next, set_order, fetch_order) {\n x @ Ok(_) => return x,\n Err(next_prev) => prev = next_prev,\n }\n }\n Err(prev)\n }\n\n /// Maximum with the current value.\n ///\n /// Finds the maximum of the current value and the argument `val`, and\n /// sets the new value to the result.\n ///\n /// Returns the previous value.\n ///\n /// `fetch_max` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_max(&self, val: $float_type, order: Ordering) -> $float_type {\n self.inner.fetch_max(val, order)\n }\n\n /// Minimum with the current value.\n ///\n /// Finds the minimum of the current value and the argument `val`, and\n /// sets the new value to the result.\n ///\n /// Returns the previous value.\n ///\n /// `fetch_min` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_min(&self, val: $float_type, order: Ordering) -> $float_type {\n self.inner.fetch_min(val, order)\n }\n } // cfg_has_atomic_cas!\n\n /// Negates the current value, and sets the new value to the result.\n ///\n /// Returns the previous value.\n ///\n /// `fetch_neg` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_neg(&self, order: Ordering) -> $float_type {\n self.inner.fetch_neg(order)\n }\n\n /// Computes the absolute value of the current value, and sets the\n /// new value to the result.\n ///\n /// Returns the previous value.\n ///\n /// `fetch_abs` takes an [`Ordering`] argument which describes the memory ordering\n /// of this operation. All ordering modes are possible. Note that using\n /// [`Acquire`] makes the store part of this operation [`Relaxed`], and\n /// using [`Release`] makes the load part [`Relaxed`].\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub fn fetch_abs(&self, order: Ordering) -> $float_type {\n self.inner.fetch_abs(order)\n }\n } // cfg_has_atomic_cas_or_amo32!\n\n #[cfg(not(portable_atomic_no_const_raw_ptr_deref))]\n doc_comment! {\n concat!(\"Raw transmutation to `&\", stringify!($atomic_int_type), \"`.\n\nSee [`\", stringify!($float_type) ,\"::from_bits`] for some discussion of the\nportability of this operation (there are almost no issues).\n\nThis is `const fn` on Rust 1.58+.\"),\n #[inline]\n pub const fn as_bits(&self) -> &$atomic_int_type {\n self.inner.as_bits()\n }\n }\n #[cfg(portable_atomic_no_const_raw_ptr_deref)]\n doc_comment! {\n concat!(\"Raw transmutation to `&\", stringify!($atomic_int_type), \"`.\n\nSee [`\", stringify!($float_type) ,\"::from_bits`] for some discussion of the\nportability of this operation (there are almost no issues).\n\nThis is `const fn` on Rust 1.58+.\"),\n #[inline]\n pub fn as_bits(&self) -> &$atomic_int_type {\n self.inner.as_bits()\n }\n }\n\n const_fn! {\n const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))];\n /// Returns a mutable pointer to the underlying float.\n ///\n /// Returning an `*mut` pointer from a shared reference to this atomic is\n /// safe because the atomic types work with interior mutability. Any use of\n /// the returned raw pointer requires an `unsafe` block and has to uphold\n /// the safety requirements. If there is concurrent access, note the following\n /// additional safety requirements:\n ///\n /// - If this atomic type is [lock-free](Self::is_lock_free), any concurrent\n /// operations on it must be atomic.\n /// - Otherwise, any concurrent operations on it must be compatible with\n /// operations performed by this atomic type.\n ///\n /// This is `const fn` on Rust 1.58+.\n #[inline]\n pub const fn as_ptr(&self) -> *mut $float_type {\n self.inner.as_ptr()\n }\n }\n }\n // See https://github.com/taiki-e/portable-atomic/issues/180\n #[cfg(not(feature = \"require-cas\"))]\n cfg_no_atomic_cas! {\n #[doc(hidden)]\n #[allow(unused_variables, clippy::unused_self, clippy::extra_unused_lifetimes)]\n impl<'a> $atomic_type {\n cfg_no_atomic_cas_or_amo32! {\n #[inline]\n pub fn swap(&self, val: $float_type, order: Ordering) -> $float_type\n where\n &'a Self: HasSwap,\n {\n unimplemented!()\n }\n } // cfg_no_atomic_cas_or_amo32!\n #[inline]\n pub fn compare_exchange(\n &self,\n current: $float_type,\n new: $float_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$float_type, $float_type>\n where\n &'a Self: HasCompareExchange,\n {\n unimplemented!()\n }\n #[inline]\n pub fn compare_exchange_weak(\n &self,\n current: $float_type,\n new: $float_type,\n success: Ordering,\n failure: Ordering,\n ) -> Result<$float_type, $float_type>\n where\n &'a Self: HasCompareExchangeWeak,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_add(&self, val: $float_type, order: Ordering) -> $float_type\n where\n &'a Self: HasFetchAdd,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_sub(&self, val: $float_type, order: Ordering) -> $float_type\n where\n &'a Self: HasFetchSub,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_update(\n &self,\n set_order: Ordering,\n fetch_order: Ordering,\n f: F,\n ) -> Result<$float_type, $float_type>\n where\n F: FnMut($float_type) -> Option<$float_type>,\n &'a Self: HasFetchUpdate,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_max(&self, val: $float_type, order: Ordering) -> $float_type\n where\n &'a Self: HasFetchMax,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_min(&self, val: $float_type, order: Ordering) -> $float_type\n where\n &'a Self: HasFetchMin,\n {\n unimplemented!()\n }\n cfg_no_atomic_cas_or_amo32! {\n #[inline]\n pub fn fetch_neg(&self, order: Ordering) -> $float_type\n where\n &'a Self: HasFetchNeg,\n {\n unimplemented!()\n }\n #[inline]\n pub fn fetch_abs(&self, order: Ordering) -> $float_type\n where\n &'a Self: HasFetchAbs,\n {\n unimplemented!()\n }\n } // cfg_no_atomic_cas_or_amo32!\n }\n } // cfg_no_atomic_cas!\n };\n}\n\ncfg_has_atomic_ptr! {\n #[cfg(target_pointer_width = \"16\")]\n atomic_int!(AtomicIsize, isize, 2, cfg_has_atomic_cas_or_amo8, cfg_no_atomic_cas_or_amo8);\n #[cfg(target_pointer_width = \"16\")]\n atomic_int!(AtomicUsize, usize, 2, cfg_has_atomic_cas_or_amo8, cfg_no_atomic_cas_or_amo8);\n #[cfg(target_pointer_width = \"32\")]\n atomic_int!(AtomicIsize, isize, 4, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32);\n #[cfg(target_pointer_width = \"32\")]\n atomic_int!(AtomicUsize, usize, 4, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32);\n #[cfg(target_pointer_width = \"64\")]\n atomic_int!(AtomicIsize, isize, 8, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32);\n #[cfg(target_pointer_width = \"64\")]\n atomic_int!(AtomicUsize, usize, 8, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32);\n #[cfg(target_pointer_width = \"128\")]\n atomic_int!(AtomicIsize, isize, 16, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32);\n #[cfg(target_pointer_width = \"128\")]\n atomic_int!(AtomicUsize, usize, 16, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32);\n}\n\ncfg_has_atomic_8! {\n atomic_int!(AtomicI8, i8, 1, cfg_has_atomic_cas_or_amo8, cfg_no_atomic_cas_or_amo8);\n atomic_int!(AtomicU8, u8, 1, cfg_has_atomic_cas_or_amo8, cfg_no_atomic_cas_or_amo8);\n}\ncfg_has_atomic_16! {\n atomic_int!(AtomicI16, i16, 2, cfg_has_atomic_cas_or_amo8, cfg_no_atomic_cas_or_amo8);\n atomic_int!(AtomicU16, u16, 2, cfg_has_atomic_cas_or_amo8, cfg_no_atomic_cas_or_amo8,\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f16))] AtomicF16, f16);\n}\ncfg_has_atomic_32! {\n atomic_int!(AtomicI32, i32, 4, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32);\n atomic_int!(AtomicU32, u32, 4, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32,\n #[cfg(feature = \"float\")] AtomicF32, f32);\n}\ncfg_has_atomic_64! {\n atomic_int!(AtomicI64, i64, 8, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32);\n atomic_int!(AtomicU64, u64, 8, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32,\n #[cfg(feature = \"float\")] AtomicF64, f64);\n}\ncfg_has_atomic_128! {\n atomic_int!(AtomicI128, i128, 16, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32);\n atomic_int!(AtomicU128, u128, 16, cfg_has_atomic_cas_or_amo32, cfg_no_atomic_cas_or_amo32,\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f128))] AtomicF128, f128);\n}\n\n// See https://github.com/taiki-e/portable-atomic/issues/180\n#[cfg(not(feature = \"require-cas\"))]\ncfg_no_atomic_cas! {\ncfg_no_atomic_cas_or_amo32! {\n#[cfg(feature = \"float\")]\nuse self::diagnostic_helper::HasFetchAbs;\nuse self::diagnostic_helper::{\n HasAnd, HasBitClear, HasBitSet, HasBitToggle, HasFetchAnd, HasFetchByteAdd, HasFetchByteSub,\n HasFetchNot, HasFetchOr, HasFetchPtrAdd, HasFetchPtrSub, HasFetchXor, HasNot, HasOr, HasXor,\n};\n} // cfg_no_atomic_cas_or_amo32!\ncfg_no_atomic_cas_or_amo8! {\nuse self::diagnostic_helper::{HasAdd, HasSub, HasSwap};\n} // cfg_no_atomic_cas_or_amo8!\n#[cfg_attr(not(feature = \"float\"), allow(unused_imports))]\nuse self::diagnostic_helper::{\n HasCompareExchange, HasCompareExchangeWeak, HasFetchAdd, HasFetchMax, HasFetchMin,\n HasFetchNand, HasFetchNeg, HasFetchSub, HasFetchUpdate, HasNeg,\n};\n#[cfg_attr(\n any(\n all(\n portable_atomic_no_atomic_load_store,\n not(any(\n target_arch = \"avr\",\n target_arch = \"bpf\",\n target_arch = \"msp430\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n feature = \"critical-section\",\n portable_atomic_unsafe_assume_single_core,\n )),\n ),\n not(feature = \"float\"),\n ),\n allow(dead_code, unreachable_pub)\n)]\n#[allow(unknown_lints, unnameable_types)] // Not public API. unnameable_types is available on Rust 1.79+\nmod diagnostic_helper {\n cfg_no_atomic_cas_or_amo8! {\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`swap` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasSwap {}\n } // cfg_no_atomic_cas_or_amo8!\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`compare_exchange` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasCompareExchange {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`compare_exchange_weak` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasCompareExchangeWeak {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_add` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchAdd {}\n cfg_no_atomic_cas_or_amo8! {\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`add` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasAdd {}\n } // cfg_no_atomic_cas_or_amo8!\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_sub` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchSub {}\n cfg_no_atomic_cas_or_amo8! {\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`sub` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasSub {}\n } // cfg_no_atomic_cas_or_amo8!\n cfg_no_atomic_cas_or_amo32! {\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_ptr_add` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchPtrAdd {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_ptr_sub` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchPtrSub {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_byte_add` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchByteAdd {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_byte_sub` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchByteSub {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_and` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchAnd {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`and` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasAnd {}\n } // cfg_no_atomic_cas_or_amo32!\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_nand` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchNand {}\n cfg_no_atomic_cas_or_amo32! {\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_or` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchOr {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`or` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasOr {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_xor` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchXor {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`xor` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasXor {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_not` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchNot {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`not` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasNot {}\n } // cfg_no_atomic_cas_or_amo32!\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_neg` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchNeg {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`neg` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasNeg {}\n cfg_no_atomic_cas_or_amo32! {\n #[cfg(feature = \"float\")]\n #[cfg_attr(target_pointer_width = \"16\", allow(dead_code, unreachable_pub))]\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_abs` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchAbs {}\n } // cfg_no_atomic_cas_or_amo32!\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_min` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchMin {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_max` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchMax {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`fetch_update` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasFetchUpdate {}\n cfg_no_atomic_cas_or_amo32! {\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`bit_set` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasBitSet {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`bit_clear` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasBitClear {}\n #[doc(hidden)]\n #[cfg_attr(\n not(portable_atomic_no_diagnostic_namespace),\n diagnostic::on_unimplemented(\n message = \"`bit_toggle` requires atomic CAS but not available on this target by default\",\n label = \"this associated function is not available on this target by default\",\n note = \"consider enabling one of the `critical-section` feature or `unsafe-assume-single-core` feature (or `portable_atomic_unsafe_assume_single_core` cfg)\",\n note = \"see for more.\"\n )\n )]\n pub trait HasBitToggle {}\n } // cfg_no_atomic_cas_or_amo32!\n}\n} // cfg_no_atomic_cas!\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/tests/helper.rs", "sha256": "dc3d529032bd0a0ec4c9f25246504758042531109b3fd1a65a6be5bc1c4b7265", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![allow(unused_macros, clippy::undocumented_unsafe_blocks)]\n\nuse std::{\n eprintln,\n sync::atomic::{AtomicIsize, AtomicUsize, Ordering},\n};\n\nuse crossbeam_utils::thread;\n\nmacro_rules! __test_atomic_common {\n ($atomic_type:ty, $value_type:ty) => {\n use std::mem;\n #[test]\n fn assert_auto_traits() {\n fn _assert() {}\n _assert::<$atomic_type>();\n }\n #[test]\n fn alignment() {\n // https://github.com/rust-lang/rust/blob/1.84.0/library/core/tests/atomic.rs#L252\n assert_eq!(mem::align_of::<$atomic_type>(), mem::size_of::<$atomic_type>());\n assert_eq!(mem::size_of::<$atomic_type>(), mem::size_of::<$value_type>());\n }\n #[test]\n fn is_lock_free() {\n const IS_ALWAYS_LOCK_FREE: bool = <$atomic_type>::IS_ALWAYS_LOCK_FREE;\n assert_eq!(IS_ALWAYS_LOCK_FREE, <$atomic_type>::IS_ALWAYS_LOCK_FREE);\n let is_lock_free = <$atomic_type>::is_lock_free();\n if IS_ALWAYS_LOCK_FREE {\n // If is_always_lock_free is true, then is_lock_free must always be true.\n assert!(is_lock_free);\n }\n }\n };\n}\nmacro_rules! __test_atomic_pub_common {\n ($atomic_type:ty, $value_type:ty) => {\n #[test]\n fn is_always_lock_free() {\n assert_eq!(<$atomic_type>::IS_ALWAYS_LOCK_FREE, <$atomic_type>::is_always_lock_free());\n }\n #[test]\n fn assert_ref_unwind_safe() {\n #[cfg(not(all(portable_atomic_no_core_unwind_safe, not(feature = \"std\"))))]\n static_assertions::assert_impl_all!($atomic_type: std::panic::RefUnwindSafe);\n #[cfg(all(portable_atomic_no_core_unwind_safe, not(feature = \"std\")))]\n static_assertions::assert_not_impl_all!($atomic_type: std::panic::RefUnwindSafe);\n }\n };\n}\n\nmacro_rules! __test_atomic_int_load_store {\n ($atomic_type:ty, $int_type:ident, single_thread) => {\n __test_atomic_common!($atomic_type, $int_type);\n use crate::tests::helper::{self, *};\n #[test]\n fn accessor() {\n let a = <$atomic_type>::new(10);\n unsafe {\n assert_eq!(*a.as_ptr(), 10);\n *a.as_ptr() = 5;\n assert_eq!(a.as_ptr() as *const (), &a as *const _ as *const ());\n assert_eq!(*a.as_ptr(), 5);\n }\n }\n // https://bugs.llvm.org/show_bug.cgi?id=37061\n #[test]\n fn static_load_only() {\n static VAR: $atomic_type = <$atomic_type>::new(10);\n for &order in &helper::LOAD_ORDERINGS {\n assert_eq!(VAR.load(order), 10);\n }\n }\n #[test]\n fn load_store() {\n static VAR: $atomic_type = <$atomic_type>::new(10);\n test_load_ordering(&|order| {\n VAR.load(order);\n });\n test_store_ordering(&|order| VAR.store(10, order));\n for (&load_order, &store_order) in\n helper::LOAD_ORDERINGS.iter().zip(&helper::STORE_ORDERINGS)\n {\n assert_eq!(VAR.load(load_order), 10);\n VAR.store(5, store_order);\n assert_eq!(VAR.load(load_order), 5);\n VAR.store(10, store_order);\n let a = <$atomic_type>::new(1);\n assert_eq!(a.load(load_order), 1);\n a.store($int_type::MIN, store_order);\n assert_eq!(a.load(load_order), $int_type::MIN);\n a.store($int_type::MAX, store_order);\n assert_eq!(a.load(load_order), $int_type::MAX);\n }\n }\n };\n ($atomic_type:ty, $int_type:ident) => {\n __test_atomic_int_load_store!($atomic_type, $int_type, single_thread);\n\n use std::{collections::BTreeSet, eprintln, time::Instant, vec, vec::Vec};\n\n use crossbeam_utils::thread;\n\n #[test]\n fn stress_load_store() {\n let mut rng = fastrand::Rng::new();\n let (iterations, threads) = stress_test_config(&mut rng);\n let data1 = (0..iterations).map(|_| rng.$int_type(..)).collect::>();\n let set = data1.iter().copied().collect::>();\n let a = <$atomic_type>::new(data1[rng.usize(0..iterations)]);\n let now = &Instant::now();\n thread::scope(|s| {\n for _ in 0..threads {\n s.spawn(|_| {\n let mut rng = fastrand::Rng::new();\n let now = *now;\n for i in 0..iterations {\n a.store(data1[i], rand_store_ordering(&mut rng));\n }\n eprintln!(\"store end={:?}\", now.elapsed());\n });\n s.spawn(|_| {\n let mut rng = fastrand::Rng::new();\n let now = *now;\n let mut v = vec![0; iterations];\n for i in 0..iterations {\n v[i] = a.load(rand_load_ordering(&mut rng));\n }\n eprintln!(\"load end={:?}\", now.elapsed());\n for v in v {\n assert!(set.contains(&v), \"load unexpected={}\", v);\n }\n });\n }\n })\n .unwrap();\n }\n };\n}\nmacro_rules! __test_atomic_float_load_store {\n ($atomic_type:ty, $float_type:ident, single_thread) => {\n __test_atomic_common!($atomic_type, $float_type);\n use crate::tests::helper::{self, *};\n #[test]\n fn accessor() {\n let a = <$atomic_type>::new(10.);\n unsafe {\n assert_eq!(*a.as_ptr(), 10.);\n *a.as_ptr() = 5.;\n assert_eq!(a.as_ptr() as *const (), &a as *const _ as *const ());\n assert_eq!(*a.as_ptr(), 5.);\n }\n }\n // https://bugs.llvm.org/show_bug.cgi?id=37061\n #[test]\n fn static_load_only() {\n static VAR: $atomic_type = <$atomic_type>::new(10.);\n for &order in &helper::LOAD_ORDERINGS {\n assert_eq!(VAR.load(order), 10.);\n }\n }\n #[test]\n fn load_store() {\n static VAR: $atomic_type = <$atomic_type>::new(10.);\n test_load_ordering(&|order| {\n VAR.load(order);\n });\n test_store_ordering(&|order| VAR.store(10., order));\n for (&load_order, &store_order) in\n helper::LOAD_ORDERINGS.iter().zip(&helper::STORE_ORDERINGS)\n {\n assert_eq!(VAR.load(load_order), 10.);\n VAR.store(5., store_order);\n assert_eq!(VAR.load(load_order), 5.);\n VAR.store(10., store_order);\n let a = <$atomic_type>::new(1.);\n assert_eq!(a.load(load_order), 1.);\n a.store(2., store_order);\n assert_eq!(a.load(load_order), 2.);\n }\n }\n };\n ($atomic_type:ty, $float_type:ident) => {\n __test_atomic_float_load_store!($atomic_type, $float_type, single_thread);\n // TODO: multi thread\n };\n}\nmacro_rules! __test_atomic_bool_load_store {\n ($atomic_type:ty, single_thread) => {\n __test_atomic_common!($atomic_type, bool);\n use crate::tests::helper::{self, *};\n #[test]\n fn accessor() {\n let a = <$atomic_type>::new(false);\n unsafe {\n assert_eq!(*a.as_ptr(), false);\n *a.as_ptr() = true;\n assert_eq!(a.as_ptr() as *const (), &a as *const _ as *const ());\n assert_eq!(*a.as_ptr(), true);\n }\n }\n // https://bugs.llvm.org/show_bug.cgi?id=37061\n #[test]\n fn static_load_only() {\n static VAR: $atomic_type = <$atomic_type>::new(false);\n for &order in &helper::LOAD_ORDERINGS {\n assert_eq!(VAR.load(order), false);\n }\n }\n #[test]\n fn load_store() {\n static VAR: $atomic_type = <$atomic_type>::new(false);\n test_load_ordering(&|order| {\n VAR.load(order);\n });\n test_store_ordering(&|order| VAR.store(false, order));\n for (&load_order, &store_order) in\n helper::LOAD_ORDERINGS.iter().zip(&helper::STORE_ORDERINGS)\n {\n assert_eq!(VAR.load(load_order), false);\n VAR.store(true, store_order);\n assert_eq!(VAR.load(load_order), true);\n VAR.store(false, store_order);\n let a = <$atomic_type>::new(true);\n assert_eq!(a.load(load_order), true);\n a.store(false, store_order);\n assert_eq!(a.load(load_order), false);\n }\n }\n };\n ($atomic_type:ty) => {\n __test_atomic_bool_load_store!($atomic_type, single_thread);\n // TODO: multi thread\n };\n}\nmacro_rules! __test_atomic_ptr_load_store {\n ($atomic_type:ty, single_thread) => {\n __test_atomic_common!($atomic_type, *mut u8);\n\n use std::ptr;\n\n use crate::tests::helper::{self, *};\n\n #[test]\n fn accessor() {\n let mut v = 1;\n let a = <$atomic_type>::new(ptr::null_mut());\n unsafe {\n assert!((*a.as_ptr()).is_null());\n *a.as_ptr() = &mut v;\n assert_eq!(a.as_ptr() as *const (), &a as *const _ as *const ());\n assert!(!(*a.as_ptr()).is_null());\n }\n }\n // https://bugs.llvm.org/show_bug.cgi?id=37061\n #[test]\n fn static_load_only() {\n static VAR: $atomic_type = <$atomic_type>::new(ptr::null_mut());\n for &order in &helper::LOAD_ORDERINGS {\n assert_eq!(VAR.load(order), ptr::null_mut());\n }\n }\n #[test]\n fn load_store() {\n static VAR: $atomic_type = <$atomic_type>::new(ptr::null_mut());\n test_load_ordering(&|order| {\n VAR.load(order);\n });\n test_store_ordering(&|order| VAR.store(ptr::null_mut(), order));\n let mut v = 1_u8;\n let p = &mut v as *mut u8;\n for (&load_order, &store_order) in\n helper::LOAD_ORDERINGS.iter().zip(&helper::STORE_ORDERINGS)\n {\n assert_eq!(VAR.load(load_order), ptr::null_mut());\n VAR.store(p, store_order);\n assert_eq!(VAR.load(load_order), p);\n VAR.store(ptr::null_mut(), store_order);\n let a = <$atomic_type>::new(p);\n assert_eq!(a.load(load_order), p);\n a.store(ptr::null_mut(), store_order);\n assert_eq!(a.load(load_order), ptr::null_mut());\n }\n }\n };\n ($atomic_type:ty) => {\n __test_atomic_ptr_load_store!($atomic_type, single_thread);\n // TODO: multi thread\n };\n}\n\nmacro_rules! __test_atomic_int {\n ($atomic_type:ty, $int_type:ident, single_thread) => {\n #[test]\n fn swap() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(5);\n test_swap_ordering(&|order| {\n a.swap(5, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n assert_eq!(a.swap(10, order), 5);\n assert_eq!(a.swap($int_type::MIN, order), 10);\n assert_eq!(a.swap($int_type::MAX, order), $int_type::MIN);\n assert_eq!(a.swap(5, order), $int_type::MAX);\n }\n }\n #[test]\n fn compare_exchange() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(5);\n test_compare_exchange_ordering(&|success, failure| {\n let _ = a.compare_exchange(5, 5, success, failure);\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(5);\n assert_eq!(a.compare_exchange(5, 10, success, failure), Ok(5));\n assert_eq!(a.load(Ordering::Relaxed), 10);\n assert_eq!(a.compare_exchange(6, 12, success, failure), Err(10));\n assert_eq!(a.load(Ordering::Relaxed), 10);\n }\n }\n #[test]\n fn compare_exchange_weak() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(4);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n test_compare_exchange_ordering(&|success, failure| {\n let _ = a.compare_exchange_weak(4, 4, success, failure);\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(4);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.compare_exchange_weak(6, 8, success, failure), Err(4));\n let mut old = a.load(Ordering::Relaxed);\n loop {\n let new = old * 2;\n match a.compare_exchange_weak(old, new, success, failure) {\n Ok(_) => break,\n Err(x) => old = x,\n }\n }\n assert_eq!(a.load(Ordering::Relaxed), 8);\n }\n }\n #[test]\n fn fetch_add() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(0);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n test_swap_ordering(&|order| {\n a.fetch_add(0, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_add(10, order), 0);\n assert_eq!(a.load(Ordering::Relaxed), 10);\n let a = <$atomic_type>::new($int_type::MAX);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_add(1, order), $int_type::MAX);\n assert_eq!(a.load(Ordering::Relaxed), $int_type::MAX.wrapping_add(1));\n }\n }\n #[test]\n fn add() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(0);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n test_swap_ordering(&|order| a.add(0, order));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n a.add(10, order);\n assert_eq!(a.load(Ordering::Relaxed), 10);\n let a = <$atomic_type>::new($int_type::MAX);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n a.add(1, order);\n assert_eq!(a.load(Ordering::Relaxed), $int_type::MAX.wrapping_add(1));\n }\n }\n #[test]\n fn fetch_sub() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(20);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n test_swap_ordering(&|order| {\n a.fetch_sub(0, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(20);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_sub(10, order), 20);\n assert_eq!(a.load(Ordering::Relaxed), 10);\n let a = <$atomic_type>::new($int_type::MIN);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_sub(1, order), $int_type::MIN);\n assert_eq!(a.load(Ordering::Relaxed), $int_type::MIN.wrapping_sub(1));\n }\n }\n #[test]\n fn sub() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(20);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n test_swap_ordering(&|order| a.sub(0, order));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(20);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n a.sub(10, order);\n assert_eq!(a.load(Ordering::Relaxed), 10);\n let a = <$atomic_type>::new($int_type::MIN);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n a.sub(1, order);\n assert_eq!(a.load(Ordering::Relaxed), $int_type::MIN.wrapping_sub(1));\n }\n }\n #[test]\n fn fetch_and() {\n let a = <$atomic_type>::new(0b101101);\n test_swap_ordering(&|order| {\n a.fetch_and(0b101101, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0b101101);\n assert_eq!(a.fetch_and(0b110011, order), 0b101101);\n assert_eq!(a.load(Ordering::Relaxed), 0b100001);\n }\n }\n #[test]\n fn and() {\n let a = <$atomic_type>::new(0b101101);\n test_swap_ordering(&|order| a.and(0b101101, order));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0b101101);\n a.and(0b110011, order);\n assert_eq!(a.load(Ordering::Relaxed), 0b100001);\n }\n }\n #[test]\n fn fetch_nand() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") {\n return;\n }\n let a = <$atomic_type>::new(0x13);\n test_swap_ordering(&|order| {\n a.fetch_nand(0x31, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0x13);\n assert_eq!(a.fetch_nand(0x31, order), 0x13);\n assert_eq!(a.load(Ordering::Relaxed), !(0x13 & 0x31));\n }\n }\n #[test]\n fn fetch_or() {\n let a = <$atomic_type>::new(0b101101);\n test_swap_ordering(&|order| {\n a.fetch_or(0, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0b101101);\n assert_eq!(a.fetch_or(0b110011, order), 0b101101);\n assert_eq!(a.load(Ordering::Relaxed), 0b111111);\n }\n }\n #[test]\n fn or() {\n let a = <$atomic_type>::new(0b101101);\n test_swap_ordering(&|order| a.or(0, order));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0b101101);\n a.or(0b110011, order);\n assert_eq!(a.load(Ordering::Relaxed), 0b111111);\n }\n }\n #[test]\n fn fetch_xor() {\n let a = <$atomic_type>::new(0b101101);\n test_swap_ordering(&|order| {\n a.fetch_xor(0, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0b101101);\n assert_eq!(a.fetch_xor(0b110011, order), 0b101101);\n assert_eq!(a.load(Ordering::Relaxed), 0b011110);\n }\n }\n #[test]\n fn xor() {\n let a = <$atomic_type>::new(0b101101);\n test_swap_ordering(&|order| a.xor(0, order));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0b101101);\n a.xor(0b110011, order);\n assert_eq!(a.load(Ordering::Relaxed), 0b011110);\n }\n }\n #[test]\n fn fetch_max() {\n // TODO(gcc): failure: https://github.com/rust-lang/rustc_codegen_gcc/issues/821#issuecomment-3793567607\n if option_env!(\"RUSTC_CODEGEN_GCC\").unwrap_or_default() == \"1\"\n && $int_type::MIN == 0\n && mem::size_of::<$int_type>() <= 8\n {\n return;\n }\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(23);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n test_swap_ordering(&|order| {\n a.fetch_max(23, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(23);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_max(22, order), 23);\n assert_eq!(a.load(Ordering::Relaxed), 23);\n assert_eq!(a.fetch_max(24, order), 23);\n assert_eq!(a.load(Ordering::Relaxed), 24);\n let a = <$atomic_type>::new(0);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_max(1, order), 0);\n assert_eq!(a.load(Ordering::Relaxed), 1);\n assert_eq!(a.fetch_max(0, order), 1);\n assert_eq!(a.load(Ordering::Relaxed), 1);\n let a = <$atomic_type>::new(!0);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_max(0, order), !0);\n assert_eq!(a.load(Ordering::Relaxed), core::cmp::max(!0, 0));\n }\n }\n #[test]\n fn fetch_min() {\n // TODO(gcc): failure: https://github.com/rust-lang/rustc_codegen_gcc/issues/821#issuecomment-3793567607\n if option_env!(\"RUSTC_CODEGEN_GCC\").unwrap_or_default() == \"1\"\n && $int_type::MIN == 0\n && mem::size_of::<$int_type>() <= 8\n {\n return;\n }\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(23);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n test_swap_ordering(&|order| {\n a.fetch_min(23, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(23);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_min(24, order), 23);\n assert_eq!(a.load(Ordering::Relaxed), 23);\n assert_eq!(a.fetch_min(22, order), 23);\n assert_eq!(a.load(Ordering::Relaxed), 22);\n let a = <$atomic_type>::new(1);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_min(0, order), 1);\n assert_eq!(a.load(Ordering::Relaxed), 0);\n assert_eq!(a.fetch_min(1, order), 0);\n assert_eq!(a.load(Ordering::Relaxed), 0);\n let a = <$atomic_type>::new(!0);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_min(0, order), !0);\n assert_eq!(a.load(Ordering::Relaxed), core::cmp::min(!0, 0));\n }\n }\n #[test]\n fn fetch_not() {\n let a = <$atomic_type>::new(1);\n test_swap_ordering(&|order| {\n a.fetch_not(order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(1);\n assert_eq!(a.fetch_not(order), 1);\n assert_eq!(a.load(Ordering::Relaxed), !1);\n }\n }\n #[test]\n fn not() {\n let a = <$atomic_type>::new(1);\n test_swap_ordering(&|order| a.not(order));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(1);\n a.not(order);\n assert_eq!(a.load(Ordering::Relaxed), !1);\n }\n }\n #[test]\n fn fetch_neg() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(5);\n test_swap_ordering(&|order| {\n a.fetch_neg(order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(5);\n assert_eq!(a.fetch_neg(order), 5);\n assert_eq!(a.load(Ordering::Relaxed), <$int_type>::wrapping_neg(5));\n assert_eq!(a.fetch_neg(order), <$int_type>::wrapping_neg(5));\n assert_eq!(a.load(Ordering::Relaxed), 5);\n let a = <$atomic_type>::new(<$int_type>::MIN);\n assert_eq!(a.fetch_neg(order), <$int_type>::MIN);\n assert_eq!(a.load(Ordering::Relaxed), <$int_type>::MIN.wrapping_neg());\n assert_eq!(a.fetch_neg(order), <$int_type>::MIN.wrapping_neg());\n assert_eq!(a.load(Ordering::Relaxed), <$int_type>::MIN);\n }\n }\n #[test]\n fn neg() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(5);\n test_swap_ordering(&|order| a.neg(order));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(5);\n a.neg(order);\n assert_eq!(a.load(Ordering::Relaxed), <$int_type>::wrapping_neg(5));\n a.neg(order);\n assert_eq!(a.load(Ordering::Relaxed), 5);\n let a = <$atomic_type>::new(<$int_type>::MIN);\n a.neg(order);\n assert_eq!(a.load(Ordering::Relaxed), <$int_type>::MIN.wrapping_neg());\n a.neg(order);\n assert_eq!(a.load(Ordering::Relaxed), <$int_type>::MIN);\n }\n }\n #[test]\n fn bit_set() {\n let a = <$atomic_type>::new(0b0001);\n test_swap_ordering(&|order| assert!(a.bit_set(0, order)));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0b0000);\n assert!(!a.bit_set(0, order));\n assert_eq!(a.load(Ordering::Relaxed), 0b0001);\n assert!(a.bit_set(0, order));\n assert_eq!(a.load(Ordering::Relaxed), 0b0001);\n }\n }\n #[test]\n fn bit_clear() {\n let a = <$atomic_type>::new(0b0000);\n test_swap_ordering(&|order| assert!(!a.bit_clear(0, order)));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0b0001);\n assert!(a.bit_clear(0, order));\n assert_eq!(a.load(Ordering::Relaxed), 0b0000);\n assert!(!a.bit_clear(0, order));\n assert_eq!(a.load(Ordering::Relaxed), 0b0000);\n }\n }\n #[test]\n fn bit_toggle() {\n let a = <$atomic_type>::new(0b0000);\n test_swap_ordering(&|order| {\n a.bit_toggle(0, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0b0000);\n assert!(!a.bit_toggle(0, order));\n assert_eq!(a.load(Ordering::Relaxed), 0b0001);\n assert!(a.bit_toggle(0, order));\n assert_eq!(a.load(Ordering::Relaxed), 0b0000);\n }\n }\n ::quickcheck::quickcheck! {\n fn quickcheck_swap(x: $int_type, y: $int_type) -> bool {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.swap(y, order), x);\n assert_eq!(a.swap(x, order), y);\n }\n true\n }\n fn quickcheck_compare_exchange(x: $int_type, y: $int_type) -> bool {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n #[cfg(all(\n target_arch = \"arm\",\n not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\")),\n ))]\n {\n // TODO: LLVM bug:\n // https://github.com/llvm/llvm-project/issues/61880\n // https://github.com/taiki-e/portable-atomic/issues/2\n if mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n }\n let mut rng = fastrand::Rng::new();\n let z = loop {\n let z = rng.$int_type(..);\n if z != y {\n break z;\n }\n };\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.compare_exchange(x, y, success, failure).unwrap(), x);\n assert_eq!(a.load(Ordering::Relaxed), y);\n assert_eq!(a.compare_exchange(z, x, success, failure).unwrap_err(), y);\n assert_eq!(a.load(Ordering::Relaxed), y);\n }\n true\n }\n fn quickcheck_fetch_add(x: $int_type, y: $int_type) -> bool {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_add(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x.wrapping_add(y));\n let a = <$atomic_type>::new(y);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_add(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y.wrapping_add(x));\n }\n true\n }\n fn quickcheck_add(x: $int_type, y: $int_type) -> bool {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n a.add(y, order);\n assert_eq!(a.load(Ordering::Relaxed), x.wrapping_add(y));\n let a = <$atomic_type>::new(y);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n a.add(x, order);\n assert_eq!(a.load(Ordering::Relaxed), y.wrapping_add(x));\n }\n true\n }\n fn quickcheck_fetch_sub(x: $int_type, y: $int_type) -> bool {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_sub(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x.wrapping_sub(y));\n let a = <$atomic_type>::new(y);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_sub(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y.wrapping_sub(x));\n }\n true\n }\n fn quickcheck_sub(x: $int_type, y: $int_type) -> bool {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n a.sub(y, order);\n assert_eq!(a.load(Ordering::Relaxed), x.wrapping_sub(y));\n let a = <$atomic_type>::new(y);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n a.sub(x, order);\n assert_eq!(a.load(Ordering::Relaxed), y.wrapping_sub(x));\n }\n true\n }\n fn quickcheck_fetch_and(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_and(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x & y);\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_and(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y & x);\n }\n true\n }\n fn quickcheck_and(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n a.and(y, order);\n assert_eq!(a.load(Ordering::Relaxed), x & y);\n let a = <$atomic_type>::new(y);\n a.and(x, order);\n assert_eq!(a.load(Ordering::Relaxed), y & x);\n }\n true\n }\n fn quickcheck_fetch_nand(x: $int_type, y: $int_type) -> bool {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") {\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_nand(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), !(x & y));\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_nand(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), !(y & x));\n }\n true\n }\n fn quickcheck_fetch_or(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_or(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x | y);\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_or(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y | x);\n }\n true\n }\n fn quickcheck_or(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n a.or(y, order);\n assert_eq!(a.load(Ordering::Relaxed), x | y);\n let a = <$atomic_type>::new(y);\n a.or(x, order);\n assert_eq!(a.load(Ordering::Relaxed), y | x);\n }\n true\n }\n fn quickcheck_fetch_xor(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_xor(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), x ^ y);\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_xor(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), y ^ x);\n }\n true\n }\n fn quickcheck_xor(x: $int_type, y: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n a.xor(y, order);\n assert_eq!(a.load(Ordering::Relaxed), x ^ y);\n let a = <$atomic_type>::new(y);\n a.xor(x, order);\n assert_eq!(a.load(Ordering::Relaxed), y ^ x);\n }\n true\n }\n fn quickcheck_fetch_max(x: $int_type, y: $int_type) -> bool {\n // TODO(gcc): failure: https://github.com/rust-lang/rustc_codegen_gcc/issues/821#issuecomment-3793567607\n if option_env!(\"RUSTC_CODEGEN_GCC\").unwrap_or_default() == \"1\"\n && $int_type::MIN == 0\n && mem::size_of::<$int_type>() <= 8\n {\n return true;\n }\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_max(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), core::cmp::max(x, y));\n let a = <$atomic_type>::new(y);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_max(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), core::cmp::max(y, x));\n }\n true\n }\n fn quickcheck_fetch_min(x: $int_type, y: $int_type) -> bool {\n // TODO(gcc): failure: https://github.com/rust-lang/rustc_codegen_gcc/issues/821#issuecomment-3793567607\n if option_env!(\"RUSTC_CODEGEN_GCC\").unwrap_or_default() == \"1\"\n && $int_type::MIN == 0\n && mem::size_of::<$int_type>() <= 8\n {\n return true;\n }\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_min(y, order), x);\n assert_eq!(a.load(Ordering::Relaxed), core::cmp::min(x, y));\n let a = <$atomic_type>::new(y);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_min(x, order), y);\n assert_eq!(a.load(Ordering::Relaxed), core::cmp::min(y, x));\n }\n true\n }\n fn quickcheck_fetch_not(x: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_not(order), x);\n assert_eq!(a.load(Ordering::Relaxed), !x);\n assert_eq!(a.fetch_not(order), !x);\n assert_eq!(a.load(Ordering::Relaxed), x);\n }\n true\n }\n fn quickcheck_not(x: $int_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n a.not(order);\n assert_eq!(a.load(Ordering::Relaxed), !x);\n a.not(order);\n assert_eq!(a.load(Ordering::Relaxed), x);\n }\n true\n }\n fn quickcheck_fetch_neg(x: $int_type) -> bool {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n #[cfg(all(\n target_arch = \"arm\",\n not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\")),\n ))]\n {\n // TODO: LLVM bug:\n // https://github.com/llvm/llvm-project/issues/61880\n // https://github.com/taiki-e/portable-atomic/issues/2\n if mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_neg(order), x);\n assert_eq!(a.load(Ordering::Relaxed), x.wrapping_neg());\n assert_eq!(a.fetch_neg(order), x.wrapping_neg());\n assert_eq!(a.load(Ordering::Relaxed), x);\n }\n true\n }\n fn quickcheck_neg(x: $int_type) -> bool {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n a.neg(order);\n assert_eq!(a.load(Ordering::Relaxed), x.wrapping_neg());\n a.neg(order);\n assert_eq!(a.load(Ordering::Relaxed), x);\n }\n true\n }\n fn quickcheck_bit_set(x: $int_type, bit: u32) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n let b = a.bit_set(bit, order);\n let mask = <$int_type>::wrapping_shl(1, bit);\n assert_eq!(a.load(Ordering::Relaxed), x | mask);\n assert_eq!(b, x & mask != 0);\n }\n true\n }\n fn quickcheck_bit_clear(x: $int_type, bit: u32) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n let b = a.bit_clear(bit, order);\n let mask = <$int_type>::wrapping_shl(1, bit);\n assert_eq!(a.load(Ordering::Relaxed), x & !mask);\n assert_eq!(b, x & mask != 0);\n }\n true\n }\n fn quickcheck_bit_toggle(x: $int_type, bit: u32) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n let b = a.bit_toggle(bit, order);\n let mask = <$int_type>::wrapping_shl(1, bit);\n assert_eq!(a.load(Ordering::Relaxed), x ^ mask);\n assert_eq!(b, x & mask != 0);\n }\n true\n }\n }\n };\n ($atomic_type:ty, $int_type:ident) => {\n __test_atomic_int!($atomic_type, $int_type, single_thread);\n\n #[test]\n fn stress_swap() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let mut rng = fastrand::Rng::new();\n let (iterations, threads) = stress_test_config(&mut rng);\n let data1 = &(0..threads)\n .map(|_| (0..iterations).map(|_| rng.$int_type(..)).collect::>())\n .collect::>();\n let data2 = &(0..threads)\n .map(|_| (0..iterations).map(|_| rng.$int_type(..)).collect::>())\n .collect::>();\n let set = &data1\n .iter()\n .flat_map(|v| v.iter().copied())\n .chain(data2.iter().flat_map(|v| v.iter().copied()))\n .collect::>();\n let a = &<$atomic_type>::new(data2[0][rng.usize(0..iterations)]);\n let now = &Instant::now();\n thread::scope(|s| {\n for thread in 0..threads {\n if thread % 2 == 0 {\n s.spawn(move |_| {\n let mut rng = fastrand::Rng::new();\n let now = *now;\n for i in 0..iterations {\n a.store(data1[thread][i], rand_store_ordering(&mut rng));\n }\n eprintln!(\"store end={:?}\", now.elapsed());\n });\n } else {\n s.spawn(|_| {\n let mut rng = fastrand::Rng::new();\n let now = *now;\n let mut v = vec![0; iterations];\n for i in 0..iterations {\n v[i] = a.load(rand_load_ordering(&mut rng));\n }\n eprintln!(\"load end={:?}\", now.elapsed());\n for v in v {\n assert!(set.contains(&v), \"load unexpected={}\", v);\n }\n });\n }\n s.spawn(move |_| {\n let mut rng = fastrand::Rng::new();\n let now = *now;\n let mut v = vec![0; iterations];\n for i in 0..iterations {\n v[i] = a.swap(data2[thread][i], rand_swap_ordering(&mut rng));\n }\n eprintln!(\"swap end={:?}\", now.elapsed());\n for v in v {\n assert!(set.contains(&v), \"swap unexpected={}\", v);\n }\n });\n }\n })\n .unwrap();\n }\n #[test]\n fn stress_compare_exchange() {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let mut rng = fastrand::Rng::new();\n let (iterations, threads) = stress_test_config(&mut rng);\n let data1 = &(0..threads)\n .map(|_| (0..iterations).map(|_| rng.$int_type(..)).collect::>())\n .collect::>();\n let data2 = &(0..threads)\n .map(|_| (0..iterations).map(|_| rng.$int_type(..)).collect::>())\n .collect::>();\n let set = &data1\n .iter()\n .flat_map(|v| v.iter().copied())\n .chain(data2.iter().flat_map(|v| v.iter().copied()))\n .collect::>();\n let a = &<$atomic_type>::new(data2[0][rng.usize(0..iterations)]);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(a);\n }\n let now = &Instant::now();\n thread::scope(|s| {\n for thread in 0..threads {\n s.spawn(move |_| {\n let mut rng = fastrand::Rng::new();\n let now = *now;\n for i in 0..iterations {\n a.store(data1[thread][i], rand_store_ordering(&mut rng));\n }\n eprintln!(\"store end={:?}\", now.elapsed());\n });\n s.spawn(|_| {\n let mut rng = fastrand::Rng::new();\n let now = *now;\n let mut v = vec![data2[0][0]; iterations];\n for i in 0..iterations {\n v[i] = a.load(rand_load_ordering(&mut rng));\n }\n eprintln!(\"load end={:?}\", now.elapsed());\n for v in v {\n assert!(set.contains(&v), \"load unexpected={}\", v);\n }\n });\n s.spawn(move |_| {\n let mut rng = fastrand::Rng::new();\n let now = *now;\n let mut v = vec![data2[0][0]; iterations];\n for i in 0..iterations {\n let old = if i % 2 == 0 {\n rng.$int_type(..)\n } else {\n a.load(Ordering::Relaxed)\n };\n let new = data2[thread][i];\n let o = rand_compare_exchange_ordering(&mut rng);\n match a.compare_exchange(old, new, o.0, o.1) {\n Ok(r) => assert_eq!(old, r),\n Err(r) => v[i] = r,\n }\n }\n eprintln!(\"compare_exchange end={:?}\", now.elapsed());\n for v in v {\n assert!(set.contains(&v), \"cas unexpected={}\", v);\n }\n });\n }\n })\n .unwrap();\n }\n };\n}\nmacro_rules! __test_atomic_float {\n ($atomic_type:ty, $float_type:ident, single_thread) => {\n #[test]\n fn swap() {\n let a = <$atomic_type>::new(5.);\n test_swap_ordering(&|order| { a.swap(5., order); });\n for &order in &helper::SWAP_ORDERINGS {\n assert_eq!(a.swap(10., order), 5.);\n assert_eq!(a.swap(5., order), 10.);\n }\n }\n #[test]\n fn compare_exchange() {\n let a = <$atomic_type>::new(5.);\n test_compare_exchange_ordering(&|success, failure| {\n let _ = a.compare_exchange(5., 5., success, failure);\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(5.);\n assert_eq!(a.compare_exchange(5., 10., success, failure), Ok(5.));\n assert_eq!(a.load(Ordering::Relaxed), 10.);\n assert_eq!(a.compare_exchange(6., 12., success, failure), Err(10.));\n assert_eq!(a.load(Ordering::Relaxed), 10.);\n }\n }\n #[test]\n fn compare_exchange_weak() {\n let a = <$atomic_type>::new(4.);\n test_compare_exchange_ordering(&|success, failure| {\n let _ = a.compare_exchange_weak(4., 4., success, failure);\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(4.);\n assert_eq!(a.compare_exchange_weak(6., 8., success, failure), Err(4.));\n let mut old = a.load(Ordering::Relaxed);\n loop {\n let new = old * 2.;\n match a.compare_exchange_weak(old, new, success, failure) {\n Ok(_) => break,\n Err(x) => old = x,\n }\n }\n assert_eq!(a.load(Ordering::Relaxed), 8.);\n }\n }\n #[test]\n fn fetch_add() {\n let a = <$atomic_type>::new(0.);\n test_swap_ordering(&|order| { a.fetch_add(0., order); });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(0.);\n assert_eq!(a.fetch_add(10., order), 0.);\n assert_eq!(a.load(Ordering::Relaxed), 10.);\n let a = <$atomic_type>::new($float_type::MAX);\n assert_eq!(a.fetch_add(1., order), $float_type::MAX);\n assert_eq!(a.load(Ordering::Relaxed), $float_type::MAX + 1.);\n }\n }\n #[test]\n fn fetch_sub() {\n let a = <$atomic_type>::new(20.);\n test_swap_ordering(&|order| { a.fetch_sub(0., order); });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(20.);\n assert_eq!(a.fetch_sub(10., order), 20.);\n assert_eq!(a.load(Ordering::Relaxed), 10.);\n let a = <$atomic_type>::new($float_type::MIN);\n assert_eq!(a.fetch_sub(1., order), $float_type::MIN);\n assert_eq!(a.load(Ordering::Relaxed), $float_type::MIN - 1.);\n }\n }\n #[test]\n fn fetch_max() {\n if mem::size_of::<$float_type>() == 16\n && cfg!(any(\n target_arch = \"arm\",\n target_arch = \"mips\",\n target_arch = \"mips32r6\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_vendor = \"apple\",\n windows,\n ))\n {\n // TODO(f128):\n return;\n }\n let a = <$atomic_type>::new(23.);\n test_swap_ordering(&|order| { a.fetch_max(23., order); });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(23.);\n assert_eq!(a.fetch_max(22., order), 23.);\n assert_eq!(a.load(Ordering::Relaxed), 23.);\n assert_eq!(a.fetch_max(24., order), 23.);\n assert_eq!(a.load(Ordering::Relaxed), 24.);\n }\n }\n #[test]\n fn fetch_min() {\n if mem::size_of::<$float_type>() == 16\n && cfg!(any(\n target_arch = \"arm\",\n target_arch = \"mips\",\n target_arch = \"mips32r6\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_vendor = \"apple\",\n windows,\n ))\n {\n // TODO(f128):\n return;\n }\n let a = <$atomic_type>::new(23.);\n test_swap_ordering(&|order| { a.fetch_min(23., order); });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(23.);\n assert_eq!(a.fetch_min(24., order), 23.);\n assert_eq!(a.load(Ordering::Relaxed), 23.);\n assert_eq!(a.fetch_min(22., order), 23.);\n assert_eq!(a.load(Ordering::Relaxed), 22.);\n }\n }\n #[test]\n fn fetch_neg() {\n let a = <$atomic_type>::new(5.);\n test_swap_ordering(&|order| { a.fetch_neg(order); });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(5.);\n assert_eq!(a.fetch_neg(order), 5.);\n assert_eq!(a.load(Ordering::Relaxed), -5.);\n assert_eq!(a.fetch_neg(order), -5.);\n assert_eq!(a.load(Ordering::Relaxed), 5.);\n }\n }\n #[test]\n fn fetch_abs() {\n let a = <$atomic_type>::new(23.);\n test_swap_ordering(&|order| { a.fetch_abs(order); });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(-23.);\n assert_eq!(a.fetch_abs(order), -23.);\n assert_eq!(a.load(Ordering::Relaxed), 23.);\n assert_eq!(a.fetch_abs(order), 23.);\n assert_eq!(a.load(Ordering::Relaxed), 23.);\n }\n }\n ::quickcheck::quickcheck! {\n fn quickcheck_swap(x: $float_type, y: $float_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_float_op_eq!(a.swap(y, order), x);\n assert_float_op_eq!(a.swap(x, order), y);\n }\n true\n }\n fn quickcheck_compare_exchange(x: $float_type, y: $float_type) -> bool {\n #[cfg(all(\n target_arch = \"arm\",\n not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\")),\n ))]\n {\n // TODO: LLVM bug:\n // https://github.com/llvm/llvm-project/issues/61880\n // https://github.com/taiki-e/portable-atomic/issues/2\n if mem::size_of::<$float_type>() <= 2 {\n return true;\n }\n }\n let mut rng = fastrand::Rng::new();\n let z = loop {\n let z = float_rand::$float_type(&mut rng);\n if z != y {\n break z;\n }\n };\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_float_op_eq!(a.compare_exchange(x, y, success, failure).unwrap(), x);\n assert_float_op_eq!(a.load(Ordering::Relaxed), y);\n assert_float_op_eq!(\n a.compare_exchange(z, x, success, failure).unwrap_err(),\n y,\n );\n assert_float_op_eq!(a.load(Ordering::Relaxed), y);\n }\n true\n }\n fn quickcheck_fetch_add(x: $float_type, y: $float_type) -> bool {\n if cfg!(all(not(debug_assertions), target_arch = \"x86\", not(target_feature = \"sse2\"))) {\n // TODO: rustc bug: https://github.com/rust-lang/rust/issues/114479\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_float_op_eq!(a.fetch_add(y, order), x);\n assert_float_op_eq!(a.load(Ordering::Relaxed), x + y);\n let a = <$atomic_type>::new(y);\n assert_float_op_eq!(a.fetch_add(x, order), y);\n assert_float_op_eq!(a.load(Ordering::Relaxed), y + x);\n }\n true\n }\n fn quickcheck_fetch_sub(x: $float_type, y: $float_type) -> bool {\n if cfg!(all(not(debug_assertions), target_arch = \"x86\", not(target_feature = \"sse2\"))) {\n // TODO: rustc bug: https://github.com/rust-lang/rust/issues/114479\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_float_op_eq!(a.fetch_sub(y, order), x);\n assert_float_op_eq!(a.load(Ordering::Relaxed), x - y);\n let a = <$atomic_type>::new(y);\n assert_float_op_eq!(a.fetch_sub(x, order), y);\n assert_float_op_eq!(a.load(Ordering::Relaxed), y - x);\n }\n true\n }\n fn quickcheck_fetch_max(x: $float_type, y: $float_type) -> bool {\n if mem::size_of::<$float_type>() == 16\n && cfg!(any(\n target_arch = \"arm\",\n target_arch = \"mips\",\n target_arch = \"mips32r6\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_vendor = \"apple\",\n windows,\n ))\n {\n // TODO(f128):\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_float_op_eq!(a.fetch_max(y, order), x);\n assert_float_op_eq!(a.load(Ordering::Relaxed), x.max(y));\n let a = <$atomic_type>::new(y);\n assert_float_op_eq!(a.fetch_max(x, order), y);\n assert_float_op_eq!(a.load(Ordering::Relaxed), y.max(x));\n }\n true\n }\n fn quickcheck_fetch_min(x: $float_type, y: $float_type) -> bool {\n if mem::size_of::<$float_type>() == 16\n && cfg!(any(\n target_arch = \"arm\",\n target_arch = \"mips\",\n target_arch = \"mips32r6\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_vendor = \"apple\",\n windows,\n ))\n {\n // TODO(f128):\n return true;\n }\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_float_op_eq!(a.fetch_min(y, order), x);\n assert_float_op_eq!(a.load(Ordering::Relaxed), x.min(y));\n let a = <$atomic_type>::new(y);\n assert_float_op_eq!(a.fetch_min(x, order), y);\n assert_float_op_eq!(a.load(Ordering::Relaxed), y.min(x));\n }\n true\n }\n fn quickcheck_fetch_neg(x: $float_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_float_op_eq!(a.fetch_neg(order), x);\n assert_float_op_eq!(a.load(Ordering::Relaxed), -x);\n assert_float_op_eq!(a.fetch_neg(order), -x);\n assert_float_op_eq!(a.load(Ordering::Relaxed), x);\n }\n true\n }\n fn quickcheck_fetch_abs(x: $float_type) -> bool {\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_float_op_eq!(a.fetch_abs(order), x);\n assert_float_op_eq!(a.fetch_abs(order), x.abs());\n assert_float_op_eq!(a.load(Ordering::Relaxed), x.abs());\n }\n true\n }\n }\n };\n ($atomic_type:ty, $float_type:ident) => {\n __test_atomic_float!($atomic_type, $float_type, single_thread);\n // TODO: multi thread\n };\n}\nmacro_rules! __test_atomic_bool {\n ($atomic_type:ty, single_thread) => {\n #[test]\n fn swap() {\n let a = <$atomic_type>::new(true);\n test_swap_ordering(&|order| {\n a.swap(true, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n assert_eq!(a.swap(true, order), true);\n assert_eq!(a.swap(false, order), true);\n assert_eq!(a.swap(false, order), false);\n assert_eq!(a.swap(true, order), false);\n }\n }\n #[test]\n fn compare_exchange() {\n let a = <$atomic_type>::new(true);\n test_compare_exchange_ordering(&|success, failure| {\n let _ = a.compare_exchange(true, true, success, failure);\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(true);\n assert_eq!(a.compare_exchange(true, false, success, failure), Ok(true));\n assert_eq!(a.load(Ordering::Relaxed), false);\n assert_eq!(a.compare_exchange(true, true, success, failure), Err(false));\n assert_eq!(a.load(Ordering::Relaxed), false);\n }\n }\n #[test]\n fn compare_exchange_weak() {\n let a = <$atomic_type>::new(false);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS {\n mark_aligned_defined(&a);\n }\n test_compare_exchange_ordering(&|success, failure| {\n let _ = a.compare_exchange_weak(false, false, success, failure);\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(false);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.compare_exchange_weak(true, true, success, failure), Err(false));\n let mut old = a.load(Ordering::Relaxed);\n let new = true;\n loop {\n match a.compare_exchange_weak(old, new, success, failure) {\n Ok(_) => break,\n Err(x) => old = x,\n }\n }\n assert_eq!(a.load(Ordering::Relaxed), true);\n }\n }\n #[test]\n fn fetch_and() {\n let a = <$atomic_type>::new(true);\n test_swap_ordering(&|order| assert_eq!(a.fetch_and(true, order), true));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(true);\n assert_eq!(a.fetch_and(false, order), true);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(true);\n assert_eq!(a.fetch_and(true, order), true);\n assert_eq!(a.load(Ordering::Relaxed), true);\n let a = <$atomic_type>::new(false);\n assert_eq!(a.fetch_and(false, order), false);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(false);\n assert_eq!(a.fetch_and(true, order), false);\n assert_eq!(a.load(Ordering::Relaxed), false);\n }\n }\n #[test]\n fn and() {\n let a = <$atomic_type>::new(true);\n test_swap_ordering(&|order| a.and(true, order));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(true);\n a.and(false, order);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(true);\n a.and(true, order);\n assert_eq!(a.load(Ordering::Relaxed), true);\n let a = <$atomic_type>::new(false);\n a.and(false, order);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(false);\n a.and(true, order);\n assert_eq!(a.load(Ordering::Relaxed), false);\n }\n }\n #[test]\n fn fetch_or() {\n let a = <$atomic_type>::new(true);\n test_swap_ordering(&|order| assert_eq!(a.fetch_or(false, order), true));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(true);\n assert_eq!(a.fetch_or(false, order), true);\n assert_eq!(a.load(Ordering::Relaxed), true);\n let a = <$atomic_type>::new(true);\n assert_eq!(a.fetch_or(true, order), true);\n assert_eq!(a.load(Ordering::Relaxed), true);\n let a = <$atomic_type>::new(false);\n assert_eq!(a.fetch_or(false, order), false);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(false);\n assert_eq!(a.fetch_or(true, order), false);\n assert_eq!(a.load(Ordering::Relaxed), true);\n }\n }\n #[test]\n fn or() {\n let a = <$atomic_type>::new(true);\n test_swap_ordering(&|order| a.or(false, order));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(true);\n a.or(false, order);\n assert_eq!(a.load(Ordering::Relaxed), true);\n let a = <$atomic_type>::new(true);\n a.or(true, order);\n assert_eq!(a.load(Ordering::Relaxed), true);\n let a = <$atomic_type>::new(false);\n a.or(false, order);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(false);\n a.or(true, order);\n assert_eq!(a.load(Ordering::Relaxed), true);\n }\n }\n #[test]\n fn fetch_xor() {\n let a = <$atomic_type>::new(true);\n test_swap_ordering(&|order| assert_eq!(a.fetch_xor(false, order), true));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(true);\n assert_eq!(a.fetch_xor(false, order), true);\n assert_eq!(a.load(Ordering::Relaxed), true);\n let a = <$atomic_type>::new(true);\n assert_eq!(a.fetch_xor(true, order), true);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(false);\n assert_eq!(a.fetch_xor(false, order), false);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(false);\n assert_eq!(a.fetch_xor(true, order), false);\n assert_eq!(a.load(Ordering::Relaxed), true);\n }\n }\n #[test]\n fn xor() {\n let a = <$atomic_type>::new(true);\n test_swap_ordering(&|order| a.xor(false, order));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(true);\n a.xor(false, order);\n assert_eq!(a.load(Ordering::Relaxed), true);\n let a = <$atomic_type>::new(true);\n a.xor(true, order);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(false);\n a.xor(false, order);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(false);\n a.xor(true, order);\n assert_eq!(a.load(Ordering::Relaxed), true);\n }\n }\n ::quickcheck::quickcheck! {\n fn quickcheck_compare_exchange(x: bool, y: bool) -> bool {\n let z = !y;\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(x);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.compare_exchange(x, y, success, failure).unwrap(), x);\n assert_eq!(a.load(Ordering::Relaxed), y);\n assert_eq!(a.compare_exchange(z, x, success, failure).unwrap_err(), y);\n assert_eq!(a.load(Ordering::Relaxed), y);\n }\n true\n }\n }\n };\n ($atomic_type:ty) => {\n __test_atomic_bool!($atomic_type, single_thread);\n // TODO: multi thread\n };\n}\nmacro_rules! __test_atomic_ptr {\n ($atomic_type:ty, single_thread) => {\n #[allow(unused_imports)]\n use sptr::Strict as _; // for old rustc\n #[test]\n fn swap() {\n let a = <$atomic_type>::new(ptr::null_mut());\n test_swap_ordering(&|order| {\n a.swap(ptr::null_mut(), order);\n });\n let x = &mut 1;\n for &order in &helper::SWAP_ORDERINGS {\n assert_eq!(a.swap(x, order), ptr::null_mut());\n assert_eq!(a.swap(ptr::null_mut(), order), x as *mut _);\n }\n }\n #[test]\n fn compare_exchange() {\n let a = <$atomic_type>::new(ptr::null_mut());\n test_compare_exchange_ordering(&|success, failure| {\n let _ = a.compare_exchange(ptr::null_mut(), ptr::null_mut(), success, failure);\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(ptr::null_mut());\n let x = &mut 1;\n assert_eq!(\n a.compare_exchange(ptr::null_mut(), x, success, failure),\n Ok(ptr::null_mut()),\n );\n assert_eq!(a.load(Ordering::Relaxed), x as *mut _);\n assert_eq!(\n a.compare_exchange(ptr::null_mut(), ptr::null_mut(), success, failure),\n Err(x as *mut _),\n );\n assert_eq!(a.load(Ordering::Relaxed), x as *mut _);\n }\n }\n #[test]\n fn compare_exchange_weak() {\n let a = <$atomic_type>::new(ptr::null_mut());\n test_compare_exchange_ordering(&|success, failure| {\n let _ = a.compare_exchange_weak(ptr::null_mut(), ptr::null_mut(), success, failure);\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(ptr::null_mut());\n let x = &mut 1;\n assert_eq!(a.compare_exchange_weak(x, x, success, failure), Err(ptr::null_mut()));\n let mut old = a.load(Ordering::Relaxed);\n loop {\n match a.compare_exchange_weak(old, x, success, failure) {\n Ok(_) => break,\n Err(x) => old = x,\n }\n }\n assert_eq!(a.load(Ordering::Relaxed), x as *mut _);\n }\n }\n // https://github.com/rust-lang/rust/blob/1.84.0/library/core/tests/atomic.rs#L130-L213\n #[test]\n fn ptr_add_null() {\n let atom = AtomicPtr::::new(core::ptr::null_mut());\n assert_eq!(atom.fetch_ptr_add(1, Ordering::SeqCst).addr(), 0);\n assert_eq!(atom.load(Ordering::SeqCst).addr(), 8);\n\n assert_eq!(atom.fetch_byte_add(1, Ordering::SeqCst).addr(), 8);\n assert_eq!(atom.load(Ordering::SeqCst).addr(), 9);\n\n assert_eq!(atom.fetch_ptr_sub(1, Ordering::SeqCst).addr(), 9);\n assert_eq!(atom.load(Ordering::SeqCst).addr(), 1);\n\n assert_eq!(atom.fetch_byte_sub(1, Ordering::SeqCst).addr(), 1);\n assert_eq!(atom.load(Ordering::SeqCst).addr(), 0);\n }\n #[test]\n fn ptr_add_data() {\n let num = 0i64;\n let n = &num as *const i64 as *mut _;\n let atom = AtomicPtr::::new(n);\n assert_eq!(atom.fetch_ptr_add(1, Ordering::SeqCst), n);\n assert_eq!(atom.load(Ordering::SeqCst), n.wrapping_add(1));\n\n assert_eq!(atom.fetch_ptr_sub(1, Ordering::SeqCst), n.wrapping_add(1));\n assert_eq!(atom.load(Ordering::SeqCst), n);\n #[allow(clippy::cast_ptr_alignment)]\n let bytes_from_n = |b| n.cast::().wrapping_add(b).cast::();\n\n assert_eq!(atom.fetch_byte_add(1, Ordering::SeqCst), n);\n assert_eq!(atom.load(Ordering::SeqCst), bytes_from_n(1));\n\n assert_eq!(atom.fetch_byte_add(5, Ordering::SeqCst), bytes_from_n(1));\n assert_eq!(atom.load(Ordering::SeqCst), bytes_from_n(6));\n\n assert_eq!(atom.fetch_byte_sub(1, Ordering::SeqCst), bytes_from_n(6));\n assert_eq!(atom.load(Ordering::SeqCst), bytes_from_n(5));\n\n assert_eq!(atom.fetch_byte_sub(5, Ordering::SeqCst), bytes_from_n(5));\n assert_eq!(atom.load(Ordering::SeqCst), n);\n }\n #[test]\n fn ptr_bitops() {\n let atom = AtomicPtr::::new(core::ptr::null_mut());\n assert_eq!(atom.fetch_or(0b0111, Ordering::SeqCst).addr(), 0);\n assert_eq!(atom.load(Ordering::SeqCst).addr(), 0b0111);\n\n assert_eq!(atom.fetch_and(0b1101, Ordering::SeqCst).addr(), 0b0111);\n assert_eq!(atom.load(Ordering::SeqCst).addr(), 0b0101);\n\n assert_eq!(atom.fetch_xor(0b1111, Ordering::SeqCst).addr(), 0b0101);\n assert_eq!(atom.load(Ordering::SeqCst).addr(), 0b1010);\n }\n #[test]\n fn ptr_bitops_tagging() {\n const MASK_TAG: usize = 0b1111;\n const MASK_PTR: usize = !MASK_TAG;\n\n #[repr(align(16))]\n struct Tagme(#[allow(dead_code)] u128);\n\n let tagme = Tagme(1000);\n let ptr = &tagme as *const Tagme as *mut Tagme;\n let atom: AtomicPtr = AtomicPtr::new(ptr);\n\n assert_eq!(ptr.addr() & MASK_TAG, 0);\n\n assert_eq!(atom.fetch_or(0b0111, Ordering::SeqCst), ptr);\n assert_eq!(atom.load(Ordering::SeqCst), ptr.map_addr(|a| a | 0b111));\n\n assert_eq!(\n atom.fetch_and(MASK_PTR | 0b0010, Ordering::SeqCst),\n ptr.map_addr(|a| a | 0b111)\n );\n assert_eq!(atom.load(Ordering::SeqCst), ptr.map_addr(|a| a | 0b0010));\n\n assert_eq!(atom.fetch_xor(0b1011, Ordering::SeqCst), ptr.map_addr(|a| a | 0b0010));\n assert_eq!(atom.load(Ordering::SeqCst), ptr.map_addr(|a| a | 0b1001));\n\n assert_eq!(atom.fetch_and(MASK_PTR, Ordering::SeqCst), ptr.map_addr(|a| a | 0b1001));\n assert_eq!(atom.load(Ordering::SeqCst), ptr);\n }\n #[test]\n fn bit_set() {\n let a = <$atomic_type>::new(ptr::null_mut::().cast::().map_addr(|a| a | 1));\n test_swap_ordering(&|order| assert!(a.bit_set(0, order)));\n for &order in &helper::SWAP_ORDERINGS {\n let pointer = &mut 1u64 as *mut u64 as *mut u8;\n let atom = <$atomic_type>::new(pointer);\n // Tag the bottom bit of the pointer.\n assert!(!atom.bit_set(0, order));\n // Extract and untag.\n let tagged = atom.load(Ordering::Relaxed);\n assert_eq!(tagged.addr() & 1, 1);\n assert_eq!(tagged.map_addr(|p| p & !1), pointer);\n }\n }\n #[test]\n fn bit_clear() {\n let a = <$atomic_type>::new(ptr::null_mut::().cast::());\n test_swap_ordering(&|order| assert!(!a.bit_clear(0, order)));\n for &order in &helper::SWAP_ORDERINGS {\n let pointer = &mut 1u64 as *mut u64 as *mut u8;\n // A tagged pointer\n let atom = <$atomic_type>::new(pointer.map_addr(|a| a | 1));\n assert!(atom.bit_set(0, order));\n // Untag\n assert!(atom.bit_clear(0, order));\n }\n }\n #[test]\n fn bit_toggle() {\n let a = <$atomic_type>::new(ptr::null_mut::().cast::());\n test_swap_ordering(&|order| {\n a.bit_toggle(0, order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let pointer = &mut 1u64 as *mut u64 as *mut u8;\n let atom = <$atomic_type>::new(pointer);\n // Toggle a tag bit on the pointer.\n atom.bit_toggle(0, order);\n assert_eq!(atom.load(Ordering::Relaxed).addr() & 1, 1);\n }\n }\n ::quickcheck::quickcheck! {\n fn quickcheck_swap(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.swap(y, order), x);\n assert_eq!(a.swap(x, order), y);\n }\n true\n }\n fn quickcheck_compare_exchange(x: usize, y: usize) -> bool {\n let mut rng = fastrand::Rng::new();\n let z = loop {\n let z = rng.usize(..);\n if z != y {\n break z;\n }\n };\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut(y);\n let z = sptr::invalid_mut(z);\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.compare_exchange(x, y, success, failure).unwrap(), x);\n assert_eq!(a.load(Ordering::Relaxed), y);\n assert_eq!(a.compare_exchange(z, x, success, failure).unwrap_err(), y);\n assert_eq!(a.load(Ordering::Relaxed), y);\n }\n true\n }\n fn quickcheck_fetch_byte_add(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut::(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_byte_add(y.addr(), order), x);\n assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr().wrapping_add(y.addr()));\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_byte_add(x.addr(), order), y);\n assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr().wrapping_add(x.addr()));\n }\n true\n }\n fn quickcheck_fetch_byte_sub(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut::(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_byte_sub(y.addr(), order), x);\n assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr().wrapping_sub(y.addr()));\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_byte_sub(x.addr(), order), y);\n assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr().wrapping_sub(x.addr()));\n }\n true\n }\n fn quickcheck_fetch_and(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut::(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_and(y.addr(), order), x);\n assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() & y.addr());\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_and(x.addr(), order), y);\n assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() & x.addr());\n }\n true\n }\n fn quickcheck_fetch_or(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut::(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_or(y.addr(), order), x);\n assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() | y.addr());\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_or(x.addr(), order), y);\n assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() | x.addr());\n }\n true\n }\n fn quickcheck_fetch_xor(x: usize, y: usize) -> bool {\n let x = sptr::invalid_mut(x);\n let y = sptr::invalid_mut::(y);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(a.fetch_xor(y.addr(), order), x);\n assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() ^ y.addr());\n let a = <$atomic_type>::new(y);\n assert_eq!(a.fetch_xor(x.addr(), order), y);\n assert_eq!(a.load(Ordering::Relaxed).addr(), y.addr() ^ x.addr());\n }\n true\n }\n fn quickcheck_bit_set(x: usize, bit: u32) -> bool {\n let x = sptr::invalid_mut(x);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n let b = a.bit_set(bit, order);\n let mask = ::wrapping_shl(1, bit);\n assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() | mask);\n assert_eq!(b, x.addr() & mask != 0);\n }\n true\n }\n fn quickcheck_bit_clear(x: usize, bit: u32) -> bool {\n let x = sptr::invalid_mut(x);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n let b = a.bit_clear(bit, order);\n let mask = ::wrapping_shl(1, bit);\n assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() & !mask);\n assert_eq!(b, x.addr() & mask != 0);\n }\n true\n }\n fn quickcheck_bit_toggle(x: usize, bit: u32) -> bool {\n let x = sptr::invalid_mut(x);\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(x);\n let b = a.bit_toggle(bit, order);\n let mask = ::wrapping_shl(1, bit);\n assert_eq!(a.load(Ordering::Relaxed).addr(), x.addr() ^ mask);\n assert_eq!(b, x.addr() & mask != 0);\n }\n true\n }\n }\n };\n ($atomic_type:ty) => {\n __test_atomic_ptr!($atomic_type, single_thread);\n // TODO: multi thread\n };\n}\n\nmacro_rules! __test_atomic_int_pub {\n ($atomic_type:ty, $int_type:ident) => {\n __test_atomic_pub_common!($atomic_type, $int_type);\n use std::boxed::Box;\n\n #[allow(unused_imports)]\n use sptr::Strict as _; // for old rustc\n #[test]\n fn fetch_update() {\n // TODO(gcc): failure\n if option_env!(\"RUSTC_CODEGEN_GCC\").unwrap_or_default() == \"1\"\n && !cfg!(debug_assertions)\n {\n return;\n }\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return;\n }\n let a = <$atomic_type>::new(7);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n test_compare_exchange_ordering(&|set, fetch| {\n let _ = a.fetch_update(set, fetch, |x| Some(x));\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(7);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS && mem::size_of::<$int_type>() <= 2 {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_update(success, failure, |_| None), Err(7));\n assert_eq!(a.fetch_update(success, failure, |x| Some(x + 1)), Ok(7));\n assert_eq!(a.fetch_update(success, failure, |x| Some(x + 1)), Ok(8));\n assert_eq!(a.load(Ordering::SeqCst), 9);\n }\n }\n #[test]\n fn impls() {\n #[cfg(not(portable_atomic_no_const_transmute))]\n const INTO_INNER: $int_type = {\n let a = <$atomic_type>::new(10);\n a.into_inner()\n };\n #[cfg(not(portable_atomic_no_const_mut_refs))]\n const GET_MUT: $atomic_type = {\n let mut a = <$atomic_type>::new(10);\n let _ = unsafe { <$atomic_type>::from_ptr(a.as_ptr()) };\n *a.get_mut() = 5;\n a\n };\n #[cfg(not(portable_atomic_no_const_transmute))]\n {\n assert_eq!(INTO_INNER, 10);\n }\n #[cfg(not(portable_atomic_no_const_mut_refs))]\n {\n assert_eq!(GET_MUT.into_inner(), 5);\n }\n let a = <$atomic_type>::default();\n let b = <$atomic_type>::from(0);\n assert_eq!(a.load(Ordering::SeqCst), b.load(Ordering::SeqCst));\n assert_eq!(std::format!(\"{:?}\", a), std::format!(\"{:?}\", a.load(Ordering::SeqCst)));\n assert_eq!(a.into_inner(), 0);\n assert_eq!(b.into_inner(), 0);\n\n unsafe {\n let ptr: *mut Align16<$int_type> = Box::into_raw(Box::new(Align16(0)));\n assert!(ptr.addr() % mem::align_of::<$atomic_type>() == 0);\n {\n let a = <$atomic_type>::from_ptr(ptr.cast::<$int_type>());\n *a.as_ptr() = 1;\n }\n assert_eq!((*ptr).0, 1);\n drop(Box::from_raw(ptr));\n }\n }\n ::quickcheck::quickcheck! {\n fn quickcheck_fetch_update(x: $int_type, y: $int_type) -> bool {\n // TODO(riscv): wrong result (as of Valgrind 3.26)\n #[cfg(valgrind)]\n if cfg!(target_arch = \"riscv64\") && mem::size_of::<$int_type>() <= 2 {\n return true;\n }\n let mut rng = fastrand::Rng::new();\n let z = loop {\n let z = rng.$int_type(..);\n if z != y {\n break z;\n }\n };\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(x);\n assert_eq!(\n a.fetch_update(success, failure, |_| Some(y))\n .unwrap(),\n x\n );\n assert_eq!(\n a.fetch_update(success, failure, |_| Some(z))\n .unwrap(),\n y\n );\n assert_eq!(a.load(Ordering::Relaxed), z);\n assert_eq!(\n a.fetch_update(success, failure, |z| if z == y { Some(z) } else { None })\n .unwrap_err(),\n z\n );\n assert_eq!(a.load(Ordering::Relaxed), z);\n }\n true\n }\n }\n };\n}\nmacro_rules! __test_atomic_float_pub {\n ($atomic_type:ty, $float_type:ident) => {\n __test_atomic_pub_common!($atomic_type, $float_type);\n use std::boxed::Box;\n\n #[allow(unused_imports)]\n use sptr::Strict as _; // for old rustc\n #[test]\n fn fetch_update() {\n let a = <$atomic_type>::new(7.);\n test_compare_exchange_ordering(&|set, fetch| {\n let _ = a.fetch_update(set, fetch, |x| Some(x));\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(7.);\n assert_eq!(a.fetch_update(success, failure, |_| None), Err(7.));\n assert_eq!(a.fetch_update(success, failure, |x| Some(x + 1.)), Ok(7.));\n assert_eq!(a.fetch_update(success, failure, |x| Some(x + 1.)), Ok(8.));\n assert_eq!(a.load(Ordering::SeqCst), 9.);\n }\n }\n #[test]\n fn impls() {\n #[cfg(not(portable_atomic_no_const_transmute))]\n const INTO_INNER: $float_type = {\n let a = <$atomic_type>::new(10.);\n a.into_inner()\n };\n #[cfg(not(portable_atomic_no_const_mut_refs))]\n const GET_MUT: $atomic_type = {\n let mut a = <$atomic_type>::new(10.);\n let _ = unsafe { <$atomic_type>::from_ptr(a.as_ptr()) };\n *a.get_mut() = 5.;\n a\n };\n #[cfg(not(portable_atomic_no_const_transmute))]\n {\n assert_eq!(INTO_INNER, 10.);\n }\n #[cfg(not(portable_atomic_no_const_mut_refs))]\n {\n assert_eq!(GET_MUT.into_inner(), 5.);\n }\n let a = <$atomic_type>::default();\n let b = <$atomic_type>::from(0.);\n assert_eq!(a.load(Ordering::SeqCst), b.load(Ordering::SeqCst));\n assert_eq!(std::format!(\"{:?}\", a), std::format!(\"{:?}\", a.load(Ordering::SeqCst)));\n assert_eq!(a.into_inner(), 0.);\n assert_eq!(b.into_inner(), 0.);\n\n unsafe {\n let ptr: *mut Align16<$float_type> = Box::into_raw(Box::new(Align16(0.)));\n assert!(ptr.addr() % mem::align_of::<$atomic_type>() == 0);\n {\n let a = <$atomic_type>::from_ptr(ptr.cast::<$float_type>());\n *a.as_ptr() = 1.;\n }\n assert_eq!((*ptr).0, 1.);\n drop(Box::from_raw(ptr));\n }\n }\n };\n}\nmacro_rules! __test_atomic_bool_pub {\n ($atomic_type:ty) => {\n __test_atomic_pub_common!($atomic_type, bool);\n use std::boxed::Box;\n\n #[allow(unused_imports)]\n use sptr::Strict as _; // for old rustc\n #[test]\n fn fetch_nand() {\n let a = <$atomic_type>::new(true);\n test_swap_ordering(&|order| assert_eq!(a.fetch_nand(false, order), true));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(true);\n assert_eq!(a.fetch_nand(false, order), true);\n assert_eq!(a.load(Ordering::Relaxed), true);\n let a = <$atomic_type>::new(true);\n assert_eq!(a.fetch_nand(true, order), true);\n assert_eq!(a.load(Ordering::Relaxed) as usize, 0);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(false);\n assert_eq!(a.fetch_nand(false, order), false);\n assert_eq!(a.load(Ordering::Relaxed), true);\n let a = <$atomic_type>::new(false);\n assert_eq!(a.fetch_nand(true, order), false);\n assert_eq!(a.load(Ordering::Relaxed), true);\n }\n }\n #[test]\n fn fetch_not() {\n let a = <$atomic_type>::new(true);\n test_swap_ordering(&|order| {\n a.fetch_not(order);\n });\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(true);\n assert_eq!(a.fetch_not(order), true);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(false);\n assert_eq!(a.fetch_not(order), false);\n assert_eq!(a.load(Ordering::Relaxed), true);\n }\n }\n #[test]\n fn not() {\n let a = <$atomic_type>::new(true);\n test_swap_ordering(&|order| a.not(order));\n for &order in &helper::SWAP_ORDERINGS {\n let a = <$atomic_type>::new(true);\n a.not(order);\n assert_eq!(a.load(Ordering::Relaxed), false);\n let a = <$atomic_type>::new(false);\n a.not(order);\n assert_eq!(a.load(Ordering::Relaxed), true);\n }\n }\n #[test]\n fn fetch_update() {\n // TODO(gcc): failure\n if option_env!(\"RUSTC_CODEGEN_GCC\").unwrap_or_default() == \"1\"\n && !cfg!(debug_assertions)\n {\n return;\n }\n let a = <$atomic_type>::new(false);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS {\n mark_aligned_defined(&a);\n }\n test_compare_exchange_ordering(&|set, fetch| {\n let _ = a.fetch_update(set, fetch, |x| Some(x));\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(false);\n #[cfg(valgrind)]\n if IMP_EMU_SUB_WORD_CAS {\n mark_aligned_defined(&a);\n }\n assert_eq!(a.fetch_update(success, failure, |_| None), Err(false));\n assert_eq!(a.fetch_update(success, failure, |x| Some(!x)), Ok(false));\n assert_eq!(a.fetch_update(success, failure, |x| Some(!x)), Ok(true));\n assert_eq!(a.load(Ordering::SeqCst), false);\n }\n }\n #[test]\n fn impls() {\n #[cfg(not(portable_atomic_no_const_transmute))]\n const INTO_INNER: bool = {\n let a = <$atomic_type>::new(true);\n a.into_inner()\n };\n #[cfg(not(portable_atomic_no_const_mut_refs))]\n const GET_MUT: $atomic_type = {\n let mut a = <$atomic_type>::new(true);\n let _ = unsafe { <$atomic_type>::from_ptr(a.as_ptr()) };\n *a.get_mut() = false;\n a\n };\n #[cfg(not(portable_atomic_no_const_transmute))]\n {\n assert_eq!(INTO_INNER, true);\n }\n #[cfg(not(portable_atomic_no_const_mut_refs))]\n {\n assert_eq!(GET_MUT.into_inner(), false);\n }\n let a = <$atomic_type>::default();\n let b = <$atomic_type>::from(false);\n assert_eq!(a.load(Ordering::SeqCst), b.load(Ordering::SeqCst));\n assert_eq!(std::format!(\"{:?}\", a), std::format!(\"{:?}\", a.load(Ordering::SeqCst)));\n assert_eq!(a.into_inner(), false);\n assert_eq!(b.into_inner(), false);\n\n unsafe {\n let ptr: *mut bool = Box::into_raw(Box::new(false));\n assert!(ptr.addr() % mem::align_of::<$atomic_type>() == 0);\n {\n let a = <$atomic_type>::from_ptr(ptr);\n *a.as_ptr() = true;\n }\n assert_eq!((*ptr), true);\n drop(Box::from_raw(ptr));\n }\n }\n };\n}\nmacro_rules! __test_atomic_ptr_pub {\n ($atomic_type:ty) => {\n __test_atomic_pub_common!($atomic_type, *mut u8);\n use std::boxed::Box;\n #[test]\n fn fetch_update() {\n // TODO(gcc): failure\n if option_env!(\"RUSTC_CODEGEN_GCC\").unwrap_or_default() == \"1\"\n && !cfg!(debug_assertions)\n {\n return;\n }\n let a = <$atomic_type>::new(ptr::null_mut());\n test_compare_exchange_ordering(&|set, fetch| {\n let _ = a.fetch_update(set, fetch, |x| Some(x));\n });\n for &(success, failure) in &helper::COMPARE_EXCHANGE_ORDERINGS {\n let a = <$atomic_type>::new(ptr::null_mut());\n assert_eq!(a.fetch_update(success, failure, |_| None), Err(ptr::null_mut()));\n assert_eq!(\n a.fetch_update(success, failure, |_| Some(&a as *const _ as *mut _)),\n Ok(ptr::null_mut())\n );\n assert_eq!(a.load(Ordering::SeqCst), &a as *const _ as *mut _);\n }\n }\n #[test]\n fn impls() {\n #[cfg(not(portable_atomic_no_const_transmute))]\n const INTO_INNER: *mut u8 = {\n let a = <$atomic_type>::new(ptr::null_mut());\n a.into_inner()\n };\n #[cfg(not(portable_atomic_no_const_mut_refs))]\n const GET_MUT: $atomic_type = {\n let mut a = <$atomic_type>::new(ptr::null_mut());\n let _ = unsafe { <$atomic_type>::from_ptr(a.as_ptr()) };\n *a.get_mut() = ptr::null_mut::().wrapping_add(1);\n a\n };\n #[cfg(not(portable_atomic_no_const_transmute))]\n {\n assert!(INTO_INNER.is_null());\n }\n #[cfg(not(portable_atomic_no_const_mut_refs))]\n {\n assert_eq!(GET_MUT.into_inner(), ptr::null_mut::().wrapping_add(1));\n }\n let a = <$atomic_type>::default();\n let b = <$atomic_type>::from(ptr::null_mut());\n assert_eq!(a.load(Ordering::SeqCst), b.load(Ordering::SeqCst));\n assert_eq!(std::format!(\"{:?}\", a), std::format!(\"{:?}\", a.load(Ordering::SeqCst)));\n assert_eq!(std::format!(\"{:p}\", a), std::format!(\"{:p}\", a.load(Ordering::SeqCst)));\n assert_eq!(a.into_inner(), ptr::null_mut());\n assert_eq!(b.into_inner(), ptr::null_mut());\n\n unsafe {\n let ptr: *mut Align16<*mut u8> = Box::into_raw(Box::new(Align16(ptr::null_mut())));\n assert!(ptr.addr() % mem::align_of::<$atomic_type>() == 0);\n {\n let a = <$atomic_type>::from_ptr(ptr.cast::<*mut u8>());\n *a.as_ptr() = ptr::null_mut::().wrapping_add(1);\n }\n assert_eq!((*ptr).0, ptr::null_mut::().wrapping_add(1));\n drop(Box::from_raw(ptr));\n }\n }\n };\n}\n\nmacro_rules! test_atomic_int_load_store {\n ($int_type:ident) => {\n paste::paste! {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod [] {\n use super::*;\n __test_atomic_int_load_store!([], $int_type);\n }\n }\n };\n}\nmacro_rules! test_atomic_ptr_load_store {\n () => {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod test_atomic_ptr {\n use super::*;\n __test_atomic_ptr_load_store!(AtomicPtr);\n }\n };\n}\n\nmacro_rules! test_atomic_int_single_thread {\n ($int_type:ident) => {\n paste::paste! {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod [] {\n use super::*;\n __test_atomic_int_load_store!([], $int_type, single_thread);\n __test_atomic_int!([], $int_type, single_thread);\n }\n }\n };\n}\nmacro_rules! test_atomic_ptr_single_thread {\n () => {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod test_atomic_ptr {\n use super::*;\n __test_atomic_ptr_load_store!(AtomicPtr, single_thread);\n __test_atomic_ptr!(AtomicPtr, single_thread);\n }\n };\n}\n\nmacro_rules! test_atomic_int {\n ($int_type:ident) => {\n paste::paste! {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod [] {\n use super::*;\n __test_atomic_int_load_store!([], $int_type);\n __test_atomic_int!([], $int_type);\n }\n }\n };\n}\nmacro_rules! test_atomic_ptr {\n () => {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n #[allow(unstable_name_collisions)] // for sptr crate\n mod test_atomic_ptr {\n use super::*;\n __test_atomic_ptr_load_store!(AtomicPtr);\n __test_atomic_ptr!(AtomicPtr);\n }\n };\n}\n\nmacro_rules! test_atomic_int_pub {\n ($int_type:ident) => {\n paste::paste! {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod [] {\n use super::*;\n __test_atomic_int_load_store!([], $int_type);\n __test_atomic_int!([], $int_type);\n __test_atomic_int_pub!([], $int_type);\n }\n }\n };\n}\n#[cfg(feature = \"float\")]\nmacro_rules! test_atomic_float_pub {\n ($float_type:ident) => {\n paste::paste! {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::float_arithmetic,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod [] {\n use super::*;\n __test_atomic_float_load_store!([], $float_type);\n __test_atomic_float!([], $float_type);\n __test_atomic_float_pub!([], $float_type);\n }\n }\n };\n}\nmacro_rules! test_atomic_bool_pub {\n () => {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod test_atomic_bool {\n use super::*;\n __test_atomic_bool_load_store!(AtomicBool);\n __test_atomic_bool!(AtomicBool);\n __test_atomic_bool_pub!(AtomicBool);\n }\n };\n}\nmacro_rules! test_atomic_ptr_pub {\n () => {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n #[allow(unstable_name_collisions)] // for sptr crate\n mod test_atomic_ptr {\n use super::*;\n __test_atomic_ptr_load_store!(AtomicPtr);\n __test_atomic_ptr!(AtomicPtr);\n __test_atomic_ptr_pub!(AtomicPtr);\n }\n };\n}\n\n// Asserts that `$a` and `$b` have performed equivalent operations.\n#[cfg(feature = \"float\")]\nmacro_rules! assert_float_op_eq {\n ($a:expr, $b:expr $(,)?) => {{\n // See also:\n // - https://github.com/rust-lang/unsafe-code-guidelines/issues/237.\n // - https://github.com/rust-lang/portable-simd/issues/39.\n let a = $a;\n let b = $b;\n if a.is_nan() && b.is_nan() // don't check sign of NaN: https://github.com/rust-lang/rust/issues/55131\n || a.is_infinite()\n && b.is_infinite()\n && a.is_sign_positive() == b.is_sign_positive()\n && a.is_sign_negative() == b.is_sign_negative()\n {\n // ok\n } else {\n assert_eq!(a, b);\n }\n }};\n}\n\n#[allow(unused_unsafe)] // for old rustc\nfn assert_panic(f: &dyn Fn()) -> std::string::String {\n let backtrace = std::env::var_os(\"RUST_BACKTRACE\");\n let hook = std::panic::take_hook();\n // set_var/remove_var is fine as we run tests with RUST_TEST_THREADS=1\n // std::panic::set_backtrace_style is better way here, but is unstable.\n unsafe { std::env::set_var(\"RUST_BACKTRACE\", \"0\") } // Suppress backtrace\n std::panic::set_hook(std::boxed::Box::new(|_| {})); // Suppress panic msg\n let res = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f));\n std::panic::set_hook(hook);\n match backtrace {\n Some(v) => unsafe { std::env::set_var(\"RUST_BACKTRACE\", v) },\n None => unsafe { std::env::remove_var(\"RUST_BACKTRACE\") },\n }\n let msg = res.unwrap_err();\n msg.downcast_ref::()\n .cloned()\n .unwrap_or_else(|| msg.downcast_ref::<&'static str>().copied().unwrap().into())\n}\npub(crate) const LOAD_ORDERINGS: [Ordering; 3] =\n [Ordering::Relaxed, Ordering::Acquire, Ordering::SeqCst];\npub(crate) fn rand_load_ordering(rng: &mut fastrand::Rng) -> Ordering {\n LOAD_ORDERINGS[rng.usize(0..LOAD_ORDERINGS.len())]\n}\npub(crate) fn test_load_ordering(f: &dyn Fn(Ordering)) {\n for &order in &LOAD_ORDERINGS {\n f(order);\n }\n if !skip_should_panic_test() {\n assert_eq!(\n assert_panic(&|| f(Ordering::Release)),\n \"there is no such thing as a release load\"\n );\n assert_eq!(\n assert_panic(&|| f(Ordering::AcqRel)),\n \"there is no such thing as an acquire-release load\"\n );\n }\n}\npub(crate) const STORE_ORDERINGS: [Ordering; 3] =\n [Ordering::Relaxed, Ordering::Release, Ordering::SeqCst];\npub(crate) fn rand_store_ordering(rng: &mut fastrand::Rng) -> Ordering {\n STORE_ORDERINGS[rng.usize(0..STORE_ORDERINGS.len())]\n}\npub(crate) fn test_store_ordering(f: &dyn Fn(Ordering)) {\n for &order in &STORE_ORDERINGS {\n f(order);\n }\n if !skip_should_panic_test() {\n assert_eq!(\n assert_panic(&|| f(Ordering::Acquire)),\n \"there is no such thing as an acquire store\"\n );\n assert_eq!(\n assert_panic(&|| f(Ordering::AcqRel)),\n \"there is no such thing as an acquire-release store\"\n );\n }\n}\npub(crate) const SWAP_ORDERINGS: [Ordering; 5] =\n [Ordering::Relaxed, Ordering::Release, Ordering::Acquire, Ordering::AcqRel, Ordering::SeqCst];\npub(crate) fn rand_swap_ordering(rng: &mut fastrand::Rng) -> Ordering {\n SWAP_ORDERINGS[rng.usize(0..SWAP_ORDERINGS.len())]\n}\npub(crate) fn test_swap_ordering(f: &dyn Fn(Ordering)) {\n for &order in &SWAP_ORDERINGS {\n f(order);\n }\n}\npub(crate) const COMPARE_EXCHANGE_ORDERINGS: [(Ordering, Ordering); 15] = [\n (Ordering::Relaxed, Ordering::Relaxed),\n (Ordering::Relaxed, Ordering::Acquire),\n (Ordering::Relaxed, Ordering::SeqCst),\n (Ordering::Acquire, Ordering::Relaxed),\n (Ordering::Acquire, Ordering::Acquire),\n (Ordering::Acquire, Ordering::SeqCst),\n (Ordering::Release, Ordering::Relaxed),\n (Ordering::Release, Ordering::Acquire),\n (Ordering::Release, Ordering::SeqCst),\n (Ordering::AcqRel, Ordering::Relaxed),\n (Ordering::AcqRel, Ordering::Acquire),\n (Ordering::AcqRel, Ordering::SeqCst),\n (Ordering::SeqCst, Ordering::Relaxed),\n (Ordering::SeqCst, Ordering::Acquire),\n (Ordering::SeqCst, Ordering::SeqCst),\n];\npub(crate) fn rand_compare_exchange_ordering(rng: &mut fastrand::Rng) -> (Ordering, Ordering) {\n COMPARE_EXCHANGE_ORDERINGS[rng.usize(0..COMPARE_EXCHANGE_ORDERINGS.len())]\n}\npub(crate) fn test_compare_exchange_ordering(f: &dyn Fn(Ordering, Ordering)) {\n for &(success, failure) in &COMPARE_EXCHANGE_ORDERINGS {\n f(success, failure);\n }\n if !skip_should_panic_test() {\n for &order in &SWAP_ORDERINGS {\n let msg = assert_panic(&|| f(order, Ordering::AcqRel));\n assert!(\n msg == \"there is no such thing as an acquire-release failure ordering\"\n || msg == \"there is no such thing as an acquire-release load\",\n \"{}\",\n msg\n );\n let msg = assert_panic(&|| f(order, Ordering::Release));\n assert!(\n msg == \"there is no such thing as a release failure ordering\"\n || msg == \"there is no such thing as a release load\",\n \"{}\",\n msg\n );\n }\n }\n}\n\n// Used in stress tests to find atomicity bug generated by __test_atomic_* macro.\npub(crate) fn stress_test_config(rng: &mut fastrand::Rng) -> (usize, usize) {\n let iterations = if cfg!(miri) {\n 50\n } else if cfg!(any(debug_assertions, qemu, valgrind)) {\n 5_000\n } else {\n 25_000\n };\n let threads = if cfg!(any(debug_assertions, valgrind)) { 2 } else { rng.usize(2..=8) };\n eprintln!(\"threads={}\", threads);\n (iterations, threads)\n}\n\n// true if the current implementation implements sub-word atomics using word-size CAS.\n#[cfg(valgrind)]\npub(crate) const IMP_EMU_SUB_WORD_CAS: bool = cfg!(target_arch = \"s390x\");\n\n#[cfg(valgrind)]\n#[inline(always)]\npub(crate) fn mark_aligned_defined(a: &T) {\n test_helper::valgrind::make_aligned_mem_defined::(a);\n}\n\nfn skip_should_panic_test() -> bool {\n // Miri's panic handling is slow\n // MSAN false positive: https://gist.github.com/taiki-e/dd6269a8ffec46284fdc764a4849f884\n // Valgrind false positive on s390x\n is_panic_abort()\n || cfg!(miri)\n || option_env!(\"CARGO_PROFILE_RELEASE_LTO\").map_or(false, |v| v == \"fat\")\n && build_context::SANITIZE.contains(\"memory\")\n || cfg!(all(valgrind, target_arch = \"s390x\"))\n}\n\n// For -C panic=abort -Z panic_abort_tests: https://github.com/rust-lang/rust/issues/67650\nfn is_panic_abort() -> bool {\n !matches!(build_context::PANIC, \"unwind\" | \"\") // cfg(panic) requires Rust 1.60\n}\n\n#[repr(C, align(16))]\npub(crate) struct Align16(pub(crate) T);\n\n// Stress tests to find memory ordering bug.\n// Note that this is not exhaustive and only tests a few cases.\nmacro_rules! __stress_acquire_release {\n (should_pass,\n $int_type:ident, $read:ident, $write:ident, $load_order:ident, $store_order:ident\n ) => {\n paste::paste! {\n #[test]\n #[cfg_attr(all(any(debug_assertions, valgrind_cross), not(miri)), ignore = \"slow in some environments\")] // debug mode is slow.\n #[allow(\n clippy::cast_possible_wrap,\n clippy::cast_sign_loss,\n clippy::cast_possible_truncation,\n )]\n fn []() {\n __call_stress_test_fn!(acquire_release1, [],\n $int_type, $read, $write, $load_order, $store_order);\n }\n }\n };\n (can_panic,\n $int_type:ident, $read:ident, $write:ident, $load_order:ident, $store_order:ident\n ) => {\n paste::paste! {\n // Currently, to make this test work well enough outside of Miri, tens of thousands\n // of iterations are needed, but this test is slow in some environments.\n // So, ignore on non-Miri environments by default. See also catch_unwind_on_weak_memory_arch.\n #[test]\n #[cfg_attr(not(miri), ignore = \"slow in some environments\")]\n #[allow(\n clippy::cast_possible_wrap,\n clippy::cast_sign_loss,\n clippy::cast_possible_truncation,\n )]\n fn []() {\n use crate::tests::helper::catch_unwind_on_weak_memory_arch as can_panic;\n can_panic(\"a=\", test_helper::function_name!(), || {\n __call_stress_test_fn!(acquire_release1, [],\n $int_type, $read, $write, $load_order, $store_order);\n });\n }\n }\n };\n}\nmacro_rules! __stress_seqcst {\n (should_pass,\n $int_type:ident, $read:ident, $write:ident, $load_order:ident, $store_order:ident\n ) => {\n paste::paste! {\n // Currently, to make this test work well enough outside of Miri, tens of thousands\n // of iterations are needed, but this test is slow in some environments.\n // So, ignore on QEMU by default.\n #[test]\n #[cfg_attr(any(all(any(debug_assertions, valgrind_cross), not(miri)), qemu), ignore = \"slow in some environments\")] // debug mode is slow.\n #[allow(\n clippy::cast_possible_wrap,\n clippy::cast_sign_loss,\n clippy::cast_possible_truncation,\n )]\n fn []() {\n __call_stress_test_fn!(seqcst1, [],\n $int_type, $read, $write, $load_order, $store_order);\n }\n }\n };\n (can_panic,\n $int_type:ident, $read:ident, $write:ident, $load_order:ident, $store_order:ident\n ) => {\n paste::paste! {\n // Currently, to make this test work well enough outside of Miri, tens of thousands\n // of iterations are needed, but this test is slow in some environments.\n // So, ignore on non-Miri environments by default. See also catch_unwind_on_non_seqcst_arch.\n #[test]\n #[cfg_attr(not(miri), ignore = \"slow in some environments\")]\n #[allow(\n clippy::cast_possible_wrap,\n clippy::cast_sign_loss,\n clippy::cast_possible_truncation,\n )]\n fn []() {\n use crate::tests::helper::catch_unwind_on_non_seqcst_arch as can_panic;\n can_panic(\"c=2\", test_helper::function_name!(), || {\n __call_stress_test_fn!(seqcst1, [],\n $int_type, $read, $write, $load_order, $store_order);\n });\n }\n }\n };\n}\n// To test this working correctly:\n// - Uncomment can_panic case in __stress_seqcst,\n// and run test with `--release -- acquire_release1 --ignored`.\n// At least on AArch64 machine, some \"panicked as expected\" will be shown.\npub(crate) fn acquire_release1(\n min: isize,\n max: isize,\n new: impl Fn(isize) -> T,\n load: impl Fn(&T) -> isize + Send + Sync,\n store: impl Fn(&T, isize) + Send + Sync,\n) {\n let n: usize = if cfg!(miri) {\n 10\n } else if cfg!(any(qemu, valgrind)) {\n 5_000\n } else {\n 50_000\n };\n\n let mut start: isize = 0;\n #[allow(clippy::cast_possible_wrap)]\n let mut end: isize = n as isize;\n // This test is relatively fast because it spawns only one thread, but\n // the iterations are limited to a maximum value of integers.\n if max < end {\n start = min;\n end = max;\n }\n assert!((start..end).len() <= n, \"{}\", (start..end).len());\n let a = &new(start);\n let b = &AtomicIsize::new(start);\n let a_ready = &AtomicUsize::new(0);\n let b_ready = &AtomicUsize::new(0);\n thread::scope(|s| {\n s.spawn(|_| {\n while a_ready.load(Ordering::Relaxed) == 0 {}\n b_ready.store(1, Ordering::Relaxed);\n for i in start..end {\n b.store(i, Ordering::Relaxed);\n store(a, i);\n }\n });\n let end = end - 1;\n a_ready.store(1, Ordering::Relaxed);\n while b_ready.load(Ordering::Relaxed) == 0 {}\n loop {\n let a = load(a);\n let b = b.load(Ordering::Relaxed);\n assert!(a <= b, \"a={},b={}\", a, b);\n if a == end && b == end {\n break;\n }\n }\n })\n .unwrap();\n}\n// To test this working correctly:\n// - Uncomment can_panic case in __stress_seqcst,\n// and run test with `--release -- seqcst1 --ignored`.\n// At least on AArch64 machine, some \"panicked as expected\" will be shown.\n// - Change aarch64.rs's 128-bit seqcst load from `ldar; ldp; dmb ishld` to `ldp; dmb ish`\n// and run test with `--release -- seqcst1` on AArch64 macOS or other AArch64 which supports FEAT_LSE2\n// with `RUSTFLAGS=-C target-feature=+lse2`.\n// tests::stress_swap_u128::seqcst1_load_seqcst_swap_seqcst should fail within a few repetitions.\npub(crate) fn seqcst1(\n _min: isize,\n #[allow(unused_variables)] max: isize,\n new: impl Fn(isize) -> T,\n load: impl Fn(&T) -> isize + Send + Sync,\n store: impl Fn(&T, isize) + Send + Sync,\n) {\n let n: usize = if cfg!(miri) {\n 8\n } else if cfg!(valgrind)\n || build_context::SANITIZE.contains(\"address\")\n || build_context::SANITIZE.contains(\"memory\")\n {\n 50\n } else if cfg!(qemu)\n || option_env!(\"GITHUB_ACTIONS\").is_some() && cfg!(not(target_os = \"linux\"))\n {\n // GitHub Actions' macOS and Windows runners are slow.\n 5_000\n } else {\n 50_000\n };\n\n let a = &new(0);\n let b = &new(0);\n let c = &AtomicUsize::new(0);\n let a_done = &AtomicUsize::new(0);\n let a_ready = &AtomicUsize::new(0);\n let b_ready = &AtomicUsize::new(0);\n let panicked = &AtomicUsize::new(0);\n thread::scope(|s| {\n s.spawn(|_| {\n for n in 1..n {\n // Acquire to make initialization of a/b/c visible.\n while a_ready.load(Ordering::Acquire) != n {\n if panicked.load(Ordering::Relaxed) != 0 {\n return;\n }\n }\n b_ready.store(n, Ordering::Relaxed);\n store(a, 1);\n if load(b) == 0 {\n c.fetch_add(1, Ordering::Relaxed);\n }\n // Release to make update of c visible.\n a_done.store(n, Ordering::Release);\n }\n });\n for n in 1..n {\n store(a, 0);\n store(b, 0);\n c.store(0, Ordering::Relaxed);\n // Release to make initialization of a/b/c visible.\n a_ready.store(n, Ordering::Release);\n while b_ready.load(Ordering::Relaxed) != n {}\n store(b, 1);\n if load(a) == 0 {\n c.fetch_add(1, Ordering::Relaxed);\n }\n // Acquire to make update of c visible.\n while a_done.load(Ordering::Acquire) != n {}\n let c = c.load(Ordering::Relaxed);\n let ok = matches!(c, 0 | 1);\n if !ok {\n panicked.store(1, Ordering::Relaxed);\n panic!(\"c={},n={}\", c, n);\n }\n }\n })\n .unwrap();\n}\n// Catches unwinding panic on architectures with weak memory models.\n#[allow(dead_code)]\npub(crate) fn catch_unwind_on_weak_memory_arch(pat: &str, function_name: &str, f: impl Fn()) {\n // With x86 TSO, RISC-V TSO (optional), SPARC TSO (optional, default?),\n // and IBM-370 memory models should never be a panic here.\n // Miri emulates weak memory models regardless of target architectures.\n if cfg!(all(\n any(\n target_arch = \"x86\",\n target_arch = \"x86_64\",\n target_arch = \"s390x\",\n target_arch = \"sparc\",\n target_arch = \"sparc64\",\n all(any(target_arch = \"riscv32\", target_arch = \"riscv64\"), target_feature = \"ztso\"),\n ),\n not(any(miri)),\n )) {\n f();\n } else if !is_panic_abort() {\n // This could be Err on architectures with weak memory models.\n // However, this does not necessarily mean that it will always be panic,\n // and implementing it with stronger orderings is also okay.\n match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {\n Ok(()) => {\n // panic!();\n }\n Err(msg) => {\n let msg = msg\n .downcast_ref::()\n .cloned()\n .unwrap_or_else(|| msg.downcast_ref::<&'static str>().copied().unwrap().into());\n assert!(msg.contains(pat), \"{}\", msg);\n test_helper::eprintln_nocapture!(\" {} panicked as expected\", function_name);\n }\n }\n }\n}\n// Catches unwinding panic on architectures with non-sequentially consistent memory models.\n#[allow(dead_code)]\npub(crate) fn catch_unwind_on_non_seqcst_arch(pat: &str, function_name: &str, f: impl Fn()) {\n if !is_panic_abort() {\n // This could be Err on architectures with non-sequentially consistent memory models.\n // However, this does not necessarily mean that it will always be panic,\n // and implementing it with stronger orderings is also okay.\n match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {\n Ok(()) => {\n // panic!();\n }\n Err(msg) => {\n let msg = msg\n .downcast_ref::()\n .cloned()\n .unwrap_or_else(|| msg.downcast_ref::<&'static str>().copied().unwrap().into());\n assert!(msg.contains(pat), \"{}\", msg);\n test_helper::eprintln_nocapture!(\" {} panicked as expected\", function_name);\n }\n }\n }\n}\nmacro_rules! __call_stress_test_fn {\n ($name:ident, $atomic_type:ident,\n $int_type:ident, load, cas, $load_order:ident, $store_order:ident\n ) => {\n $name(\n $int_type::MIN.try_into().unwrap_or(isize::MIN),\n $int_type::MAX.try_into().unwrap_or(isize::MAX),\n |v| <$atomic_type>::new(v as $int_type),\n |a| a.load(Ordering::$load_order) as isize,\n |a, v| {\n a.fetch_update(Ordering::$store_order, Ordering::Relaxed, |_| Some(v as $int_type))\n .unwrap();\n },\n )\n };\n ($name:ident, $atomic_type:ident,\n $int_type:ident, load, $write:ident, $load_order:ident, $store_order:ident\n ) => {\n $name(\n $int_type::MIN.try_into().unwrap_or(isize::MIN),\n $int_type::MAX.try_into().unwrap_or(isize::MAX),\n |v| <$atomic_type>::new(v as $int_type),\n |a| a.load(Ordering::$load_order) as isize,\n |a, v| {\n a.$write(v as $int_type, Ordering::$store_order);\n },\n )\n };\n ($name:ident, $atomic_type:ident,\n $int_type:ident, cas, $write:ident, $load_order:ident, $store_order:ident\n ) => {\n $name(\n $int_type::MIN.try_into().unwrap_or(isize::MIN),\n $int_type::MAX.try_into().unwrap_or(isize::MAX),\n |v| <$atomic_type>::new(v as $int_type),\n |a| {\n a.compare_exchange(0, 0, Ordering::$load_order, match Ordering::$load_order {\n Ordering::Release => Ordering::Relaxed,\n Ordering::AcqRel => Ordering::Acquire,\n _ => Ordering::$load_order,\n })\n .unwrap_or_else(|x| x) as isize\n },\n |a, v| {\n a.$write(v as $int_type, Ordering::$store_order);\n },\n )\n };\n ($name:ident, $atomic_type:ident,\n $int_type:ident, cas_weak, $write:ident, $load_order:ident, $store_order:ident\n ) => {\n $name(\n $int_type::MIN.try_into().unwrap_or(isize::MIN),\n $int_type::MAX.try_into().unwrap_or(isize::MAX),\n |v| <$atomic_type>::new(v as $int_type),\n |a| {\n a.compare_exchange_weak(0, 0, Ordering::$load_order, match Ordering::$load_order {\n Ordering::Release => Ordering::Relaxed,\n Ordering::AcqRel => Ordering::Acquire,\n _ => Ordering::$load_order,\n })\n .unwrap_or_else(|x| x) as isize\n },\n |a, v| {\n a.$write(v as $int_type, Ordering::$store_order);\n },\n )\n };\n}\nmacro_rules! stress_test_load_store {\n ($int_type:ident) => {\n paste::paste! {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod [] {\n use std::{convert::TryInto as _, sync::atomic::Ordering};\n\n use crate::tests::helper::*;\n use super::*;\n\n __stress_acquire_release!(can_panic, $int_type, load, store, Relaxed, Relaxed);\n __stress_acquire_release!(can_panic, $int_type, load, store, Relaxed, Release);\n __stress_acquire_release!(can_panic, $int_type, load, store, Relaxed, SeqCst);\n __stress_acquire_release!(can_panic, $int_type, load, store, Acquire, Relaxed);\n __stress_acquire_release!(should_pass, $int_type, load, store, Acquire, Release);\n __stress_acquire_release!(should_pass, $int_type, load, store, Acquire, SeqCst);\n __stress_acquire_release!(can_panic, $int_type, load, store, SeqCst, Relaxed);\n __stress_acquire_release!(should_pass, $int_type, load, store, SeqCst, Release);\n __stress_acquire_release!(should_pass, $int_type, load, store, SeqCst, SeqCst);\n\n __stress_seqcst!(can_panic, $int_type, load, store, Relaxed, Relaxed);\n __stress_seqcst!(can_panic, $int_type, load, store, Relaxed, Release);\n __stress_seqcst!(can_panic, $int_type, load, store, Relaxed, SeqCst);\n __stress_seqcst!(can_panic, $int_type, load, store, Acquire, Relaxed);\n __stress_seqcst!(can_panic, $int_type, load, store, Acquire, Release);\n __stress_seqcst!(can_panic, $int_type, load, store, Acquire, SeqCst);\n __stress_seqcst!(can_panic, $int_type, load, store, SeqCst, Relaxed);\n __stress_seqcst!(can_panic, $int_type, load, store, SeqCst, Release);\n __stress_seqcst!(should_pass, $int_type, load, store, SeqCst, SeqCst);\n }\n }\n };\n}\nmacro_rules! stress_test {\n ($int_type:ident) => {\n stress_test_load_store!($int_type);\n paste::paste! {\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks\n )]\n mod [] {\n use std::{convert::TryInto as _, sync::atomic::Ordering};\n\n use crate::tests::helper::*;\n use super::*;\n\n __stress_acquire_release!(can_panic, $int_type, load, swap, Relaxed, Relaxed);\n __stress_acquire_release!(can_panic, $int_type, load, swap, Relaxed, Acquire);\n __stress_acquire_release!(can_panic, $int_type, load, swap, Relaxed, Release);\n __stress_acquire_release!(can_panic, $int_type, load, swap, Relaxed, AcqRel);\n __stress_acquire_release!(can_panic, $int_type, load, swap, Relaxed, SeqCst);\n __stress_acquire_release!(can_panic, $int_type, load, swap, Acquire, Relaxed);\n __stress_acquire_release!(can_panic, $int_type, load, swap, Acquire, Acquire);\n __stress_acquire_release!(should_pass, $int_type, load, swap, Acquire, Release);\n __stress_acquire_release!(should_pass, $int_type, load, swap, Acquire, AcqRel);\n __stress_acquire_release!(should_pass, $int_type, load, swap, Acquire, SeqCst);\n __stress_acquire_release!(can_panic, $int_type, load, swap, SeqCst, Relaxed);\n __stress_acquire_release!(can_panic, $int_type, load, swap, SeqCst, Acquire);\n __stress_acquire_release!(should_pass, $int_type, load, swap, SeqCst, Release);\n __stress_acquire_release!(should_pass, $int_type, load, swap, SeqCst, AcqRel);\n __stress_acquire_release!(should_pass, $int_type, load, swap, SeqCst, SeqCst);\n\n __stress_seqcst!(can_panic, $int_type, load, swap, Relaxed, Relaxed);\n __stress_seqcst!(can_panic, $int_type, load, swap, Relaxed, Acquire);\n __stress_seqcst!(can_panic, $int_type, load, swap, Relaxed, Release);\n __stress_seqcst!(can_panic, $int_type, load, swap, Relaxed, AcqRel);\n __stress_seqcst!(can_panic, $int_type, load, swap, Relaxed, SeqCst);\n __stress_seqcst!(can_panic, $int_type, load, swap, Acquire, Relaxed);\n __stress_seqcst!(can_panic, $int_type, load, swap, Acquire, Acquire);\n __stress_seqcst!(can_panic, $int_type, load, swap, Acquire, Release);\n __stress_seqcst!(can_panic, $int_type, load, swap, Acquire, AcqRel);\n __stress_seqcst!(can_panic, $int_type, load, swap, Acquire, SeqCst);\n __stress_seqcst!(can_panic, $int_type, load, swap, SeqCst, Relaxed);\n __stress_seqcst!(can_panic, $int_type, load, swap, SeqCst, Acquire);\n __stress_seqcst!(can_panic, $int_type, load, swap, SeqCst, Release);\n __stress_seqcst!(can_panic, $int_type, load, swap, SeqCst, AcqRel);\n __stress_seqcst!(should_pass, $int_type, load, swap, SeqCst, SeqCst);\n }\n }\n // TODO: cas\n };\n}\n\n#[cfg(feature = \"float\")]\npub(crate) mod float_rand {\n #[cfg(portable_atomic_unstable_f16)]\n pub(crate) fn f16(rng: &mut fastrand::Rng) -> f16 {\n f16::from_bits(rng.u16(..))\n }\n pub(crate) fn f32(rng: &mut fastrand::Rng) -> f32 {\n f32::from_bits(rng.u32(..))\n }\n #[cfg(not(all(not(debug_assertions), target_arch = \"x86\", not(target_feature = \"sse2\"))))]\n pub(crate) fn f64(rng: &mut fastrand::Rng) -> f64 {\n f64::from_bits(rng.u64(..))\n }\n #[cfg(portable_atomic_unstable_f128)]\n pub(crate) fn f128(rng: &mut fastrand::Rng) -> f128 {\n f128::from_bits(rng.u128(..))\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/tests/mod.rs", "sha256": "74aff58531edbba0c833d6a3ec7a45b987896945074763ab3a88f6d99feff05e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks,\n clippy::wildcard_imports\n)]\n\n#[macro_use]\npub(crate) mod helper;\n\n#[allow(dead_code)]\n#[path = \"../../version.rs\"]\nmod version;\n\nuse test_helper as _; // For critical-section test\n\nuse super::*;\n\ntest_atomic_bool_pub!();\ntest_atomic_ptr_pub!();\n\ntest_atomic_int_pub!(isize);\ntest_atomic_int_pub!(usize);\ntest_atomic_int_pub!(i8);\ntest_atomic_int_pub!(u8);\ntest_atomic_int_pub!(i16);\ntest_atomic_int_pub!(u16);\ntest_atomic_int_pub!(i32);\ntest_atomic_int_pub!(u32);\ntest_atomic_int_pub!(i64);\ntest_atomic_int_pub!(u64);\n#[cfg(not(all(valgrind, target_arch = \"powerpc64\")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)\ntest_atomic_int_pub!(i128);\n#[cfg(not(all(valgrind, target_arch = \"powerpc64\")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)\ntest_atomic_int_pub!(u128);\n\n#[cfg(all(feature = \"float\", portable_atomic_unstable_f16))]\ntest_atomic_float_pub!(f16);\n#[cfg(feature = \"float\")]\ntest_atomic_float_pub!(f32);\n#[cfg(feature = \"float\")]\n// TODO: rustc bug: https://github.com/rust-lang/rust/issues/114479\n#[cfg(not(all(not(debug_assertions), target_arch = \"x86\", not(target_feature = \"sse2\"))))]\ntest_atomic_float_pub!(f64);\n#[cfg(all(feature = \"float\", portable_atomic_unstable_f128))]\ntest_atomic_float_pub!(f128);\n\n#[deny(improper_ctypes)]\nextern \"C\" {\n fn _atomic_bool_ffi_safety(_: AtomicBool);\n fn _atomic_ptr_ffi_safety(_: AtomicPtr);\n fn _atomic_isize_ffi_safety(_: AtomicIsize);\n fn _atomic_usize_ffi_safety(_: AtomicUsize);\n fn _atomic_i8_ffi_safety(_: AtomicI8);\n fn _atomic_u8_ffi_safety(_: AtomicU8);\n fn _atomic_i16_ffi_safety(_: AtomicI16);\n fn _atomic_u16_ffi_safety(_: AtomicU16);\n fn _atomic_i32_ffi_safety(_: AtomicI32);\n fn _atomic_u32_ffi_safety(_: AtomicU32);\n fn _atomic_i64_ffi_safety(_: AtomicI64);\n fn _atomic_u64_ffi_safety(_: AtomicU64);\n #[rustversion::since(1.89)] // https://github.com/rust-lang/rust/pull/137306\n fn _atomic_i128_ffi_safety(_: AtomicI128);\n #[rustversion::since(1.89)] // https://github.com/rust-lang/rust/pull/137306\n fn _atomic_u128_ffi_safety(_: AtomicU128);\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f16))]\n fn _atomic_f16_ffi_safety(_: AtomicF16);\n #[cfg(feature = \"float\")]\n fn _atomic_f32_ffi_safety(_: AtomicF32);\n #[cfg(feature = \"float\")]\n fn _atomic_f64_ffi_safety(_: AtomicF64);\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f128))]\n fn _atomic_f128_ffi_safety(_: AtomicF128);\n}\n\n#[test]\nfn test_is_lock_free() {\n assert!(AtomicI8::is_always_lock_free());\n assert!(AtomicI8::is_lock_free());\n assert!(AtomicU8::is_always_lock_free());\n assert!(AtomicU8::is_lock_free());\n assert!(AtomicI16::is_always_lock_free());\n assert!(AtomicI16::is_lock_free());\n assert!(AtomicU16::is_always_lock_free());\n assert!(AtomicU16::is_lock_free());\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f16))]\n assert!(AtomicF16::is_always_lock_free());\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f16))]\n assert!(AtomicF16::is_lock_free());\n assert!(AtomicI32::is_always_lock_free());\n assert!(AtomicI32::is_lock_free());\n assert!(AtomicU32::is_always_lock_free());\n assert!(AtomicU32::is_lock_free());\n #[cfg(feature = \"float\")]\n assert!(AtomicF32::is_always_lock_free());\n #[cfg(feature = \"float\")]\n assert!(AtomicF32::is_lock_free());\n #[cfg(not(portable_atomic_no_cfg_target_has_atomic))]\n {\n if cfg!(any(\n target_has_atomic = \"64\",\n all(\n target_arch = \"riscv32\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"),\n ),\n )) {\n assert!(AtomicI64::is_always_lock_free());\n assert!(AtomicI64::is_lock_free());\n assert!(AtomicU64::is_always_lock_free());\n assert!(AtomicU64::is_lock_free());\n #[cfg(feature = \"float\")]\n assert!(AtomicF64::is_always_lock_free());\n #[cfg(feature = \"float\")]\n assert!(AtomicF64::is_lock_free());\n } else if cfg!(all(\n feature = \"fallback\",\n target_arch = \"arm\",\n not(any(miri, portable_atomic_sanitize_thread)),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n any(target_os = \"linux\", target_os = \"android\"),\n not(any(target_feature = \"v6\", portable_atomic_target_feature = \"v6\")),\n not(portable_atomic_no_outline_atomics),\n not(target_has_atomic = \"64\"),\n not(portable_atomic_test_detect_false),\n )) {\n assert!(!AtomicI64::is_always_lock_free());\n assert!(!AtomicU64::is_always_lock_free());\n #[cfg(feature = \"float\")]\n assert!(!AtomicF64::is_always_lock_free());\n assert!(AtomicI64::is_lock_free());\n assert!(AtomicU64::is_lock_free());\n #[cfg(feature = \"float\")]\n assert!(AtomicF64::is_lock_free());\n } else {\n assert!(!AtomicI64::is_always_lock_free());\n assert!(!AtomicU64::is_always_lock_free());\n #[cfg(feature = \"float\")]\n assert!(!AtomicF64::is_always_lock_free());\n #[cfg(not(target_arch = \"riscv32\"))]\n {\n assert!(!AtomicI64::is_lock_free());\n assert!(!AtomicU64::is_lock_free());\n #[cfg(feature = \"float\")]\n assert!(!AtomicF64::is_lock_free());\n }\n #[cfg(target_arch = \"riscv32\")]\n {\n // TODO(riscv): check detect.has_zacas\n }\n }\n }\n if cfg!(portable_atomic_no_asm) && cfg!(not(portable_atomic_unstable_asm)) {\n assert!(!AtomicI128::is_always_lock_free());\n assert!(!AtomicI128::is_lock_free());\n assert!(!AtomicU128::is_always_lock_free());\n assert!(!AtomicU128::is_lock_free());\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f128))]\n assert!(!AtomicF128::is_always_lock_free());\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f128))]\n assert!(!AtomicF128::is_lock_free());\n } else if cfg!(any(\n all(\n target_arch = \"aarch64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),\n ),\n all(\n target_arch = \"arm64ec\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n ),\n all(\n target_arch = \"x86_64\",\n any(target_feature = \"cmpxchg16b\", portable_atomic_target_feature = \"cmpxchg16b\"),\n ),\n all(\n target_arch = \"riscv64\",\n any(target_feature = \"zacas\", portable_atomic_target_feature = \"zacas\"),\n ),\n all(\n target_arch = \"powerpc64\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n any(\n target_feature = \"quadword-atomics\",\n portable_atomic_target_feature = \"quadword-atomics\",\n ),\n ),\n all(\n target_arch = \"s390x\",\n not(all(\n any(miri, portable_atomic_sanitize_thread),\n not(portable_atomic_atomic_intrinsics),\n )),\n not(portable_atomic_no_asm),\n ),\n )) {\n assert!(AtomicI128::is_always_lock_free());\n assert!(AtomicI128::is_lock_free());\n assert!(AtomicU128::is_always_lock_free());\n assert!(AtomicU128::is_lock_free());\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f128))]\n assert!(AtomicF128::is_always_lock_free());\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f128))]\n assert!(AtomicF128::is_lock_free());\n } else {\n assert!(!AtomicI128::is_always_lock_free());\n assert!(!AtomicU128::is_always_lock_free());\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f128))]\n assert!(!AtomicF128::is_always_lock_free());\n #[cfg(not(any(\n target_arch = \"x86_64\",\n target_arch = \"powerpc64\",\n target_arch = \"riscv64\",\n )))]\n {\n assert!(!AtomicI128::is_lock_free());\n assert!(!AtomicU128::is_lock_free());\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f128))]\n assert!(!AtomicF128::is_lock_free());\n }\n #[cfg(target_arch = \"x86_64\")]\n {\n let has_cmpxchg16b = cfg!(all(\n feature = \"fallback\",\n not(portable_atomic_no_outline_atomics),\n not(any(target_env = \"sgx\", miri)),\n not(portable_atomic_test_detect_false),\n )) && std::is_x86_feature_detected!(\"cmpxchg16b\");\n assert_eq!(AtomicI128::is_lock_free(), has_cmpxchg16b);\n assert_eq!(AtomicU128::is_lock_free(), has_cmpxchg16b);\n #[cfg(all(feature = \"float\", portable_atomic_unstable_f128))]\n assert_eq!(AtomicF128::is_lock_free(), has_cmpxchg16b);\n }\n #[cfg(target_arch = \"powerpc64\")]\n {\n // TODO(powerpc64): is_powerpc_feature_detected is unstable\n }\n #[cfg(target_arch = \"riscv64\")]\n {\n // TODO(riscv): check detect.has_zacas\n }\n }\n}\n\n// test version parsing code used in the build script.\n#[test]\nfn test_rustc_version() {\n use self::version::Version;\n\n // rustc 1.34 (rustup)\n let v = Version::parse(\n \"rustc 1.34.2 (6c2484dc3 2019-05-13)\nbinary: rustc\ncommit-hash: 6c2484dc3c532c052f159264e970278d8b77cdc9\ncommit-date: 2019-05-13\nhost: x86_64-apple-darwin\nrelease: 1.34.2\nLLVM version: 8.0\",\n )\n .unwrap();\n assert_eq!(v, Version::stable(34, 8));\n\n // rustc 1.50 (rustup)\n let v = Version::parse(\n \"rustc 1.50.0 (cb75ad5db 2021-02-10)\nbinary: rustc\ncommit-hash: cb75ad5db02783e8b0222fee363c5f63f7e2cf5b\ncommit-date: 2021-02-10\nhost: aarch64-unknown-linux-gnu\nrelease: 1.50.0\",\n )\n .unwrap();\n assert_eq!(v, Version::stable(50, 0));\n\n // rustc 1.67 (rustup)\n let v = Version::parse(\n \"rustc 1.67.0 (fc594f156 2023-01-24)\nbinary: rustc\ncommit-hash: fc594f15669680fa70d255faec3ca3fb507c3405\ncommit-date: 2023-01-24\nhost: aarch64-apple-darwin\nrelease: 1.67.0\nLLVM version: 15.0.6\",\n )\n .unwrap();\n assert_eq!(v, Version::stable(67, 15));\n\n // rustc 1.68-beta (rustup)\n let v = Version::parse(\n \"rustc 1.68.0-beta.2 (10b73bf73 2023-02-01)\nbinary: rustc\ncommit-hash: 10b73bf73a6b770cd92ad8ff538173bc3298411c\ncommit-date: 2023-02-01\nhost: aarch64-apple-darwin\nrelease: 1.68.0-beta.2\nLLVM version: 15.0.6\",\n )\n .unwrap();\n // We do not distinguish between stable and beta because we are only\n // interested in whether unstable features are potentially available.\n assert_eq!(v, Version::stable(68, 15));\n\n // rustc nightly-2019-01-27 (rustup)\n let v = Version::parse(\n \"rustc 1.33.0-nightly (20c2cba61 2019-01-26)\nbinary: rustc\ncommit-hash: 20c2cba61dc83e612d25ed496025171caa3db30f\ncommit-date: 2019-01-26\nhost: x86_64-apple-darwin\nrelease: 1.33.0-nightly\nLLVM version: 8.0\",\n )\n .unwrap();\n assert_eq!(v.minor, 33);\n assert!(v.nightly);\n assert_eq!(v.llvm, 8);\n assert_eq!(v.commit_date().year, 2019);\n assert_eq!(v.commit_date().month, 1);\n assert_eq!(v.commit_date().day, 26);\n\n // rustc 1.69-nightly (rustup)\n let v = Version::parse(\n \"rustc 1.69.0-nightly (bd39bbb4b 2023-02-07)\nbinary: rustc\ncommit-hash: bd39bbb4bb92df439bf6d85470e296cc6a47ffbd\ncommit-date: 2023-02-07\nhost: aarch64-apple-darwin\nrelease: 1.69.0-nightly\nLLVM version: 15.0.7\",\n )\n .unwrap();\n assert_eq!(v.minor, 69);\n assert!(v.nightly);\n assert_eq!(v.llvm, 15);\n assert_eq!(v.commit_date().year, 2023);\n assert_eq!(v.commit_date().month, 2);\n assert_eq!(v.commit_date().day, 7);\n\n // clippy-driver 1.69-nightly (rustup)\n let v = Version::parse(\n \"rustc 1.69.0-nightly (bd39bbb4b 2023-02-07)\nbinary: rustc\ncommit-hash: bd39bbb4bb92df439bf6d85470e296cc6a47ffbd\ncommit-date: 2023-02-07\nhost: aarch64-apple-darwin\nrelease: 1.69.0-nightly\nLLVM version: 15.0.7\",\n )\n .unwrap();\n assert_eq!(v.minor, 69);\n assert!(v.nightly);\n assert_eq!(v.llvm, 15);\n assert_eq!(v.commit_date().year, 2023);\n assert_eq!(v.commit_date().month, 2);\n assert_eq!(v.commit_date().day, 7);\n\n // rustc 1.69-dev (from source: ./x.py build)\n let v = Version::parse(\n \"rustc 1.69.0-dev\nbinary: rustc\ncommit-hash: unknown\ncommit-date: unknown\nhost: aarch64-unknown-linux-gnu\nrelease: 1.69.0-dev\nLLVM version: 16.0.0\",\n )\n .unwrap();\n assert_eq!(v.minor, 69);\n assert!(v.nightly);\n assert_eq!(v.llvm, 16);\n assert_eq!(v.commit_date().year, 0);\n assert_eq!(v.commit_date().month, 0);\n assert_eq!(v.commit_date().day, 0);\n\n // rustc 1.48 (debian 11: apt-get install cargo)\n let v = Version::parse(\n \"rustc 1.48.0\nbinary: rustc\ncommit-hash: unknown\ncommit-date: unknown\nhost: aarch64-unknown-linux-gnu\nrelease: 1.48.0\nLLVM version: 11.0\",\n )\n .unwrap();\n assert_eq!(v, Version::stable(48, 11));\n\n // rustc 1.67 (fedora: dnf install cargo)\n let v = Version::parse(\n \"rustc 1.67.0 (fc594f156 2023-01-24) (Fedora 1.67.0-2.fc37)\nbinary: rustc\ncommit-hash: fc594f15669680fa70d255faec3ca3fb507c3405\ncommit-date: 2023-01-24\nhost: aarch64-unknown-linux-gnu\nrelease: 1.67.0\nLLVM version: 15.0.7\",\n )\n .unwrap();\n assert_eq!(v, Version::stable(67, 15));\n\n // rustc 1.64 (alpine: apk add cargo)\n let v = Version::parse(\n \"rustc 1.64.0\nbinary: rustc\ncommit-hash: unknown\ncommit-date: unknown\nhost: aarch64-alpine-linux-musl\nrelease: 1.64.0\nLLVM version: 15.0.3\",\n )\n .unwrap();\n assert_eq!(v, Version::stable(64, 15));\n}\n\n#[cfg(feature = \"serde\")]\n#[test]\nfn test_serde() {\n use std::{eprint, eprintln, fmt};\n\n use serde::{\n de::{Deserialize, Deserializer},\n ser::{Serialize, Serializer},\n };\n use serde_test::{Token, assert_tokens};\n\n #[derive(Debug)]\n struct DebugPartialEq(T);\n impl PartialEq for DebugPartialEq {\n fn eq(&self, other: &Self) -> bool {\n std::format!(\"{:?}\", self) == std::format!(\"{:?}\", other)\n }\n }\n impl Serialize for DebugPartialEq {\n fn serialize(&self, serializer: S) -> Result\n where\n S: Serializer,\n {\n self.0.serialize(serializer)\n }\n }\n impl<'de, T: Deserialize<'de>> Deserialize<'de> for DebugPartialEq {\n fn deserialize(deserializer: D) -> Result\n where\n D: Deserializer<'de>,\n {\n T::deserialize(deserializer).map(Self)\n }\n }\n\n macro_rules! t {\n ($atomic_type:ty, $value_type:ident $(as $token_value_type:ident)?, $token_type:ident) => {\n eprint!(\"test_serde {} ... \", stringify!($value_type));\n assert_tokens(&DebugPartialEq(<$atomic_type>::new($value_type::MAX)), &[\n Token::$token_type($value_type::MAX $(as $token_value_type)?),\n ]);\n assert_tokens(&DebugPartialEq(<$atomic_type>::new($value_type::MIN)), &[\n Token::$token_type($value_type::MIN $(as $token_value_type)?),\n ]);\n eprintln!(\"ok\");\n };\n }\n\n assert_tokens(&DebugPartialEq(AtomicBool::new(true)), &[Token::Bool(true)]);\n assert_tokens(&DebugPartialEq(AtomicBool::new(false)), &[Token::Bool(false)]);\n t!(AtomicIsize, isize as i64, I64);\n t!(AtomicUsize, usize as u64, U64);\n t!(AtomicI8, i8, I8);\n t!(AtomicU8, u8, U8);\n t!(AtomicI16, i16, I16);\n t!(AtomicU16, u16, U16);\n t!(AtomicI32, i32, I32);\n t!(AtomicU32, u32, U32);\n t!(AtomicI64, i64, I64);\n t!(AtomicU64, u64, U64);\n #[cfg(not(all(valgrind, target_arch = \"powerpc64\")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)\n t!(AtomicI128, i128, I128);\n #[cfg(not(all(valgrind, target_arch = \"powerpc64\")))] // TODO(powerpc64): Hang (as of Valgrind 3.26)\n t!(AtomicU128, u128, U128);\n // TODO(f16_and_f128): Test f16 & f128 once stabilized.\n #[cfg(feature = \"float\")]\n t!(AtomicF32, f32, F32);\n #[cfg(feature = \"float\")]\n t!(AtomicF64, f64, F64);\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "src/utils.rs", "sha256": "9211b8295c0ec7dc02592731264a0d5516622541126ddd768006d79ef6749dfe", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![cfg_attr(not(all(test, feature = \"float\")), allow(dead_code, unused_macros))]\n\n#[allow(unused_imports)]\npub(crate) use self::generated::{RegISize, RegSize};\n#[macro_use]\n#[path = \"gen/utils.rs\"]\nmod generated;\n\nuse core::sync::atomic::Ordering;\n\nmacro_rules! static_assert {\n ($cond:expr $(,)?) => {{\n let [()] = [(); (true /* type check */ & $cond) as usize];\n }};\n}\n\nmacro_rules! static_assert_layout {\n ($atomic_type:ty, $value_type:ty) => {\n static_assert!(\n core::mem::align_of::<$atomic_type>() == core::mem::size_of::<$atomic_type>()\n );\n static_assert!(core::mem::size_of::<$atomic_type>() == core::mem::size_of::<$value_type>());\n };\n}\n\n// #[doc = concat!(...)] requires Rust 1.54\nmacro_rules! doc_comment {\n ($doc:expr, $($tt:tt)*) => {\n #[doc = $doc]\n $($tt)*\n };\n}\n\n// Adapted from https://github.com/BurntSushi/memchr/blob/2.4.1/src/memchr/x86/mod.rs#L9-L71.\n/// # Safety\n///\n/// - the caller must uphold the safety contract for the function returned by $detect_body.\n/// - the memory pointed by the function pointer returned by $detect_body must be visible from any threads.\n///\n/// The second requirement is always met if the function pointer is to the function definition.\n/// (Currently, all uses of this macro in our code are in this case.)\n#[allow(unused_macros)]\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(\n target_arch = \"aarch64\",\n target_arch = \"arm\",\n target_arch = \"arm64ec\",\n target_arch = \"powerpc64\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n all(target_arch = \"x86_64\", not(any(target_env = \"sgx\", miri))),\n))]\nmacro_rules! ifunc {\n (unsafe fn($($arg_pat:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)? { $($init_body:tt)* }) => {{\n type FnTy = unsafe fn($($arg_ty),*) $(-> $ret_ty)?;\n static FUNC: core::sync::atomic::AtomicPtr<()>\n = core::sync::atomic::AtomicPtr::new(init as *mut ());\n #[cold]\n unsafe fn init($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {\n let func: FnTy = { $($init_body)* };\n FUNC.store(func as *mut (), core::sync::atomic::Ordering::Relaxed);\n // SAFETY: the caller must uphold the safety contract for the function returned by $init_body.\n unsafe { func($($arg_pat),*) }\n }\n // SAFETY: `FnTy` is a function pointer, which is always safe to transmute with a `*mut ()`.\n // (To force the caller to use unsafe block for this macro, do not use\n // unsafe block here.)\n let func = {\n core::mem::transmute::<*mut (), FnTy>(FUNC.load(core::sync::atomic::Ordering::Relaxed))\n };\n // SAFETY: the caller must uphold the safety contract for the function returned by $init_body.\n // (To force the caller to use unsafe block for this macro, do not use\n // unsafe block here.)\n func($($arg_pat),*)\n }};\n}\n\n#[cfg(not(portable_atomic_no_asm))]\n#[allow(unused_macros)]\nmacro_rules! __asm {\n ($($tt:tt)*) => {\n core::arch::asm!($($tt)*)\n };\n}\n#[cfg(portable_atomic_no_asm)]\n#[allow(unused_macros)]\nmacro_rules! __asm {\n ($($tt:tt)*) => {\n asm!($($tt)*)\n };\n}\n\n#[allow(unused_macros)]\n#[cfg(not(portable_atomic_no_outline_atomics))]\n#[cfg(any(\n target_arch = \"aarch64\",\n target_arch = \"arm\",\n target_arch = \"arm64ec\",\n target_arch = \"powerpc64\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n all(target_arch = \"x86_64\", not(any(target_env = \"sgx\", miri))),\n))]\nmacro_rules! fn_alias {\n (\n $(#[$($fn_attr:tt)*])*\n $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;\n $(#[$($alias_attr:tt)*])*\n $new:ident = $from:ident($($last_args:tt)*);\n $($rest:tt)*\n ) => {\n $(#[$($fn_attr)*])*\n $(#[$($alias_attr)*])*\n $vis unsafe fn $new($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { $from($($arg_pat,)* $($last_args)*) }\n }\n fn_alias! {\n $(#[$($fn_attr)*])*\n $vis unsafe fn($($arg_pat: $arg_ty),*) $(-> $ret_ty)?;\n $($rest)*\n }\n };\n (\n $(#[$($attr:tt)*])*\n $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;\n ) => {}\n}\n\n/// Make the given function const if the given condition is true.\nmacro_rules! const_fn {\n (\n const_if: #[cfg($($cfg:tt)+)];\n $(#[$($attr:tt)*])*\n $vis:vis const $($rest:tt)*\n ) => {\n #[cfg($($cfg)+)]\n $(#[$($attr)*])*\n $vis const $($rest)*\n #[cfg(not($($cfg)+))]\n $(#[$($attr)*])*\n $vis $($rest)*\n };\n}\n\n/// Implements `core::fmt::Debug` and `serde::{Serialize, Deserialize}` (when serde\n/// feature is enabled) for atomic bool, integer, or float.\nmacro_rules! impl_debug_and_serde {\n // TODO(f16_and_f128): Implement serde traits for f16 & f128 once stabilized.\n (AtomicF16) => {\n impl_debug!(AtomicF16);\n };\n (AtomicF128) => {\n impl_debug!(AtomicF128);\n };\n ($atomic_type:ident) => {\n impl_debug!($atomic_type);\n #[cfg(feature = \"serde\")]\n #[cfg_attr(docsrs, doc(cfg(feature = \"serde\")))]\n impl serde::ser::Serialize for $atomic_type {\n #[allow(clippy::missing_inline_in_public_items)] // serde doesn't use inline on std atomic's Serialize/Deserialize impl\n fn serialize(&self, serializer: S) -> Result\n where\n S: serde::ser::Serializer,\n {\n // https://github.com/serde-rs/serde/blob/v1.0.152/serde/src/ser/impls.rs#L958-L959\n self.load(Ordering::Relaxed).serialize(serializer)\n }\n }\n #[cfg(feature = \"serde\")]\n #[cfg_attr(docsrs, doc(cfg(feature = \"serde\")))]\n impl<'de> serde::de::Deserialize<'de> for $atomic_type {\n #[allow(clippy::missing_inline_in_public_items)] // serde doesn't use inline on std atomic's Serialize/Deserialize impl\n fn deserialize(deserializer: D) -> Result\n where\n D: serde::de::Deserializer<'de>,\n {\n serde::de::Deserialize::deserialize(deserializer).map(Self::new)\n }\n }\n };\n}\nmacro_rules! impl_debug {\n ($atomic_type:ident) => {\n impl fmt::Debug for $atomic_type {\n #[inline] // fmt is not hot path, but #[inline] on fmt seems to still be useful: https://github.com/rust-lang/rust/pull/117727\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n // std atomic types use Relaxed in Debug::fmt: https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L2188\n fmt::Debug::fmt(&self.load(Ordering::Relaxed), f)\n }\n }\n };\n}\n\n// We do not provide `nand` because it cannot be optimized on neither x86 nor MSP430.\n// https://godbolt.org/z/ahWejchbT\nmacro_rules! impl_default_no_fetch_ops {\n ($atomic_type:ident, bool) => {\n impl $atomic_type {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn and(&self, val: bool, order: Ordering) {\n self.fetch_and(val, order);\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn or(&self, val: bool, order: Ordering) {\n self.fetch_or(val, order);\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn xor(&self, val: bool, order: Ordering) {\n self.fetch_xor(val, order);\n }\n }\n };\n ($atomic_type:ident, $int_type:ty) => {\n impl $atomic_type {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn add(&self, val: $int_type, order: Ordering) {\n self.fetch_add(val, order);\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn sub(&self, val: $int_type, order: Ordering) {\n self.fetch_sub(val, order);\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn and(&self, val: $int_type, order: Ordering) {\n self.fetch_and(val, order);\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn or(&self, val: $int_type, order: Ordering) {\n self.fetch_or(val, order);\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn xor(&self, val: $int_type, order: Ordering) {\n self.fetch_xor(val, order);\n }\n }\n };\n}\nmacro_rules! impl_default_bit_opts {\n (AtomicPtr, $int_type:ty) => {\n impl AtomicPtr {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn bit_set(&self, bit: u32, order: Ordering) -> bool {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n let mask = <$int_type>::wrapping_shl(1, bit);\n self.fetch_or(mask, order).addr() & mask != 0\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn bit_clear(&self, bit: u32, order: Ordering) -> bool {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n let mask = <$int_type>::wrapping_shl(1, bit);\n self.fetch_and(!mask, order).addr() & mask != 0\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn bit_toggle(&self, bit: u32, order: Ordering) -> bool {\n #[cfg(portable_atomic_no_strict_provenance)]\n use crate::utils::ptr::PtrExt as _;\n let mask = <$int_type>::wrapping_shl(1, bit);\n self.fetch_xor(mask, order).addr() & mask != 0\n }\n }\n };\n ($atomic_type:ident, $int_type:ty) => {\n impl $atomic_type {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn bit_set(&self, bit: u32, order: Ordering) -> bool {\n let mask = <$int_type>::wrapping_shl(1, bit);\n self.fetch_or(mask, order) & mask != 0\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn bit_clear(&self, bit: u32, order: Ordering) -> bool {\n let mask = <$int_type>::wrapping_shl(1, bit);\n self.fetch_and(!mask, order) & mask != 0\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn bit_toggle(&self, bit: u32, order: Ordering) -> bool {\n let mask = <$int_type>::wrapping_shl(1, bit);\n self.fetch_xor(mask, order) & mask != 0\n }\n }\n };\n}\n\n// This just outputs the input as is, but can be used like an item-level block by using it with cfg.\n// Note: This macro is items!({ }), not items! { }.\n// An extra brace is used in input to make contents rustfmt-able.\nmacro_rules! items {\n ({$($tt:tt)*}) => {\n $($tt)*\n };\n}\n\n// rustfmt-compatible cfg_select/cfg_if alternative\n// Note: This macro is cfg_sel!({ }), not cfg_sel! { }.\n// An extra brace is used in input to make contents rustfmt-able.\nmacro_rules! cfg_sel {\n ({#[cfg(else)] { $($output:tt)* }}) => {\n $($output)*\n };\n ({\n #[cfg($cfg:meta)]\n { $($output:tt)* }\n $($( $rest:tt )+)?\n }) => {\n #[cfg($cfg)]\n cfg_sel! {{#[cfg(else)] { $($output)* }}}\n $(\n #[cfg(not($cfg))]\n cfg_sel! {{ $($rest)+ }}\n )?\n };\n ({\n #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg($cfg1:meta))]\n #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg($cfg2:meta))]\n { $($output:tt)* }\n $($( $rest:tt )+)?\n }) => {\n #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg($cfg1))]\n #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg($cfg2))]\n cfg_sel! {{#[cfg(else)] { $($output)* }}}\n $(\n #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not($cfg1)))]\n #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not($cfg2)))]\n cfg_sel! {{ $($rest)+ }}\n )?\n };\n}\n\n// Equivalent to core::hint::cold_path, but compatible with pre-1.95 rustc.\n#[allow(dead_code)]\n#[inline(always)]\n#[cold]\nfn cold_path() {}\n// Stable equivalent of core::hint::{likely, unlikely}.\n#[allow(dead_code)]\n#[inline(always)]\npub(crate) fn likely(b: bool) -> bool {\n if b {\n true\n } else {\n cold_path();\n false\n }\n}\n#[allow(dead_code)]\n#[inline(always)]\npub(crate) fn unlikely(b: bool) -> bool {\n if b {\n cold_path();\n true\n } else {\n false\n }\n}\n\n// Equivalent to core::hint::assert_unchecked, but compatible with pre-1.81 rustc.\n#[cfg(any(target_arch = \"x86\", target_arch = \"x86_64\"))]\n#[allow(dead_code)]\n#[inline(always)]\n#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\npub(crate) unsafe fn assert_unchecked(cond: bool) {\n if !cond {\n #[cfg(debug_assertions)]\n unreachable!();\n #[cfg(not(debug_assertions))]\n // SAFETY: the caller promised `cond` is true.\n unsafe {\n core::hint::unreachable_unchecked()\n }\n }\n}\n\n// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3338\n#[inline]\n#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\npub(crate) fn assert_load_ordering(order: Ordering) {\n match order {\n Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}\n Ordering::Release => panic!(\"there is no such thing as a release load\"),\n Ordering::AcqRel => panic!(\"there is no such thing as an acquire-release load\"),\n _ => unreachable!(),\n }\n}\n// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3323\n#[inline]\n#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\npub(crate) fn assert_store_ordering(order: Ordering) {\n match order {\n Ordering::Release | Ordering::Relaxed | Ordering::SeqCst => {}\n Ordering::Acquire => panic!(\"there is no such thing as an acquire store\"),\n Ordering::AcqRel => panic!(\"there is no such thing as an acquire-release store\"),\n _ => unreachable!(),\n }\n}\n// https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/sync/atomic.rs#L3404\n#[inline]\n#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\npub(crate) fn assert_compare_exchange_ordering(success: Ordering, failure: Ordering) {\n match success {\n Ordering::AcqRel\n | Ordering::Acquire\n | Ordering::Relaxed\n | Ordering::Release\n | Ordering::SeqCst => {}\n _ => unreachable!(),\n }\n match failure {\n Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}\n Ordering::Release => panic!(\"there is no such thing as a release failure ordering\"),\n Ordering::AcqRel => panic!(\"there is no such thing as an acquire-release failure ordering\"),\n _ => unreachable!(),\n }\n}\n\n// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0418r2.html\n// https://github.com/rust-lang/rust/pull/98383\n#[allow(dead_code)]\n#[inline]\npub(crate) fn upgrade_success_ordering(success: Ordering, failure: Ordering) -> Ordering {\n match (success, failure) {\n (Ordering::Relaxed, Ordering::Acquire) => Ordering::Acquire,\n (Ordering::Release, Ordering::Acquire) => Ordering::AcqRel,\n (_, Ordering::SeqCst) => Ordering::SeqCst,\n _ => success,\n }\n}\n\n#[cfg(not(portable_atomic_no_asm_maybe_uninit))]\n#[cfg(target_pointer_width = \"32\")]\n// SAFETY: MaybeUninit returned by zero_extend64_ptr is always initialized.\nconst _: () = assert!(unsafe {\n zero_extend64_ptr(ptr::without_provenance_mut(!0)).assume_init() == !0_u32 as u64\n});\n/// Zero-extends the given 32-bit pointer to `MaybeUninit`.\n/// This is used for 64-bit architecture's 32-bit ABI (e.g., AArch64 ILP32 ABI).\n/// See ptr_reg! macro in src/gen/utils.rs for details.\n#[cfg(not(portable_atomic_no_asm_maybe_uninit))]\n#[cfg(target_pointer_width = \"32\")]\n#[allow(dead_code)]\n#[inline]\npub(crate) const fn zero_extend64_ptr(v: *mut ()) -> core::mem::MaybeUninit {\n #[repr(C)]\n struct ZeroExtended {\n #[cfg(target_endian = \"big\")]\n pad: *mut (),\n v: *mut (),\n #[cfg(target_endian = \"little\")]\n pad: *mut (),\n }\n // SAFETY: we can safely transmute any 64-bit value to MaybeUninit.\n unsafe { core::mem::transmute(ZeroExtended { v, pad: core::ptr::null_mut() }) }\n}\n\n#[allow(dead_code)]\n#[cfg(any(\n target_arch = \"aarch64\",\n target_arch = \"arm64ec\",\n target_arch = \"powerpc64\",\n target_arch = \"riscv64\",\n target_arch = \"s390x\",\n target_arch = \"x86_64\",\n))]\n/// A 128-bit value represented as a pair of 64-bit values.\n///\n/// This type is `#[repr(C)]`, both fields have the same in-memory representation\n/// and are plain old data types, so access to the fields is always safe.\n#[derive(Clone, Copy)]\n#[repr(C)]\npub(crate) union U128 {\n pub(crate) whole: u128,\n pub(crate) pair: Pair,\n}\n#[allow(dead_code)]\n#[cfg(any(target_arch = \"arm\", target_arch = \"riscv32\"))]\n/// A 64-bit value represented as a pair of 32-bit values.\n///\n/// This type is `#[repr(C)]`, both fields have the same in-memory representation\n/// and are plain old data types, so access to the fields is always safe.\n#[derive(Clone, Copy)]\n#[repr(C)]\npub(crate) union U64 {\n pub(crate) whole: u64,\n pub(crate) pair: Pair,\n}\n#[allow(dead_code)]\n#[derive(Clone, Copy)]\n#[repr(C)]\npub(crate) struct Pair {\n // little endian order\n #[cfg(any(\n target_endian = \"little\",\n target_arch = \"aarch64\",\n target_arch = \"arm\",\n target_arch = \"arm64ec\",\n ))]\n pub(crate) lo: T,\n pub(crate) hi: T,\n // big endian order\n #[cfg(not(any(\n target_endian = \"little\",\n target_arch = \"aarch64\",\n target_arch = \"arm\",\n target_arch = \"arm64ec\",\n )))]\n pub(crate) lo: T,\n}\n\n#[cfg(any(target_arch = \"riscv32\", target_arch = \"riscv64\"))]\ntype MinWord = u32;\n#[cfg(any(target_arch = \"riscv32\", target_arch = \"riscv64\"))]\ntype RetInt = u32;\n// Adapted from https://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.6/src/utils.rs#L255.\n// Helper for implementing sub-word atomic operations using word-sized LL/SC loop or CAS loop.\n//\n// Refs: https://github.com/llvm/llvm-project/blob/llvmorg-22.1.0-rc1/llvm/lib/CodeGen/AtomicExpandPass.cpp#L811\n// (aligned_ptr, shift, mask)\n#[cfg(any(target_arch = \"riscv32\", target_arch = \"riscv64\"))]\n#[allow(dead_code)]\n#[inline]\npub(crate) fn create_sub_word_mask_values(ptr: *mut T) -> (*mut MinWord, RetInt, RetInt) {\n use core::mem;\n\n #[cfg(portable_atomic_no_strict_provenance)]\n use self::ptr::PtrExt as _;\n\n // RISC-V, MIPS, SPARC, LoongArch, Xtensa, BPF: shift amount of 32-bit shift instructions is 5 bits unsigned (0-31).\n // PowerPC, C-SKY: shift amount of 32-bit shift instructions is 6 bits unsigned (0-63) and shift amount 32-63 means \"clear\".\n // Arm: shift amount of 32-bit shift instructions is 8 bits unsigned (0-255).\n // Hexagon: shift amount of 32-bit shift instructions is 7 bits signed (-64-63) and negative shift amount means \"reverse the direction of the shift\".\n // (On s390x, we don't use the mask returned from this function.)\n // (See also https://devblogs.microsoft.com/oldnewthing/20230904-00/?p=108704 for others)\n const SHIFT_MASK: bool = !cfg!(any(\n target_arch = \"bpf\",\n target_arch = \"loongarch32\",\n target_arch = \"loongarch64\",\n target_arch = \"mips\",\n target_arch = \"mips32r6\",\n target_arch = \"mips64\",\n target_arch = \"mips64r6\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"s390x\",\n target_arch = \"sparc\",\n target_arch = \"sparc64\",\n target_arch = \"xtensa\",\n ));\n let ptr_mask = mem::size_of::() - 1;\n let aligned_ptr = ptr.with_addr(ptr.addr() & !ptr_mask) as *mut MinWord;\n let ptr_lsb = if SHIFT_MASK {\n ptr.addr() & ptr_mask\n } else {\n // We use 32-bit wrapping shift instructions in asm on these platforms.\n ptr.addr()\n };\n let shift = if cfg!(any(target_endian = \"little\", target_arch = \"s390x\")) {\n ptr_lsb.wrapping_mul(8)\n } else {\n (ptr_lsb ^ (mem::size_of::() - mem::size_of::())).wrapping_mul(8)\n };\n let mut mask: RetInt = (1 << (mem::size_of::() * 8)) - 1; // !(0 as T) as RetInt\n if SHIFT_MASK {\n mask <<= shift;\n }\n #[allow(clippy::cast_possible_truncation)]\n {\n (aligned_ptr, shift as RetInt, mask)\n }\n}\n\n// This module provides core::ptr strict_provenance/exposed_provenance polyfill for pre-1.84 rustc.\n#[allow(dead_code)]\npub(crate) mod ptr {\n cfg_sel!({\n #[cfg(not(portable_atomic_no_strict_provenance))]\n {\n #[allow(unused_imports)]\n pub(crate) use core::ptr::{\n with_exposed_provenance, with_exposed_provenance_mut, without_provenance_mut,\n };\n }\n #[cfg(else)]\n {\n #[inline(always)]\n #[must_use]\n pub(crate) const fn without_provenance_mut(addr: usize) -> *mut T {\n // An int-to-pointer transmute currently has exactly the intended semantics: it creates a\n // pointer without provenance. Note that this is *not* a stable guarantee about transmute\n // semantics, it relies on sysroot crates having special status.\n // SAFETY: every valid integer is also a valid pointer (as long as you don't dereference that\n // pointer).\n #[cfg(miri)]\n unsafe {\n core::mem::transmute(addr)\n }\n // const transmute requires Rust 1.56.\n // Using transmute doesn't work with CHERI: https://github.com/kent-weak-memory/rust/blob/0c0ca909de877f889629057e1ddf139527446d75/library/core/src/ptr/mod.rs#L607\n #[cfg(not(miri))]\n {\n addr as *mut T\n }\n }\n #[inline(always)]\n #[must_use]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn with_exposed_provenance(addr: usize) -> *const T {\n addr as *const T\n }\n #[inline(always)]\n #[must_use]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n pub(crate) fn with_exposed_provenance_mut(addr: usize) -> *mut T {\n addr as *mut T\n }\n\n pub(crate) trait PtrExt: Copy {\n #[must_use]\n fn addr(self) -> usize;\n #[must_use]\n fn with_addr(self, addr: usize) -> Self\n where\n T: Sized;\n }\n impl PtrExt for *mut T {\n #[inline(always)]\n #[must_use]\n fn addr(self) -> usize {\n // A pointer-to-integer transmute currently has exactly the right semantics: it returns the\n // address without exposing the provenance. Note that this is *not* a stable guarantee about\n // transmute semantics, it relies on sysroot crates having special status.\n // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the\n // provenance).\n #[cfg(miri)]\n unsafe {\n core::mem::transmute(self as *mut ())\n }\n // Using transmute doesn't work with CHERI: https://github.com/kent-weak-memory/rust/blob/0c0ca909de877f889629057e1ddf139527446d75/library/core/src/ptr/mut_ptr.rs#L210\n #[cfg(not(miri))]\n {\n self as *mut () as usize\n }\n }\n #[inline]\n #[must_use]\n fn with_addr(self, addr: usize) -> Self\n where\n T: Sized,\n {\n // This should probably be an intrinsic to avoid doing any sort of arithmetic, but\n // meanwhile, we can implement it with `wrapping_offset`, which preserves the pointer's\n // provenance.\n let self_addr = self.addr() as isize;\n let dest_addr = addr as isize;\n let offset = dest_addr.wrapping_sub(self_addr);\n (self as *mut u8).wrapping_offset(offset) as *mut T\n }\n }\n }\n });\n}\n\n// This module provides:\n// - core::ffi polyfill (c_* type aliases and CStr) for pre-1.64 rustc compatibility.\n// (core::ffi::* (except c_void) requires Rust 1.64)\n// - Safe abstraction (c! macro) for creating static C strings without runtime checks.\n// (c\"...\" requires Rust 1.77)\n// - Helper macros for defining FFI bindings with static signature/type/value assertions.\n// - Helper macros for defining asm-based syscalls on Linux.\n#[cfg(any(\n test,\n portable_atomic_test_no_std_static_assert_ffi,\n not(any(target_arch = \"x86\", target_arch = \"x86_64\"))\n))]\n#[cfg(any(not(portable_atomic_no_asm), portable_atomic_unstable_asm))]\n#[allow(dead_code, non_camel_case_types, unused_macros)]\n#[macro_use]\npub(crate) mod ffi {\n // -------------------------------------------------------------------------\n // core::ffi polyfill and c\"...\" equivalent\n\n pub(crate) type c_void = core::ffi::c_void;\n // c_{,u}int is {i,u}16 on 16-bit targets, otherwise {i,u}32.\n // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L156\n cfg_sel!({\n #[cfg(target_pointer_width = \"16\")]\n {\n pub(crate) type c_int = i16;\n pub(crate) type c_uint = u16;\n }\n #[cfg(else)]\n {\n pub(crate) type c_int = i32;\n pub(crate) type c_uint = u32;\n }\n });\n // c_{,u}long is {i,u}64 on non-Windows 64-bit targets, otherwise {i,u}32.\n // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L168\n cfg_sel!({\n #[cfg(all(target_pointer_width = \"64\", not(windows)))]\n {\n pub(crate) type c_long = i64;\n pub(crate) type c_ulong = u64;\n }\n #[cfg(else)]\n {\n pub(crate) type c_long = i32;\n pub(crate) type c_ulong = u32;\n }\n });\n // c_size_t is currently always usize.\n // https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/mod.rs#L76\n pub(crate) type c_size_t = usize;\n // c_char is u8 by default on non-Apple/non-Windows/non-Vita Arm/C-SKY/Hexagon/MSP430/PowerPC/RISC-V/s390x/Xtensa targets, otherwise i8 by default.\n // See references in https://github.com/rust-lang/rust/issues/129945 for details.\n cfg_sel!({\n #[cfg(all(\n not(any(target_vendor = \"apple\", windows, target_os = \"vita\")),\n any(\n target_arch = \"aarch64\",\n target_arch = \"arm\",\n target_arch = \"csky\",\n target_arch = \"hexagon\",\n target_arch = \"msp430\",\n target_arch = \"powerpc\",\n target_arch = \"powerpc64\",\n target_arch = \"riscv32\",\n target_arch = \"riscv64\",\n target_arch = \"s390x\",\n target_arch = \"xtensa\",\n ),\n ))]\n {\n pub(crate) type c_char = u8;\n }\n #[cfg(else)]\n {\n pub(crate) type c_char = i8;\n }\n });\n\n // Static assertions for C type definitions.\n #[cfg(test)]\n const _: fn() = || {\n let _: c_int = 0 as std::os::raw::c_int;\n let _: c_uint = 0 as std::os::raw::c_uint;\n let _: c_long = 0 as std::os::raw::c_long;\n let _: c_ulong = 0 as std::os::raw::c_ulong;\n #[cfg(unix)]\n let _: c_size_t = 0 as libc::size_t; // std::os::raw::c_size_t is unstable\n let _: c_char = 0 as std::os::raw::c_char;\n };\n\n #[repr(transparent)]\n pub(crate) struct CStr([c_char]);\n impl CStr {\n #[inline]\n #[must_use]\n pub(crate) const fn as_ptr(&self) -> *const c_char {\n self.0.as_ptr()\n }\n /// # Safety\n ///\n /// The provided slice **must** be nul-terminated and not contain any interior\n /// nul bytes.\n #[inline]\n #[must_use]\n pub(crate) unsafe fn from_bytes_with_nul_unchecked(bytes: &[u8]) -> &CStr {\n // SAFETY: Casting to CStr is safe because *our* CStr is #[repr(transparent)]\n // and its internal representation is a [u8] too. (Note that std's CStr\n // is not #[repr(transparent)].)\n // Dereferencing the obtained pointer is safe because it comes from a\n // reference. Making a reference is then safe because its lifetime\n // is bound by the lifetime of the given `bytes`.\n unsafe { &*(bytes as *const [u8] as *const CStr) }\n }\n #[cfg(test)]\n #[inline]\n #[must_use]\n pub(crate) fn to_bytes_with_nul(&self) -> &[u8] {\n #[allow(clippy::unnecessary_cast)] // triggered for targets that c_char is u8\n // SAFETY: Transmuting a slice of `c_char`s to a slice of `u8`s\n // is safe on all supported targets.\n unsafe {\n &*(&self.0 as *const [c_char] as *const [u8])\n }\n }\n }\n\n macro_rules! c {\n ($s:expr) => {{\n const BYTES: &[u8] = concat!($s, \"\\0\").as_bytes();\n const _: () = static_assert!(crate::utils::ffi::_const_is_c_str(BYTES));\n #[allow(unused_unsafe)]\n // SAFETY: we've checked `BYTES` is a valid C string\n unsafe {\n crate::utils::ffi::CStr::from_bytes_with_nul_unchecked(BYTES)\n }\n }};\n }\n\n #[must_use]\n pub(crate) const fn _const_is_c_str(bytes: &[u8]) -> bool {\n #[cfg(portable_atomic_no_track_caller)]\n {\n // const_if_match/const_loop was stabilized (nightly-2020-06-30) 2 days before\n // track_caller was stabilized (nightly-2020-07-02), so we reuse the cfg for\n // track_caller here instead of emitting a cfg for const_if_match/const_loop.\n // https://github.com/rust-lang/rust/pull/72437\n // track_caller was stabilized 11 days after the oldest nightly version\n // that uses this module, and is included in the same 1.46 stable release.\n // The check here is insufficient in this case, but this is fine because this function\n // is internal code that is not used to process input from the user and our CI checks\n // all builtin targets and some custom targets with some versions of newer compilers.\n !bytes.is_empty()\n }\n #[cfg(not(portable_atomic_no_track_caller))]\n {\n // Based on https://github.com/rust-lang/rust/blob/1.84.0/library/core/src/ffi/c_str.rs#L417\n // - bytes must be nul-terminated.\n // - bytes must not contain any interior nul bytes.\n if bytes.is_empty() {\n return false;\n }\n let mut i = bytes.len() - 1;\n if bytes[i] != 0 {\n return false;\n }\n // Ending null byte exists, skip to the rest.\n while i != 0 {\n i -= 1;\n if bytes[i] == 0 {\n return false;\n }\n }\n true\n }\n }\n\n // -------------------------------------------------------------------------\n // Helper macros for defining FFI bindings with static signature/type/value assertions.\n\n /// Defines types with #[cfg(test)] static assertions which checks\n /// types are the same as the platform's latest header files' ones.\n // Note: This macro is sys_ty!({ }), not sys_ty! { }.\n // An extra brace is used in input to make contents rustfmt-able.\n macro_rules! sys_type {\n ({$(\n $(#[$attr:meta])*\n $vis:vis type $([$($windows_path:ident)::+])? $name:ident = $ty:ty;\n )*}) => {\n $(\n $(#[$attr])*\n $vis type $name = $ty;\n )*\n #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]\n test_helper::static_assert_sys_type!($(\n $(#[$attr])*\n type $([$($windows_path)::+])? $name;\n )*);\n };\n }\n /// Defines #[repr(C)] structs with #[cfg(test)] static assertions which checks\n /// fields are the same as the platform's latest header files' ones.\n // Note: This macro is sys_struct!({ }), not sys_struct! { }.\n // An extra brace is used in input to make contents rustfmt-able.\n macro_rules! sys_struct {\n ({$(\n $(#[$attr:meta])*\n $vis:vis struct $([$($windows_path:ident)::+])? $name:ident {$(\n $(#[$field_attr:meta])*\n $field_vis:vis $field_name:ident: $field_ty:ty,\n )*}\n )*}) => {\n $(\n $(#[$attr])*\n #[derive(Clone, Copy)]\n #[cfg_attr(\n any(test, portable_atomic_test_no_std_static_assert_ffi),\n derive(Debug, PartialEq)\n )]\n #[repr(C)]\n $vis struct $name {$(\n $(#[$field_attr])*\n $field_vis $field_name: $field_ty,\n )*}\n )*\n #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]\n test_helper::static_assert_sys_struct!($(\n $(#[$attr])*\n struct $([$($windows_path)::+])? $name {$(\n $(#[$field_attr])*\n $field_name: $field_ty,\n )*}\n )*);\n };\n }\n /// Defines constants with #[cfg(test)] static assertions which checks\n /// values are the same as the platform's latest header files' ones.\n // Note: This macro is sys_const!({ }), not sys_const! { }.\n // An extra brace is used in input to make contents rustfmt-able.\n macro_rules! sys_const {\n ({$(\n $(#[$attr:meta])*\n $vis:vis const $([$($windows_path:ident)::+])? $name:ident: $ty:ty = $val:expr;\n )*}) => {\n $(\n $(#[$attr])*\n $vis const $name: $ty = $val;\n )*\n #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]\n test_helper::static_assert_sys_const!($(\n $(#[$attr])*\n const $([$($windows_path)::+])? $name: $ty;\n )*);\n };\n }\n /// Defines functions with #[cfg(test)] static assertions which checks\n /// signatures are the same as the platform's latest header files' ones.\n // Note: This macro is sys_fn!({ }), not sys_fn! { }.\n // An extra brace is used in input to make contents rustfmt-able.\n macro_rules! sys_fn {\n ({\n $(#[$extern_attr:meta])*\n extern $abi:tt {$(\n $(#[$fn_attr:meta])*\n $vis:vis fn $([$($windows_path:ident)::+])? $name:ident(\n $($args:tt)*\n ) $(-> $ret_ty:ty)?;\n )*}\n }) => {\n $(#[$extern_attr])*\n extern $abi {$(\n $(#[$fn_attr])*\n $vis fn $name($($args)*) $(-> $ret_ty)?;\n )*}\n #[cfg(any(test, portable_atomic_test_no_std_static_assert_ffi))]\n test_helper::static_assert_sys_fn!(\n $(#[$extern_attr])*\n extern $abi {$(\n $(#[$fn_attr])*\n fn $([$($windows_path)::+])? $name($($args)*) $(-> $ret_ty)?;\n )*}\n );\n };\n }\n\n // -----------------------------------------------------------------------------\n // Helper macros for defining asm-based syscalls on Linux.\n //\n // Use asm-based syscall on Linux for compatibility with non-libc targets if possible.\n //\n // In non-Linux environments (including Android), syscalls should be called via libc,\n // mainly for the following reasons.\n // - asm-based syscalls are not permitted in the first place. e.g.,\n // - OpenBSD https://github.com/golang/go/issues/36435\n // - CheriBSD https://www.cheribsd.org/release-notes/25.03/index.html\n // - Stability of asm-based syscalls is not guaranteed. e.g.,\n // - macOS https://go-review.googlesource.com/c/go/+/25495\n // - Syscalls via libc (or also libsys on FreeBSD) provide additional protections. e.g.,\n // - OpenBSD https://lwn.net/Articles/806863/\n // - CheriBSD https://www.cheribsd.org/release-notes/25.03/index.html\n // - FreeBSD https://www.freebsd.org/status/report-2024-01-2024-03/libsys/\n // - Android https://android.googlesource.com/platform/bionic/+/HEAD/docs/fdtrack.md\n //\n // Miri and Sanitizer do not support inline assembly.\n #[cfg(all(\n target_os = \"linux\",\n not(any(miri, portable_atomic_sanitize_thread)),\n not(portable_atomic_no_asm_syscall),\n ))]\n #[macro_use]\n mod syscall_helper {\n // Note:\n // - The syscall number and arguments must be extended to the register size by the caller of macros.\n // The kernel extends the low bits on some architectures, but it does not on many architectures. e.g.,\n // x86_64 (pre-5.14): https://github.com/torvalds/linux/commit/0595494891723a1dcca5eaa8eeca8ab54ad953b9\n // powerpc64: https://github.com/torvalds/linux/blob/v7.1/arch/powerpc/kernel/syscall.c#L16\n // riscv64: https://github.com/torvalds/linux/blob/v7.1/arch/riscv/kernel/traps.c#L328\n // loongarch64: https://github.com/torvalds/linux/blob/v7.1/arch/loongarch/kernel/syscall.c#L61\n //\n // Refs:\n // - https://man7.org/linux/man-pages/man2/syscall.2.html\n // - aarch64 (test-only)\n // https://git.musl-libc.org/cgit/musl/tree/arch/aarch64/syscall_arch.h?h=v1.2.6\n // - arm (test-only)\n // https://git.musl-libc.org/cgit/musl/tree/arch/arm/syscall_arch.h?h=v1.2.6\n // - powerpc64 (test-only)\n // https://github.com/torvalds/linux/blob/v7.1/Documentation/arch/powerpc/syscall64-abi.rst\n // https://git.musl-libc.org/cgit/musl/tree/arch/powerpc64/syscall_arch.h?h=v1.2.6\n // - riscv32/riscv64\n // https://git.musl-libc.org/cgit/musl/tree/arch/riscv32/syscall_arch.h?h=v1.2.6\n // https://git.musl-libc.org/cgit/musl/tree/arch/riscv64/syscall_arch.h?h=v1.2.6\n\n #[cfg(test)] // test-only\n #[cfg(all(target_arch = \"aarch64\", target_pointer_width = \"64\"))]\n macro_rules! asm_syscall {\n (\n $number:ident, $r:ident,\n $($arg1:ident $(, $arg2:ident $(, $arg3:ident\n $(, $arg4:ident $(, $arg5:ident $(, $arg6:ident )?)?)?\n )?)?)?\n ) => {\n __asm!(\n \"svc 0\",\n in(\"x8\") $number,\n lateout(\"x0\") $r,\n $(in(\"x0\") $arg1,\n $(in(\"x1\") $arg2,\n $(in(\"x2\") $arg3,\n $(in(\"x3\") $arg4,\n $(in(\"x4\") $arg5,\n $(in(\"x5\") $arg6, )?\n )?\n )?\n )?\n )?\n )?\n // Clobber SVE registers and do not use `preserves_flags` because\n // AArch64 Linux syscalls clears non-v[0-31] bits of z[0-31], and all of p[0-15] and ffr,\n // and calls SMSTOP SM which clears z[0-31], p[0-15], ffr, and modifies FPSR\n // when CPU is in streaming SVE mode.\n // https://github.com/torvalds/linux/blob/v7.1/Documentation/arch/arm64/sve.rst#3--system-call-behaviour\n // https://github.com/torvalds/linux/blob/v7.1/Documentation/arch/arm64/sme.rst#3--system-call-behaviour\n // https://developer.arm.com/documentation/109246/0101/SME-Overview/Streaming-SVE-mode\n out(\"z0\") _,\n out(\"z1\") _,\n out(\"z2\") _,\n out(\"z3\") _,\n out(\"z4\") _,\n out(\"z5\") _,\n out(\"z6\") _,\n out(\"z7\") _,\n out(\"z8\") _,\n out(\"z9\") _,\n out(\"z10\") _,\n out(\"z11\") _,\n out(\"z12\") _,\n out(\"z13\") _,\n out(\"z14\") _,\n out(\"z15\") _,\n out(\"z16\") _,\n out(\"z17\") _,\n out(\"z18\") _,\n out(\"z19\") _,\n out(\"z20\") _,\n out(\"z21\") _,\n out(\"z22\") _,\n out(\"z23\") _,\n out(\"z24\") _,\n out(\"z25\") _,\n out(\"z26\") _,\n out(\"z27\") _,\n out(\"z28\") _,\n out(\"z29\") _,\n out(\"z30\") _,\n out(\"z31\") _,\n out(\"p0\") _,\n out(\"p1\") _,\n out(\"p2\") _,\n out(\"p3\") _,\n out(\"p4\") _,\n out(\"p5\") _,\n out(\"p6\") _,\n out(\"p7\") _,\n out(\"p8\") _,\n out(\"p9\") _,\n out(\"p10\") _,\n out(\"p11\") _,\n out(\"p12\") _,\n out(\"p13\") _,\n out(\"p14\") _,\n out(\"p15\") _,\n out(\"ffr\") _,\n options(nostack),\n )\n };\n }\n #[cfg(test)] // test-only\n #[cfg(target_arch = \"arm\")]\n macro_rules! asm_syscall {\n (\n $number_const:path, $number:literal, $r:ident,\n $($arg1:ident $(, $arg2:ident $(, $arg3:ident\n $(, $arg4:ident $(, $arg5:ident $(, $arg6:ident )?)?)?\n )?)?)?\n ) => {{\n static_assert!($number_const == $number && 0 <= $number && $number <= 255);\n __asm!(\n // r7 is reserved on thumb and MOV requires Thumb-2 or Arm mode.\n // cfg(target_feature = \"thumb-mode\")/cfg(target_feature = \"thumb2\")\n // doesn't work on stable and register swapping is much cheaper than syscall,\n // so we always swap register and use MOVS instead of MOV.\n // Note: r6 is reserved by LLVM, so this assembly may not work for syscall6\n // with Thumb-1 (pre-v7 Arm in Thumb mode) because the allocation of\n // `tmp` may fail.\n // (syscall6 is needless in the current our use cases.)\n \"movs {tmp}, r7\",\n concat!(\"movs r7, \", $number),\n \"svc 0\",\n \"movs r7, {tmp}\",\n tmp = out(reg) _,\n lateout(\"r0\") $r,\n $(in(\"r0\") $arg1,\n $(in(\"r1\") $arg2,\n $(in(\"r2\") $arg3,\n $(in(\"r3\") $arg4,\n $(in(\"r4\") $arg5,\n $(in(\"r5\") $arg6, )?\n )?\n )?\n )?\n )?\n )?\n // Do not use `preserves_flags` because MOVS modifies the flags.\n // Do not use `nostack` because SVC pushes to stack on M-profile architectures.\n // https://github.com/torvalds/linux/blob/v7.1/arch/arm/kernel/entry-header.S#L61\n )\n }};\n }\n // POWER9+ has fast syscall using SCV, but it is needless in the current our use cases.\n #[cfg(test)] // test-only\n #[cfg(all(target_arch = \"powerpc64\", target_pointer_width = \"64\"))]\n macro_rules! asm_syscall {\n (\n $number:ident, $r:ident,\n $($arg1:ident $(, $arg2:ident $(, $arg3:ident\n $(, $arg4:ident $(, $arg5:ident $(, $arg6:ident )?)?)?\n )?)?)?\n ) => {\n __asm!(\n \"sc\",\n \"bns+ 2f\",\n \"neg %r3, %r3\",\n \"2:\",\n inout(\"r0\") $number => _,\n lateout(\"r3\") $r,\n $(in(\"r3\") $arg1,\n $(in(\"r4\") $arg2,\n $(in(\"r5\") $arg3,\n $(in(\"r6\") $arg4,\n $(in(\"r7\") $arg5,\n $(in(\"r8\") $arg6, )?\n )?\n )?\n )?\n )?\n )?\n lateout(\"r4\") _,\n lateout(\"r5\") _,\n lateout(\"r6\") _,\n lateout(\"r7\") _,\n lateout(\"r8\") _,\n out(\"r9\") _,\n out(\"r10\") _,\n out(\"r11\") _,\n out(\"r12\") _,\n out(\"cr0\") _,\n out(\"ctr\") _,\n out(\"xer\") _,\n options(nostack, preserves_flags),\n )\n };\n }\n #[cfg(any(\n target_arch = \"riscv32\",\n all(target_arch = \"riscv64\", target_pointer_width = \"64\"),\n ))]\n macro_rules! asm_syscall {\n (\n $number:ident, $r:ident,\n $($arg1:ident $(, $arg2:ident $(, $arg3:ident\n $(, $arg4:ident $(, $arg5:ident $(, $arg6:ident )?)?)?\n )?)?)?\n ) => {\n __asm!(\n \"ecall\",\n in(\"a7\") $number,\n lateout(\"a0\") $r,\n $(in(\"a0\") $arg1,\n $(in(\"a1\") $arg2,\n $(in(\"a2\") $arg3,\n $(in(\"a3\") $arg4,\n $(in(\"a4\") $arg5,\n $(in(\"a5\") $arg6, )?\n )?\n )?\n )?\n )?\n )?\n // Clobber vector registers and do not use `preserves_flags` because RISC-V Linux syscalls don't preserve them.\n // https://github.com/torvalds/linux/blob/v7.1/Documentation/arch/riscv/vector.rst#3--vector-register-state-across-system-calls\n out(\"v0\") _,\n out(\"v1\") _,\n out(\"v2\") _,\n out(\"v3\") _,\n out(\"v4\") _,\n out(\"v5\") _,\n out(\"v6\") _,\n out(\"v7\") _,\n out(\"v8\") _,\n out(\"v9\") _,\n out(\"v10\") _,\n out(\"v11\") _,\n out(\"v12\") _,\n out(\"v13\") _,\n out(\"v14\") _,\n out(\"v15\") _,\n out(\"v16\") _,\n out(\"v17\") _,\n out(\"v18\") _,\n out(\"v19\") _,\n out(\"v20\") _,\n out(\"v21\") _,\n out(\"v22\") _,\n out(\"v23\") _,\n out(\"v24\") _,\n out(\"v25\") _,\n out(\"v26\") _,\n out(\"v27\") _,\n out(\"v28\") _,\n out(\"v29\") _,\n out(\"v30\") _,\n out(\"v31\") _,\n options(nostack),\n )\n };\n }\n }\n\n #[allow(\n clippy::alloc_instead_of_core,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n clippy::undocumented_unsafe_blocks,\n clippy::wildcard_imports\n )]\n #[cfg(test)]\n mod tests {\n #[test]\n fn test_c_macro() {\n #[track_caller]\n fn t(s: &crate::utils::ffi::CStr, raw: &[u8]) {\n assert_eq!(s.to_bytes_with_nul(), raw);\n }\n t(c!(\"\"), b\"\\0\");\n t(c!(\"a\"), b\"a\\0\");\n t(c!(\"abc\"), b\"abc\\0\");\n t(c!(concat!(\"abc\", \"d\")), b\"abcd\\0\");\n }\n\n #[test]\n fn test_is_c_str() {\n #[track_caller]\n fn t(bytes: &[u8]) {\n assert_eq!(\n super::_const_is_c_str(bytes),\n std::ffi::CStr::from_bytes_with_nul(bytes).is_ok()\n );\n }\n t(b\"\\0\");\n t(b\"a\\0\");\n t(b\"abc\\0\");\n t(b\"\");\n t(b\"a\");\n t(b\"abc\");\n t(b\"\\0a\");\n t(b\"\\0a\\0\");\n t(b\"ab\\0c\\0\");\n t(b\"\\0\\0\");\n }\n }\n}\n" }, { "package": "portable-atomic@1.15.0", "sourcePath": "version.rs", "sha256": "25c4e742d5a9e49ee16793b607157e181fd79eeafcbcbf0aa389f31b8d5514d1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse std::{env, iter, process::Command, str};\n\npub(crate) fn rustc_version() -> Option {\n let rustc = env::var_os(\"RUSTC\")?;\n let rustc_wrapper = if env::var_os(\"CARGO_ENCODED_RUSTFLAGS\").is_some() {\n env::var_os(\"RUSTC_WRAPPER\").filter(|v| !v.is_empty())\n } else {\n // Cargo sets environment variables for wrappers correctly only since https://github.com/rust-lang/cargo/pull/9601.\n None\n };\n // Do not apply RUSTC_WORKSPACE_WRAPPER: https://github.com/cuviper/autocfg/issues/58#issuecomment-2067625980\n let mut rustc = rustc_wrapper.into_iter().chain(iter::once(rustc));\n let mut cmd = Command::new(rustc.next().unwrap());\n cmd.args(rustc);\n // Use verbose version output because the packagers add extra strings to the normal version output.\n // Do not use long flags (--version --verbose) because clippy-deriver doesn't handle them properly.\n // -vV is also matched with that cargo internally uses: https://github.com/rust-lang/cargo/blob/0.80.0/src/cargo/util/rustc.rs#L65\n let output = cmd.arg(\"-vV\").output().ok()?;\n let verbose_version = str::from_utf8(&output.stdout).ok()?;\n Version::parse(verbose_version)\n}\n\n#[cfg_attr(test, derive(Debug, PartialEq))]\npub(crate) struct Version {\n pub(crate) minor: u32,\n pub(crate) nightly: bool,\n commit_date: Date,\n pub(crate) llvm: u32,\n}\n\nimpl Version {\n // The known latest stable version. If we unable to determine\n // the rustc version, we assume this is the current version.\n // It is no problem if this is older than the actual latest stable.\n // LLVM version is assumed to be the minimum external LLVM version:\n // https://github.com/rust-lang/rust/blob/1.97.0/src/bootstrap/src/core/build_steps/llvm.rs#L638\n pub(crate) const LATEST: Self = Self::stable(97, 21);\n\n pub(crate) const fn stable(rustc_minor: u32, llvm_major: u32) -> Self {\n Self { minor: rustc_minor, nightly: false, commit_date: Date::UNKNOWN, llvm: llvm_major }\n }\n\n pub(crate) fn probe(&self, minor: u32, year: u16, month: u8, day: u8) -> bool {\n if self.nightly {\n self.minor > minor\n || self.minor == minor && self.commit_date >= Date::new(year, month, day)\n } else {\n self.minor >= minor\n }\n }\n\n #[cfg(test)]\n pub(crate) fn commit_date(&self) -> &Date {\n &self.commit_date\n }\n\n pub(crate) fn parse(verbose_version: &str) -> Option {\n let mut release = verbose_version\n .lines()\n .find(|line| line.starts_with(\"release: \"))\n .map(|line| &line[\"release: \".len()..])?\n .splitn(2, '-');\n let version = release.next().unwrap();\n let channel = release.next().unwrap_or_default();\n let mut digits = version.splitn(3, '.');\n let major = digits.next()?;\n if major != \"1\" {\n return None;\n }\n let minor = digits.next()?.parse::().ok()?;\n let _patch = digits.next().unwrap_or(\"0\").parse::().ok()?;\n let nightly = match env::var_os(\"RUSTC_BOOTSTRAP\") {\n // When -1 is passed rustc works like stable, e.g., cfg(target_feature = \"unstable_target_feature\") will never be set. https://github.com/rust-lang/rust/pull/132993\n Some(ref v) if v == \"-1\" => false,\n // When 1 is passed stable rustc works like nightly, but we ignore it for now.\n _ => channel == \"nightly\" || channel == \"dev\",\n };\n\n // Note that rustc 1.49-1.50 (and 1.13 or older) don't print LLVM version.\n let llvm_major = (|| {\n let version = verbose_version\n .lines()\n .find(|line| line.starts_with(\"LLVM version: \"))\n .map(|line| &line[\"LLVM version: \".len()..])?;\n let mut digits = version.splitn(3, '.');\n let major = digits.next()?.parse::().ok()?;\n let _minor = digits.next()?.parse::().ok()?;\n let _patch = digits.next().unwrap_or(\"0\").parse::().ok()?;\n Some(major)\n })()\n .unwrap_or(0);\n\n // we don't refer commit date on stable/beta.\n if nightly {\n let commit_date = (|| {\n let mut commit_date = verbose_version\n .lines()\n .find(|line| line.starts_with(\"commit-date: \"))\n .map(|line| &line[\"commit-date: \".len()..])?\n .splitn(3, '-');\n let year = commit_date.next()?.parse::().ok()?;\n let month = commit_date.next()?.parse::().ok()?;\n let day = commit_date.next()?.parse::().ok()?;\n if month > 12 || day > 31 {\n return None;\n }\n Some(Date::new(year, month, day))\n })();\n Some(Self {\n minor,\n nightly,\n commit_date: commit_date.unwrap_or(Date::UNKNOWN),\n llvm: llvm_major,\n })\n } else {\n Some(Self::stable(minor, llvm_major))\n }\n }\n}\n\n#[derive(PartialEq, PartialOrd)]\n#[cfg_attr(test, derive(Debug))]\npub(crate) struct Date {\n pub(crate) year: u16,\n pub(crate) month: u8,\n pub(crate) day: u8,\n}\n\nimpl Date {\n const UNKNOWN: Self = Self::new(0, 0, 0);\n\n const fn new(year: u16, month: u8, day: u8) -> Self {\n Self { year, month, day }\n }\n}\n" }, { "package": "portable-atomic-util@0.2.8", "sourcePath": "README.md", "sha256": "340557050bd651407ee059bc56c3296461d3be4fb9caf22b22784bd11931959b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# portable-atomic-util\n\n[![crates.io](https://img.shields.io/crates/v/portable-atomic-util?style=flat-square&logo=rust)](https://crates.io/crates/portable-atomic-util)\n[![docs.rs](https://img.shields.io/badge/docs.rs-portable--atomic--util-blue?style=flat-square&logo=docs.rs)](https://docs.rs/portable-atomic-util)\n[![license](https://img.shields.io/badge/license-Apache--2.0_OR_MIT-blue?style=flat-square)](#license)\n[![msrv](https://img.shields.io/badge/msrv-1.36-blue?style=flat-square&logo=rust)](https://www.rust-lang.org)\n[![github actions](https://img.shields.io/github/actions/workflow/status/taiki-e/portable-atomic-util/ci.yml?branch=main&style=flat-square&logo=github)](https://github.com/taiki-e/portable-atomic-util/actions)\n\n\n\nSynchronization primitives built with [portable-atomic].\n\n- Provide `Arc`. (optional, requires the `std` or `alloc` feature)\n- Provide `task::Wake`. (optional, requires the `std` or `alloc` feature)\n\n\nSee [portable-atomic#1] for other primitives being considered for addition to this crate.\n\nThis crate was originally [part of the portable-atomic repository](https://github.com/taiki-e/portable-atomic/tree/cbbee0c0d202a5944f7d66aaafaac6ed76e6f599/portable-atomic-util) and was extracted into its own repository.\n\n## Optional features\n\n- **`std`**
\n Use `std`.\n\n Note:\n - This implicitly enables the `alloc` feature.\n\n- **`alloc`**
\n Use `alloc`.\n\n- **`serde`**
\n Implement `serde::{Serialize, Deserialize}` for `Arc`.\n\n Note:\n - The MSRV when this feature is enabled is the one coming from serde, as of now it's Rust 1.56\n\n\n\n[portable-atomic]: https://github.com/taiki-e/portable-atomic\n[portable-atomic#1]: https://github.com/taiki-e/portable-atomic/issues/1\n\n## Optional cfg\n\nOne of the ways to enable cfg is to set [rustflags in the cargo config](https://doc.rust-lang.org/cargo/reference/config.html#targettriplerustflags):\n\n```toml\n# .cargo/config.toml\n[target.]\nrustflags = [\"--cfg\", \"portable_atomic_unstable_coerce_unsized\"]\n```\n\nOr set environment variable:\n\n```sh\nRUSTFLAGS=\"--cfg portable_atomic_unstable_coerce_unsized\" cargo ...\n```\n\n- **`--cfg portable_atomic_unstable_coerce_unsized`**
\n Support coercing of `Arc` to `Arc` as in `std::sync::Arc`.\n\n \n\n This cfg requires Rust nightly because this coercing requires [unstable `CoerceUnsized` trait](https://doc.rust-lang.org/nightly/core/ops/trait.CoerceUnsized.html).\n\n See [this issue comment](https://github.com/taiki-e/portable-atomic/issues/143#issuecomment-1866488569) for another known workaround.\n\n **Note:** This cfg is unstable and outside of the normal semver guarantees and minor or patch versions of portable-atomic-util may make breaking changes to them at any time.\n\n## Related Projects\n\n- [portable-atomic]: Portable atomic types including support for 128-bit atomics, atomic float, etc.\n- [atomic-maybe-uninit]: Atomic operations on potentially uninitialized integers.\n- [atomic-memcpy]: Byte-wise atomic memcpy.\n\n[atomic-maybe-uninit]: https://github.com/taiki-e/atomic-maybe-uninit\n[atomic-memcpy]: https://github.com/taiki-e/atomic-memcpy\n\n\n\n## License\n\nLicensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or\n[MIT license](LICENSE-MIT) at your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "portable-atomic-util@0.2.8", "sourcePath": "build.rs", "sha256": "f17a8f6b7c93fa6bcefc325a7b75e9e03fc9c97f23ded051f31d6dc8b8e0d478", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n// The rustc-cfg emitted by the build script are *not* public API.\n\nuse std::env;\n\nuse self::version::{Version, rustc_version};\n\nfn main() {\n println!(\"cargo:rerun-if-changed=build.rs\");\n\n let target = &*env::var(\"TARGET\").expect(\"TARGET not set\");\n\n let version = match rustc_version() {\n Some(version) => version,\n None => {\n if env::var_os(\"PORTABLE_ATOMIC_DENY_WARNINGS\").is_some() {\n panic!(\"unable to determine rustc version\")\n }\n println!(\n \"cargo:warning={}: unable to determine rustc version; assuming latest stable rustc (1.{})\",\n env!(\"CARGO_PKG_NAME\"),\n Version::LATEST.minor\n );\n Version::LATEST\n }\n };\n\n if version.minor >= 80 {\n // Custom cfgs set by build script. Not public API.\n // grep -F 'cargo:rustc-cfg=' portable-atomic-util/build.rs | grep -Ev '^ *//' | sed -E 's/^.*cargo:rustc-cfg=//; s/(=\\\\)?\".*$//' | LC_ALL=C sort -u | tr '\\n' ',' | sed -E 's/,$/\\n/'\n println!(\n \"cargo:rustc-check-cfg=cfg(portable_atomic_no_core_unwind_safe,portable_atomic_no_error_in_core,portable_atomic_no_io_safety,portable_atomic_no_min_const_generics,portable_atomic_no_strict_provenance,portable_atomic_no_track_caller,portable_atomic_no_unsafe_op_in_unsafe_fn,portable_atomic_sanitize_thread)\"\n );\n }\n\n // Note that cfgs are `no_`*, not `has_*`. This allows treating as the latest\n // stable rustc is used when the build script doesn't run. This is useful\n // for non-cargo build systems that don't run the build script.\n\n // track_caller stabilized in Rust 1.46 (nightly-2020-07-02): https://github.com/rust-lang/rust/pull/72445\n if !version.probe(46, 2020, 7, 1) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_track_caller\");\n }\n // min_const_generics stabilized in Rust 1.51 (nightly-2020-12-28): https://github.com/rust-lang/rust/pull/79135\n if !version.probe(51, 2020, 12, 27) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_min_const_generics\");\n }\n // unsafe_op_in_unsafe_fn stabilized in Rust 1.52 (nightly-2021-03-11): https://github.com/rust-lang/rust/pull/79208\n if !version.probe(52, 2021, 3, 10) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_unsafe_op_in_unsafe_fn\");\n }\n // https://github.com/rust-lang/rust/pull/84662 merged in Rust 1.56 (nightly-2021-08-02).\n if !version.probe(56, 2021, 8, 1) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_core_unwind_safe\");\n }\n // io_safety stabilized in Rust 1.63 (nightly-2022-06-16): https://github.com/rust-lang/rust/pull/95118\n // std::os::hermit::io::AsFd requires Rust 1.69 (https://github.com/rust-lang/rust/commit/b5fb4f3d9b1b308d59cab24ef2f9bf23dad948aa)\n if !version.probe(63, 2022, 6, 15)\n || version.minor < 69\n && env::var(\"CARGO_CFG_TARGET_OS\").expect(\"CARGO_CFG_TARGET_OS not set\") == \"hermit\"\n {\n println!(\"cargo:rustc-cfg=portable_atomic_no_io_safety\");\n }\n // error_in_core stabilized in Rust 1.81 (nightly-2024-06-09): https://github.com/rust-lang/rust/pull/125951\n if !version.probe(81, 2024, 6, 8) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_error_in_core\");\n }\n // strict_provenance/exposed_provenance APIs stabilized in Rust 1.84 (nightly-2024-10-22): https://github.com/rust-lang/rust/pull/130350\n if !version.probe(84, 2024, 10, 21) {\n println!(\"cargo:rustc-cfg=portable_atomic_no_strict_provenance\");\n }\n\n // `cfg(sanitize = \"..\")` is not stabilized.\n // x86_64-unknown-linux-gnutsan is available on stable since Rust 1.96: https://github.com/rust-lang/rust/pull/152757\n let sanitize = env::var(\"CARGO_CFG_SANITIZE\").unwrap_or_default();\n if sanitize.contains(\"thread\") || target == \"x86_64-unknown-linux-gnutsan\" {\n println!(\"cargo:rustc-cfg=portable_atomic_sanitize_thread\");\n }\n}\n\nmod version {\n use std::{env, iter, process::Command, str};\n\n pub(crate) fn rustc_version() -> Option {\n let rustc = env::var_os(\"RUSTC\")?;\n let rustc_wrapper = if env::var_os(\"CARGO_ENCODED_RUSTFLAGS\").is_some() {\n env::var_os(\"RUSTC_WRAPPER\").filter(|v| !v.is_empty())\n } else {\n // Cargo sets environment variables for wrappers correctly only since https://github.com/rust-lang/cargo/pull/9601.\n None\n };\n // Do not apply RUSTC_WORKSPACE_WRAPPER: https://github.com/cuviper/autocfg/issues/58#issuecomment-2067625980\n let mut rustc = rustc_wrapper.into_iter().chain(iter::once(rustc));\n let mut cmd = Command::new(rustc.next().unwrap());\n cmd.args(rustc);\n // Use verbose version output because the packagers add extra strings to the normal version output.\n // Do not use long flags (--version --verbose) because clippy-deriver doesn't handle them properly.\n // -vV is also matched with that cargo internally uses: https://github.com/rust-lang/cargo/blob/0.80.0/src/cargo/util/rustc.rs#L65\n let output = cmd.arg(\"-vV\").output().ok()?;\n let verbose_version = str::from_utf8(&output.stdout).ok()?;\n Version::parse(verbose_version)\n }\n\n pub(crate) struct Version {\n pub(crate) minor: u32,\n pub(crate) nightly: bool,\n commit_date: Date,\n }\n\n impl Version {\n // The known latest stable version. If we unable to determine\n // the rustc version, we assume this is the current version.\n // It is no problem if this is older than the actual latest stable.\n pub(crate) const LATEST: Self = Self::stable(98);\n\n pub(crate) const fn stable(rustc_minor: u32) -> Self {\n Self { minor: rustc_minor, nightly: false, commit_date: Date::UNKNOWN }\n }\n\n pub(crate) fn probe(&self, minor: u32, year: u16, month: u8, day: u8) -> bool {\n if self.nightly {\n self.minor > minor\n || self.minor == minor && self.commit_date >= Date::new(year, month, day)\n } else {\n self.minor >= minor\n }\n }\n\n pub(crate) fn parse(verbose_version: &str) -> Option {\n let mut release = verbose_version\n .lines()\n .find(|line| line.starts_with(\"release: \"))\n .map(|line| &line[\"release: \".len()..])?\n .splitn(2, '-');\n let version = release.next().unwrap();\n let channel = release.next().unwrap_or_default();\n let mut digits = version.splitn(3, '.');\n let major = digits.next()?;\n if major != \"1\" {\n return None;\n }\n let minor = digits.next()?.parse::().ok()?;\n let _patch = digits.next().unwrap_or(\"0\").parse::().ok()?;\n let nightly = match env::var_os(\"RUSTC_BOOTSTRAP\") {\n // When -1 is passed rustc works like stable, e.g., cfg(target_feature = \"unstable_target_feature\") will never be set. https://github.com/rust-lang/rust/pull/132993\n Some(ref v) if v == \"-1\" => false,\n // When 1 is passed stable rustc works like nightly, but we ignore it for now.\n _ => channel == \"nightly\" || channel == \"dev\",\n };\n\n // we don't refer commit date on stable/beta.\n if nightly {\n let commit_date = (|| {\n let mut commit_date = verbose_version\n .lines()\n .find(|line| line.starts_with(\"commit-date: \"))\n .map(|line| &line[\"commit-date: \".len()..])?\n .splitn(3, '-');\n let year = commit_date.next()?.parse::().ok()?;\n let month = commit_date.next()?.parse::().ok()?;\n let day = commit_date.next()?.parse::().ok()?;\n if month > 12 || day > 31 {\n return None;\n }\n Some(Date::new(year, month, day))\n })();\n Some(Self { minor, nightly, commit_date: commit_date.unwrap_or(Date::UNKNOWN) })\n } else {\n Some(Self::stable(minor))\n }\n }\n }\n\n #[derive(PartialEq, PartialOrd)]\n pub(crate) struct Date {\n pub(crate) year: u16,\n pub(crate) month: u8,\n pub(crate) day: u8,\n }\n\n impl Date {\n const UNKNOWN: Self = Self::new(0, 0, 0);\n\n const fn new(year: u16, month: u8, day: u8) -> Self {\n Self { year, month, day }\n }\n }\n}\n" }, { "package": "portable-atomic-util@0.2.8", "sourcePath": "src/arc.rs", "sha256": "e23f18ca0592f694ac2da85e9b83ea3add4fad978cd37ac05c742af254fa82a9", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n// This module is based on alloc::sync::Arc.\n//\n// The code has been adjusted to work with stable Rust (and optionally support some unstable features).\n//\n// Source: https://github.com/rust-lang/rust/blob/1.93.0/library/alloc/src/sync.rs\n//\n// Copyright & License of the original code:\n// - https://github.com/rust-lang/rust/blob/1.93.0/COPYRIGHT\n// - https://github.com/rust-lang/rust/blob/1.93.0/LICENSE-APACHE\n// - https://github.com/rust-lang/rust/blob/1.93.0/LICENSE-MIT\n\n#![allow(clippy::must_use_candidate)] // align to alloc::sync::Arc\n#![allow(clippy::undocumented_unsafe_blocks)] // TODO: most of the unsafe codes were inherited from alloc::sync::Arc\n\nuse alloc::{\n alloc::handle_alloc_error,\n borrow::{Cow, ToOwned},\n boxed::Box,\n string::String,\n vec::Vec,\n};\n#[cfg(not(portable_atomic_no_min_const_generics))]\nuse core::convert::TryFrom;\nuse core::{\n alloc::Layout,\n any::Any,\n borrow,\n cmp::Ordering,\n fmt,\n hash::{Hash, Hasher},\n iter::FromIterator,\n marker::PhantomData,\n mem::{self, ManuallyDrop},\n ops::Deref,\n pin::Pin,\n ptr::{self, NonNull},\n slice,\n};\n#[allow(deprecated)]\nuse core::{isize, usize};\n#[cfg(portable_atomic_unstable_coerce_unsized)]\nuse core::{marker::Unsize, ops::CoerceUnsized};\n\nuse portable_atomic::{\n self as atomic,\n Ordering::{Acquire, Relaxed, Release},\n hint,\n};\n\nuse crate::utils::ptr as strict;\n#[cfg(portable_atomic_no_strict_provenance)]\nuse crate::utils::ptr::PtrExt as _;\n\n#[allow(deprecated)] // associated constant MAX requires Rust 1.43\nconst ISIZE_MAX: isize = isize::MAX;\n#[allow(deprecated)] // associated constant MAX requires Rust 1.43\nconst USIZE_MAX: usize = usize::MAX;\n\n/// A soft limit on the amount of references that may be made to an `Arc`.\n///\n/// Going above this limit will abort your program (although not\n/// necessarily) at _exactly_ `MAX_REFCOUNT + 1` references.\n/// Trying to go above it might call a `panic` (if not actually going above it).\n///\n/// This is a global invariant, and also applies when using a compare-exchange loop.\n///\n/// See comment in `Arc::clone`.\nconst MAX_REFCOUNT: usize = ISIZE_MAX as usize;\n\n/// The error in case either counter reaches above `MAX_REFCOUNT`, and we can `panic` safely.\nconst INTERNAL_OVERFLOW_ERROR: &str = \"Arc counter overflow\";\n\n#[cfg(not(portable_atomic_sanitize_thread))]\nmacro_rules! acquire {\n ($x:expr) => {\n atomic::fence(Acquire)\n };\n}\n\n// ThreadSanitizer does not support memory fences. To avoid false positive\n// reports in Arc / Weak implementation use atomic loads for synchronization\n// instead.\n#[cfg(portable_atomic_sanitize_thread)]\nmacro_rules! acquire {\n ($x:expr) => {\n $x.load(Acquire)\n };\n}\n\n/// A thread-safe reference-counting pointer. 'Arc' stands for 'Atomically\n/// Reference Counted'.\n///\n/// This is an equivalent to [`std::sync::Arc`], but using [portable-atomic] for synchronization.\n/// See the documentation for [`std::sync::Arc`] for more details.\n///\n/// **Note:** Unlike `std::sync::Arc`, coercing `Arc` to `Arc` is only possible if\n/// the optional cfg `portable_atomic_unstable_coerce_unsized` is enabled, as documented at the crate-level documentation,\n/// and this optional cfg item is only supported with Rust nightly version.\n/// This is because coercing the pointee requires the\n/// [unstable `CoerceUnsized` trait](https://doc.rust-lang.org/nightly/core/ops/trait.CoerceUnsized.html).\n/// See [this issue comment](https://github.com/taiki-e/portable-atomic/issues/143#issuecomment-1866488569)\n/// for a workaround that works without depending on unstable features.\n///\n/// [portable-atomic]: https://crates.io/crates/portable-atomic\n///\n/// # Examples\n///\n/// ```\n/// use portable_atomic_util::Arc;\n/// use std::thread;\n///\n/// let five = Arc::new(5);\n///\n/// for _ in 0..10 {\n/// let five = Arc::clone(&five);\n///\n/// thread::spawn(move || {\n/// assert_eq!(*five, 5);\n/// });\n/// }\n/// # if cfg!(miri) { std::thread::sleep(std::time::Duration::from_millis(500)); } // wait for background threads closed: https://github.com/rust-lang/miri/issues/1371\n/// ```\npub struct Arc {\n ptr: NonNull>,\n phantom: PhantomData>,\n}\n\nunsafe impl Send for Arc {}\nunsafe impl Sync for Arc {}\n\n#[cfg(not(portable_atomic_no_core_unwind_safe))]\nimpl core::panic::UnwindSafe for Arc {}\n#[cfg(all(portable_atomic_no_core_unwind_safe, feature = \"std\"))]\nimpl std::panic::UnwindSafe for Arc {}\n\n#[cfg(portable_atomic_unstable_coerce_unsized)]\nimpl, U: ?Sized> CoerceUnsized> for Arc {}\n\nimpl Arc {\n #[inline]\n fn into_inner_non_null(this: Self) -> NonNull> {\n let this = mem::ManuallyDrop::new(this);\n this.ptr\n }\n\n #[inline]\n unsafe fn from_inner(ptr: NonNull>) -> Self {\n Self { ptr, phantom: PhantomData }\n }\n\n #[inline]\n unsafe fn from_ptr(ptr: *mut ArcInner) -> Self {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { Self::from_inner(NonNull::new_unchecked(ptr)) }\n }\n}\n\n/// `Weak` is a version of [`Arc`] that holds a non-owning reference to the\n/// managed allocation.\n///\n/// The allocation is accessed by calling [`upgrade`] on the `Weak`\n/// pointer, which returns an [Option]<[Arc]\\>.\n///\n/// This is an equivalent to [`std::sync::Weak`], but using [portable-atomic] for synchronization.\n/// See the documentation for [`std::sync::Weak`] for more details.\n///\n/// \n/// **Note:** Unlike `std::sync::Weak`, coercing `Weak` to `Weak` is not possible, not even if\n/// the optional cfg `portable_atomic_unstable_coerce_unsized` is enabled.\n///\n/// [`upgrade`]: Weak::upgrade\n/// [portable-atomic]: https://crates.io/crates/portable-atomic\n///\n/// # Examples\n///\n/// ```\n/// use portable_atomic_util::Arc;\n/// use std::thread;\n///\n/// let five = Arc::new(5);\n/// let weak_five = Arc::downgrade(&five);\n///\n/// # let t =\n/// thread::spawn(move || {\n/// let five = weak_five.upgrade().unwrap();\n/// assert_eq!(*five, 5);\n/// });\n/// # t.join().unwrap(); // join thread to avoid https://github.com/rust-lang/miri/issues/1371\n/// ```\npub struct Weak {\n // This is a `NonNull` to allow optimizing the size of this type in enums,\n // but it is not necessarily a valid pointer.\n // `Weak::new` sets this to `usize::MAX` so that it doesn't need\n // to allocate space on the heap. That's not a value a real pointer\n // will ever have because ArcInner has alignment at least 2.\n ptr: NonNull>,\n}\n\nunsafe impl Send for Weak {}\nunsafe impl Sync for Weak {}\n\nimpl fmt::Debug for Weak {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.write_str(\"(Weak)\")\n }\n}\n\n// This is repr(C) to future-proof against possible field-reordering, which\n// would interfere with otherwise safe [into|from]_raw() of transmutable\n// inner types.\n// Unlike RcInner, repr(align(2)) is not strictly required because atomic types\n// have the alignment same as its size, but we use it for consistency and clarity.\n#[repr(C, align(2))]\nstruct ArcInner {\n strong: atomic::AtomicUsize,\n\n // the value usize::MAX acts as a sentinel for temporarily \"locking\" the\n // ability to upgrade weak pointers or downgrade strong ones; this is used\n // to avoid races in `make_mut` and `get_mut`.\n weak: atomic::AtomicUsize,\n\n data: T,\n}\n\n/// Calculate layout for `ArcInner` using the inner value's layout\nfn arc_inner_layout_for_value_layout(layout: Layout) -> Layout {\n // Calculate layout using the given value layout.\n // Previously, layout was calculated on the expression\n // `&*(ptr as *const ArcInner)`, but this created a misaligned\n // reference (see #54908).\n layout::pad_to_align(layout::extend(Layout::new::>(), layout).unwrap().0)\n}\n\nunsafe impl Send for ArcInner {}\nunsafe impl Sync for ArcInner {}\n\nimpl Arc {\n /// Constructs a new `Arc`.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n /// ```\n #[inline]\n pub fn new(data: T) -> Self {\n // Start the weak pointer count as 1 which is the weak pointer that's\n // held by all the strong pointers (kinda), see std/rc.rs for more info\n let x: Box<_> = Box::new(ArcInner {\n strong: atomic::AtomicUsize::new(1),\n weak: atomic::AtomicUsize::new(1),\n data,\n });\n unsafe { Self::from_inner(Box::leak(x).into()) }\n }\n\n /// Constructs a new `Arc` while giving you a `Weak` to the allocation,\n /// to allow you to construct a `T` which holds a weak pointer to itself.\n ///\n /// Generally, a structure circularly referencing itself, either directly or\n /// indirectly, should not hold a strong reference to itself to prevent a memory leak.\n /// Using this function, you get access to the weak pointer during the\n /// initialization of `T`, before the `Arc` is created, such that you can\n /// clone and store it inside the `T`.\n ///\n /// `new_cyclic` first allocates the managed allocation for the `Arc`,\n /// then calls your closure, giving it a `Weak` to this allocation,\n /// and only afterwards completes the construction of the `Arc` by placing\n /// the `T` returned from your closure into the allocation.\n ///\n /// Since the new `Arc` is not fully-constructed until `Arc::new_cyclic`\n /// returns, calling [`upgrade`] on the weak reference inside your closure will\n /// fail and result in a `None` value.\n ///\n /// # Panics\n ///\n /// If `data_fn` panics, the panic is propagated to the caller, and the\n /// temporary [`Weak`] is dropped normally.\n ///\n /// # Example\n ///\n /// ```\n /// use portable_atomic_util::{Arc, Weak};\n ///\n /// struct Gadget {\n /// me: Weak,\n /// }\n ///\n /// impl Gadget {\n /// /// Constructs a reference counted Gadget.\n /// fn new() -> Arc {\n /// // `me` is a `Weak` pointing at the new allocation of the\n /// // `Arc` we're constructing.\n /// Arc::new_cyclic(|me| {\n /// // Create the actual struct here.\n /// Gadget { me: me.clone() }\n /// })\n /// }\n ///\n /// /// Returns a reference counted pointer to Self.\n /// fn me(&self) -> Arc {\n /// self.me.upgrade().unwrap()\n /// }\n /// }\n /// ```\n /// [`upgrade`]: Weak::upgrade\n #[inline]\n pub fn new_cyclic(data_fn: F) -> Self\n where\n F: FnOnce(&Weak) -> T,\n {\n // Construct the inner in the \"uninitialized\" state with a single\n // weak reference.\n let init_ptr = Weak::new_uninit_ptr();\n\n let weak = Weak { ptr: init_ptr };\n\n // It's important we don't give up ownership of the weak pointer, or\n // else the memory might be freed by the time `data_fn` returns. If\n // we really wanted to pass ownership, we could create an additional\n // weak pointer for ourselves, but this would result in additional\n // updates to the weak reference count which might not be necessary\n // otherwise.\n let data = data_fn(&weak);\n\n // Now we can properly initialize the inner value and turn our weak\n // reference into a strong reference.\n unsafe {\n let inner = init_ptr.as_ptr();\n ptr::write(data_ptr::(inner, &data), data);\n\n // The above write to the data field must be visible to any threads which\n // observe a non-zero strong count. Therefore we need at least \"Release\" ordering\n // in order to synchronize with the `compare_exchange_weak` in `Weak::upgrade`.\n //\n // \"Acquire\" ordering is not required. When considering the possible behaviors\n // of `data_fn` we only need to look at what it could do with a reference to a\n // non-upgradeable `Weak`:\n // - It can *clone* the `Weak`, increasing the weak reference count.\n // - It can drop those clones, decreasing the weak reference count (but never to zero).\n //\n // These side effects do not impact us in any way, and no other side effects are\n // possible with safe code alone.\n let prev_value = (*inner).strong.fetch_add(1, Release);\n debug_assert_eq!(prev_value, 0, \"No prior strong references should exist\");\n\n // Strong references should collectively own a shared weak reference,\n // so don't run the destructor for our old weak reference.\n mem::forget(weak);\n\n Self::from_inner(init_ptr)\n }\n }\n\n /// Constructs a new `Arc` with uninitialized contents.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let mut five = Arc::::new_uninit();\n ///\n /// // Deferred initialization:\n /// Arc::get_mut(&mut five).unwrap().write(5);\n ///\n /// let five = unsafe { five.assume_init() };\n ///\n /// assert_eq!(*five, 5)\n /// ```\n #[inline]\n #[must_use]\n pub fn new_uninit() -> Arc> {\n unsafe {\n Arc::from_ptr(Arc::allocate_for_layout(\n Layout::new::(),\n |layout| Global.allocate(layout),\n |ptr| ptr as *mut _,\n ))\n }\n }\n\n /// Constructs a new `Arc` with uninitialized contents, with the memory\n /// being filled with `0` bytes.\n ///\n /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage\n /// of this method.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let zero = Arc::::new_zeroed();\n /// let zero = unsafe { zero.assume_init() };\n ///\n /// assert_eq!(*zero, 0)\n /// ```\n ///\n /// [zeroed]: mem::MaybeUninit::zeroed\n #[inline]\n #[must_use]\n pub fn new_zeroed() -> Arc> {\n unsafe {\n Arc::from_ptr(Arc::allocate_for_layout(\n Layout::new::(),\n |layout| Global.allocate_zeroed(layout),\n |ptr| ptr as *mut _,\n ))\n }\n }\n\n /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then\n /// `data` will be pinned in memory and unable to be moved.\n #[must_use]\n pub fn pin(data: T) -> Pin {\n unsafe { Pin::new_unchecked(Self::new(data)) }\n }\n\n /// Returns the inner value, if the `Arc` has exactly one strong reference.\n ///\n /// Otherwise, an [`Err`] is returned with the same `Arc` that was\n /// passed in.\n ///\n /// This will succeed even if there are outstanding weak references.\n ///\n /// It is strongly recommended to use [`Arc::into_inner`] instead if you don't\n /// keep the `Arc` in the [`Err`] case.\n /// Immediately dropping the [`Err`]-value, as the expression\n /// `Arc::try_unwrap(this).ok()` does, can cause the strong count to\n /// drop to zero and the inner value of the `Arc` to be dropped.\n /// For instance, if two threads execute such an expression in parallel,\n /// there is a race condition without the possibility of unsafety:\n /// The threads could first both check whether they own the last instance\n /// in `Arc::try_unwrap`, determine that they both do not, and then both\n /// discard and drop their instance in the call to [`ok`][`Result::ok`].\n /// In this scenario, the value inside the `Arc` is safely destroyed\n /// by exactly one of the threads, but neither thread will ever be able\n /// to use the value.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let x = Arc::new(3);\n /// assert_eq!(Arc::try_unwrap(x), Ok(3));\n ///\n /// let x = Arc::new(4);\n /// let _y = Arc::clone(&x);\n /// assert_eq!(*Arc::try_unwrap(x).unwrap_err(), 4);\n /// ```\n #[inline]\n pub fn try_unwrap(this: Self) -> Result {\n if this.inner().strong.compare_exchange(1, 0, Relaxed, Relaxed).is_err() {\n return Err(this);\n }\n\n acquire!(this.inner().strong);\n\n let this = ManuallyDrop::new(this);\n let elem: T = unsafe { ptr::read(&this.ptr.as_ref().data) };\n\n // Make a weak pointer to clean up the implicit strong-weak reference\n let _weak = Weak { ptr: this.ptr };\n\n Ok(elem)\n }\n\n /// Returns the inner value, if the `Arc` has exactly one strong reference.\n ///\n /// Otherwise, [`None`] is returned and the `Arc` is dropped.\n ///\n /// This will succeed even if there are outstanding weak references.\n ///\n /// If `Arc::into_inner` is called on every clone of this `Arc`,\n /// it is guaranteed that exactly one of the calls returns the inner value.\n /// This means in particular that the inner value is not dropped.\n ///\n /// [`Arc::try_unwrap`] is conceptually similar to `Arc::into_inner`, but it\n /// is meant for different use-cases. If used as a direct replacement\n /// for `Arc::into_inner` anyway, such as with the expression\n /// [Arc::try_unwrap]\\(this).[ok][Result::ok](), then it does\n /// **not** give the same guarantee as described in the previous paragraph.\n /// For more information, see the examples below and read the documentation\n /// of [`Arc::try_unwrap`].\n ///\n /// # Examples\n ///\n /// Minimal example demonstrating the guarantee that `Arc::into_inner` gives.\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let x = Arc::new(3);\n /// let y = Arc::clone(&x);\n ///\n /// // Two threads calling `Arc::into_inner` on both clones of an `Arc`:\n /// let x_thread = std::thread::spawn(|| Arc::into_inner(x));\n /// let y_thread = std::thread::spawn(|| Arc::into_inner(y));\n ///\n /// let x_inner_value = x_thread.join().unwrap();\n /// let y_inner_value = y_thread.join().unwrap();\n ///\n /// // One of the threads is guaranteed to receive the inner value:\n /// assert!(matches!((x_inner_value, y_inner_value), (None, Some(3)) | (Some(3), None)));\n /// // The result could also be `(None, None)` if the threads called\n /// // `Arc::try_unwrap(x).ok()` and `Arc::try_unwrap(y).ok()` instead.\n /// ```\n ///\n /// A more practical example demonstrating the need for `Arc::into_inner`:\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// // Definition of a simple singly linked list using `Arc`:\n /// #[derive(Clone)]\n /// struct LinkedList(Option>>);\n /// struct Node(T, Option>>);\n ///\n /// // Dropping a long `LinkedList` relying on the destructor of `Arc`\n /// // can cause a stack overflow. To prevent this, we can provide a\n /// // manual `Drop` implementation that does the destruction in a loop:\n /// impl Drop for LinkedList {\n /// fn drop(&mut self) {\n /// let mut link = self.0.take();\n /// while let Some(arc_node) = link.take() {\n /// if let Some(Node(_value, next)) = Arc::into_inner(arc_node) {\n /// link = next;\n /// }\n /// }\n /// }\n /// }\n ///\n /// // Implementation of `new` and `push` omitted\n /// impl LinkedList {\n /// /* ... */\n /// # fn new() -> Self {\n /// # LinkedList(None)\n /// # }\n /// # fn push(&mut self, x: T) {\n /// # self.0 = Some(Arc::new(Node(x, self.0.take())));\n /// # }\n /// }\n ///\n /// // The following code could have still caused a stack overflow\n /// // despite the manual `Drop` impl if that `Drop` impl had used\n /// // `Arc::try_unwrap(arc).ok()` instead of `Arc::into_inner(arc)`.\n ///\n /// // Create a long list and clone it\n /// let mut x = LinkedList::new();\n /// let size = 100_000;\n /// # let size = if cfg!(miri) { 100 } else { size };\n /// for i in 0..size {\n /// x.push(i); // Adds i to the front of x\n /// }\n /// let y = x.clone();\n ///\n /// // Drop the clones in parallel\n /// let x_thread = std::thread::spawn(|| drop(x));\n /// let y_thread = std::thread::spawn(|| drop(y));\n /// x_thread.join().unwrap();\n /// y_thread.join().unwrap();\n /// ```\n #[inline]\n pub fn into_inner(this: Self) -> Option {\n // Make sure that the ordinary `Drop` implementation isn't called as well\n let mut this = mem::ManuallyDrop::new(this);\n\n // Following the implementation of `drop` and `drop_slow`\n if this.inner().strong.fetch_sub(1, Release) != 1 {\n return None;\n }\n\n acquire!(this.inner().strong);\n\n // SAFETY: This mirrors the line\n //\n // unsafe { ptr::drop_in_place(Self::get_mut_unchecked(self)) };\n //\n // in `drop_slow`. Instead of dropping the value behind the pointer,\n // it is read and eventually returned; `ptr::read` has the same\n // safety conditions as `ptr::drop_in_place`.\n let inner = unsafe { ptr::read(Self::get_mut_unchecked(&mut this)) };\n\n drop(Weak { ptr: this.ptr });\n\n Some(inner)\n }\n\n /// Maps the value in an `Arc`, reusing the allocation if possible.\n ///\n /// `f` is called on a reference to the value in the `Arc`, and the result is returned, also in\n /// an `Arc`.\n ///\n /// Note: this is an associated function, which means that you have\n /// to call it as `Arc::map(a, f)` instead of `r.map(a)`. This\n /// is so that there is no conflict with a method on the inner type.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let r = Arc::new(7);\n /// let new = Arc::map(r, |i| i + 7);\n /// assert_eq!(*new, 14);\n /// ```\n #[inline]\n pub fn map(this: Self, f: F) -> Arc\n where\n F: FnOnce(&T) -> U,\n {\n if mem::size_of::() == mem::size_of::()\n && mem::align_of::() == mem::align_of::()\n && Arc::is_unique(&this)\n {\n unsafe {\n let ptr = Arc::into_raw(this);\n let value = ptr.read();\n let mut allocation = Arc::from_raw(ptr as *const mem::MaybeUninit);\n\n *Arc::get_mut_unchecked(&mut allocation) = mem::MaybeUninit::new(f(&value));\n allocation.assume_init()\n }\n } else {\n Arc::new(f(&*this))\n }\n }\n}\n\nimpl Arc<[T]> {\n /// Constructs a new atomically reference-counted slice with uninitialized contents.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let mut values = Arc::<[u32]>::new_uninit_slice(3);\n ///\n /// // Deferred initialization:\n /// let data = Arc::get_mut(&mut values).unwrap();\n /// data[0].write(1);\n /// data[1].write(2);\n /// data[2].write(3);\n ///\n /// let values = unsafe { values.assume_init() };\n ///\n /// assert_eq!(*values, [1, 2, 3])\n /// ```\n #[inline]\n #[must_use]\n pub fn new_uninit_slice(len: usize) -> Arc<[mem::MaybeUninit]> {\n unsafe { Arc::from_ptr(Arc::allocate_for_slice(len)) }\n }\n\n /// Constructs a new atomically reference-counted slice with uninitialized contents, with the memory being\n /// filled with `0` bytes.\n ///\n /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and\n /// incorrect usage of this method.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let values = Arc::<[u32]>::new_zeroed_slice(3);\n /// let values = unsafe { values.assume_init() };\n ///\n /// assert_eq!(*values, [0, 0, 0])\n /// ```\n ///\n /// [zeroed]: mem::MaybeUninit::zeroed\n #[inline]\n #[must_use]\n #[allow(clippy::missing_panics_doc)]\n pub fn new_zeroed_slice(len: usize) -> Arc<[mem::MaybeUninit]> {\n unsafe {\n Arc::from_ptr(Arc::allocate_for_layout(\n layout::array::(len).unwrap(),\n |layout| Global.allocate_zeroed(layout),\n |mem| {\n // We create a slice just for metadata (we must use `[mem::MaybeUninit]`\n // instead of `[T]` because values behind `mem` is not valid initialized `T`s.\n // there is no size/alignment issue thanks to layout::array), and create a\n // pointer from `mem` and slice's metadata.\n //\n // We cannot use other ways here:\n // - ptr::slice_from_raw_parts_mut is best way here, but requires Rust 1.42.\n // - We cannot use slice::from_raw_parts_mut then casting to its pointer to\n // ArcInner due to provenance because the actual size of valid allocation\n // behind `mem` is `layout.size()` bytes (counters + values + padding) but the\n // allocation from the pointer from slice::from_raw_parts_mut only valid for\n // `size_of:: * len` bytes (only values).\n let meta: *const _ =\n slice::from_raw_parts(mem as *const mem::MaybeUninit, len);\n strict::with_metadata_of(mem, meta) as *mut ArcInner<[mem::MaybeUninit]>\n },\n ))\n }\n }\n}\n\nimpl Arc> {\n /// Converts to `Arc`.\n ///\n /// # Safety\n ///\n /// As with [`MaybeUninit::assume_init`],\n /// it is up to the caller to guarantee that the inner value\n /// really is in an initialized state.\n /// Calling this when the content is not yet fully initialized\n /// causes immediate undefined behavior.\n ///\n /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let mut five = Arc::::new_uninit();\n ///\n /// // Deferred initialization:\n /// Arc::get_mut(&mut five).unwrap().write(5);\n ///\n /// let five = unsafe { five.assume_init() };\n ///\n /// assert_eq!(*five, 5)\n /// ```\n #[inline]\n #[must_use = \"`self` will be dropped if the result is not used\"]\n pub unsafe fn assume_init(self) -> Arc {\n let ptr = Arc::into_inner_non_null(self);\n // SAFETY: MaybeUninit has the same layout as T, and\n // the caller must guarantee that the data is initialized.\n unsafe { Arc::from_inner(ptr.cast::>()) }\n }\n}\n\nimpl Arc {\n fn clone_from_ref(value: &T) -> Self {\n // `in_progress` drops the allocation if we panic before finishing initializing it.\n let mut in_progress: UniqueArcUninit = UniqueArcUninit::new(value);\n\n // Initialize with clone of value.\n unsafe {\n // Clone. If the clone panics, `in_progress` will be dropped and clean up.\n value.clone_to_uninit(in_progress.data_ptr() as *mut u8);\n // Cast type of pointer, now that it is initialized.\n in_progress.into_arc()\n }\n }\n}\n\nimpl Arc<[mem::MaybeUninit]> {\n /// Converts to `Arc<[T]>`.\n ///\n /// # Safety\n ///\n /// As with [`MaybeUninit::assume_init`],\n /// it is up to the caller to guarantee that the inner value\n /// really is in an initialized state.\n /// Calling this when the content is not yet fully initialized\n /// causes immediate undefined behavior.\n ///\n /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let mut values = Arc::<[u32]>::new_uninit_slice(3);\n ///\n /// // Deferred initialization:\n /// let data = Arc::get_mut(&mut values).unwrap();\n /// data[0].write(1);\n /// data[1].write(2);\n /// data[2].write(3);\n ///\n /// let values = unsafe { values.assume_init() };\n ///\n /// assert_eq!(*values, [1, 2, 3])\n /// ```\n #[inline]\n #[must_use = \"`self` will be dropped if the result is not used\"]\n pub unsafe fn assume_init(self) -> Arc<[T]> {\n let ptr = Arc::into_inner_non_null(self);\n // SAFETY: [MaybeUninit] has the same layout as [T], and\n // the caller must guarantee that the data is initialized.\n unsafe { Arc::from_ptr(ptr.as_ptr() as *mut ArcInner<[T]>) }\n }\n}\n\nimpl Arc {\n /// Constructs an `Arc` from a raw pointer.\n ///\n /// # Safety\n ///\n /// The raw pointer must have been previously returned by a call to\n /// [`Arc::into_raw`][into_raw] with the following requirements:\n ///\n /// * If `U` is sized, it must have the same size and alignment as `T`. This\n /// is trivially true if `U` is `T`.\n /// * If `U` is unsized, its data pointer must have the same size and\n /// alignment as `T`. This is trivially true if `Arc` was constructed\n /// through `Arc` and then converted to `Arc` through an [unsized\n /// coercion].\n ///\n /// Note that if `U` or `U`'s data pointer is not `T` but has the same size\n /// and alignment, this is basically like transmuting references of\n /// different types. See [`mem::transmute`] for more information\n /// on what restrictions apply in this case.\n ///\n /// The raw pointer must point to a block of memory allocated by the global allocator.\n ///\n /// The user of `from_raw` has to make sure a specific value of `T` is only\n /// dropped once.\n ///\n /// This function is unsafe because improper use may lead to memory unsafety,\n /// even if the returned `Arc` is never accessed.\n ///\n /// [into_raw]: Arc::into_raw\n /// [unsized coercion]: https://doc.rust-lang.org/reference/type-coercions.html#unsized-coercions\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let x = Arc::new(\"hello\".to_owned());\n /// let x_ptr = Arc::into_raw(x);\n ///\n /// unsafe {\n /// // Convert back to an `Arc` to prevent leak.\n /// let x = Arc::from_raw(x_ptr);\n /// assert_eq!(&*x, \"hello\");\n ///\n /// // Further calls to `Arc::from_raw(x_ptr)` would be memory-unsafe.\n /// }\n ///\n /// // The memory was freed when `x` went out of scope above, so `x_ptr` is now dangling!\n /// ```\n ///\n /// Convert a slice back into its original array:\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let x: Arc<[u32]> = Arc::from([1, 2, 3]);\n /// let x_ptr: *const [u32] = Arc::into_raw(x);\n ///\n /// unsafe {\n /// let x: Arc<[u32; 3]> = Arc::from_raw(x_ptr.cast::<[u32; 3]>());\n /// assert_eq!(&*x, &[1, 2, 3]);\n /// }\n /// ```\n #[inline]\n pub unsafe fn from_raw(ptr: *const T) -> Self {\n unsafe {\n let offset = data_offset::(&*ptr);\n\n // Reverse the offset to find the original ArcInner.\n let arc_ptr = strict::byte_sub(ptr as *mut T, offset) as *mut ArcInner;\n\n Self::from_ptr(arc_ptr)\n }\n }\n\n /// Consumes the `Arc`, returning the wrapped pointer.\n ///\n /// To avoid a memory leak the pointer must be converted back to an `Arc` using\n /// [`Arc::from_raw`].\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let x = Arc::new(\"hello\".to_owned());\n /// let x_ptr = Arc::into_raw(x);\n /// assert_eq!(unsafe { &*x_ptr }, \"hello\");\n /// # // Prevent leaks for Miri.\n /// # drop(unsafe { Arc::from_raw(x_ptr) });\n /// ```\n #[must_use = \"losing the pointer will leak memory\"]\n pub fn into_raw(this: Self) -> *const T {\n let this = ManuallyDrop::new(this);\n Self::as_ptr(&*this)\n }\n\n /// Increments the strong reference count on the `Arc` associated with the\n /// provided pointer by one.\n ///\n /// # Safety\n ///\n /// The pointer must have been obtained through `Arc::into_raw` and must satisfy the\n /// same layout requirements specified in [`Arc::from_raw`].\n /// The associated `Arc` instance must be valid (i.e. the strong count must be at\n /// least 1) for the duration of this method, and `ptr` must point to a block of memory\n /// allocated by the global allocator.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// unsafe {\n /// let ptr = Arc::into_raw(five);\n /// Arc::increment_strong_count(ptr);\n ///\n /// // This assertion is deterministic because we haven't shared\n /// // the `Arc` between threads.\n /// let five = Arc::from_raw(ptr);\n /// assert_eq!(2, Arc::strong_count(&five));\n /// # // Prevent leaks for Miri.\n /// # Arc::decrement_strong_count(ptr);\n /// }\n /// ```\n #[inline]\n pub unsafe fn increment_strong_count(ptr: *const T) {\n // Retain Arc, but don't touch refcount by wrapping in ManuallyDrop\n let arc = unsafe { mem::ManuallyDrop::new(Self::from_raw(ptr)) };\n // Now increase refcount, but don't drop new refcount either\n let _arc_clone: mem::ManuallyDrop<_> = arc.clone();\n }\n\n /// Decrements the strong reference count on the `Arc` associated with the\n /// provided pointer by one.\n ///\n /// # Safety\n ///\n /// The pointer must have been obtained through `Arc::into_raw` and must satisfy the\n /// same layout requirements specified in [`Arc::from_raw`].\n /// The associated `Arc` instance must be valid (i.e. the strong count must be at\n /// least 1) when invoking this method, and `ptr` must point to a block of memory\n /// allocated by the global allocator. This method can be used to release the final\n /// `Arc` and backing storage, but **should not** be called after the final `Arc` has been\n /// released.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// unsafe {\n /// let ptr = Arc::into_raw(five);\n /// Arc::increment_strong_count(ptr);\n ///\n /// // Those assertions are deterministic because we haven't shared\n /// // the `Arc` between threads.\n /// let five = Arc::from_raw(ptr);\n /// assert_eq!(2, Arc::strong_count(&five));\n /// Arc::decrement_strong_count(ptr);\n /// assert_eq!(1, Arc::strong_count(&five));\n /// }\n /// ```\n #[inline]\n pub unsafe fn decrement_strong_count(ptr: *const T) {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { drop(Self::from_raw(ptr)) }\n }\n\n /// Provides a raw pointer to the data.\n ///\n /// The counts are not affected in any way and the `Arc` is not consumed. The pointer is valid for\n /// as long as there are strong counts in the `Arc`.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let x = Arc::new(\"hello\".to_owned());\n /// let y = Arc::clone(&x);\n /// let x_ptr = Arc::as_ptr(&x);\n /// assert_eq!(x_ptr, Arc::as_ptr(&y));\n /// assert_eq!(unsafe { &*x_ptr }, \"hello\");\n /// ```\n #[must_use]\n pub fn as_ptr(this: &Self) -> *const T {\n let ptr: *mut ArcInner = NonNull::as_ptr(this.ptr);\n\n // SAFETY: This cannot go through Deref::deref or ArcInnerPtr::inner because\n // this is required to retain raw/mut provenance such that e.g. `get_mut` can\n // write through the pointer after the Arc is recovered through `from_raw`.\n unsafe { data_ptr::(ptr, &**this) }\n }\n\n /// Creates a new [`Weak`] pointer to this allocation.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// let weak_five = Arc::downgrade(&five);\n /// ```\n #[must_use = \"this returns a new `Weak` pointer, \\\n without modifying the original `Arc`\"]\n #[allow(clippy::missing_panics_doc)]\n pub fn downgrade(this: &Self) -> Weak {\n // This Relaxed is OK because we're checking the value in the CAS\n // below.\n let mut cur = this.inner().weak.load(Relaxed);\n\n loop {\n // check if the weak counter is currently \"locked\"; if so, spin.\n if cur == USIZE_MAX {\n hint::spin_loop();\n cur = this.inner().weak.load(Relaxed);\n continue;\n }\n\n // We can't allow the refcount to increase much past `MAX_REFCOUNT`.\n assert!(cur <= MAX_REFCOUNT, \"{}\", INTERNAL_OVERFLOW_ERROR);\n\n // NOTE: this code currently ignores the possibility of overflow\n // into usize::MAX; in general both Rc and Arc need to be adjusted\n // to deal with overflow.\n\n // Unlike with Clone(), we need this to be an Acquire read to\n // synchronize with the write coming from `is_unique`, so that the\n // events prior to that write happen before this read.\n match this.inner().weak.compare_exchange_weak(cur, cur + 1, Acquire, Relaxed) {\n Ok(_) => {\n // Make sure we do not create a dangling Weak\n debug_assert!(!is_dangling(this.ptr.as_ptr()));\n return Weak { ptr: this.ptr };\n }\n Err(old) => cur = old,\n }\n }\n }\n\n /// Gets the number of [`Weak`] pointers to this allocation.\n ///\n /// # Safety\n ///\n /// This method by itself is safe, but using it correctly requires extra care.\n /// Another thread can change the weak count at any time,\n /// including potentially between calling this method and acting on the result.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n /// let _weak_five = Arc::downgrade(&five);\n ///\n /// // This assertion is deterministic because we haven't shared\n /// // the `Arc` or `Weak` between threads.\n /// assert_eq!(1, Arc::weak_count(&five));\n /// ```\n #[inline]\n #[must_use]\n pub fn weak_count(this: &Self) -> usize {\n let cnt = this.inner().weak.load(Relaxed);\n // If the weak count is currently locked, the value of the\n // count was 0 just before taking the lock.\n if cnt == USIZE_MAX { 0 } else { cnt - 1 }\n }\n\n /// Gets the number of strong (`Arc`) pointers to this allocation.\n ///\n /// # Safety\n ///\n /// This method by itself is safe, but using it correctly requires extra care.\n /// Another thread can change the strong count at any time,\n /// including potentially between calling this method and acting on the result.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n /// let _also_five = Arc::clone(&five);\n ///\n /// // This assertion is deterministic because we haven't shared\n /// // the `Arc` between threads.\n /// assert_eq!(2, Arc::strong_count(&five));\n /// ```\n #[inline]\n #[must_use]\n pub fn strong_count(this: &Self) -> usize {\n this.inner().strong.load(Relaxed)\n }\n\n #[inline]\n fn inner(&self) -> &ArcInner {\n // This unsafety is ok because while this arc is alive we're guaranteed\n // that the inner pointer is valid. Furthermore, we know that the\n // `ArcInner` structure itself is `Sync` because the inner data is\n // `Sync` as well, so we're ok loaning out an immutable pointer to these\n // contents.\n unsafe { self.ptr.as_ref() }\n }\n\n // Non-inlined part of `drop`.\n #[inline(never)]\n unsafe fn drop_slow(&mut self) {\n // Drop the weak ref collectively held by all strong references when this\n // variable goes out of scope. This ensures that the memory is deallocated\n // even if the destructor of `T` panics.\n // Take a reference to `self.alloc` instead of cloning because 1. it'll last long\n // enough, and 2. you should be able to drop `Arc`s with unclonable allocators\n let _weak = Weak { ptr: self.ptr };\n\n // Destroy the data at this time, even though we must not free the box\n // allocation itself (there might still be weak pointers lying around).\n // We cannot use `get_mut_unchecked` here, because `self.alloc` is borrowed.\n unsafe { ptr::drop_in_place(&mut (*self.ptr.as_ptr()).data) }\n }\n\n /// Returns `true` if the two `Arc`s point to the same allocation in a vein similar to\n /// [`ptr::eq`]. This function ignores the metadata of `dyn Trait` pointers.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n /// let same_five = Arc::clone(&five);\n /// let other_five = Arc::new(5);\n ///\n /// assert!(Arc::ptr_eq(&five, &same_five));\n /// assert!(!Arc::ptr_eq(&five, &other_five));\n /// ```\n ///\n /// [`ptr::eq`]: core::ptr::eq \"ptr::eq\"\n #[inline]\n #[must_use]\n pub fn ptr_eq(this: &Self, other: &Self) -> bool {\n ptr::eq(this.ptr.as_ptr() as *const (), other.ptr.as_ptr() as *const ())\n }\n}\n\nimpl Arc {\n /// Allocates an `ArcInner` with sufficient space for\n /// a possibly-unsized inner value where the value has the layout provided.\n ///\n /// The function `mem_to_arc_inner` is called with the data pointer\n /// and must return back a (potentially fat)-pointer for the `ArcInner`.\n unsafe fn allocate_for_layout(\n value_layout: Layout,\n allocate: impl FnOnce(Layout) -> Option>,\n mem_to_arc_inner: impl FnOnce(*mut u8) -> *mut ArcInner,\n ) -> *mut ArcInner {\n let layout = arc_inner_layout_for_value_layout(value_layout);\n\n let ptr = allocate(layout).unwrap_or_else(|| handle_alloc_error(layout));\n\n unsafe { Self::initialize_arc_inner(ptr, layout, mem_to_arc_inner) }\n }\n\n unsafe fn initialize_arc_inner(\n ptr: NonNull,\n _layout: Layout,\n mem_to_arc_inner: impl FnOnce(*mut u8) -> *mut ArcInner,\n ) -> *mut ArcInner {\n let inner: *mut ArcInner = mem_to_arc_inner(ptr.as_ptr());\n // debug_assert_eq!(unsafe { Layout::for_value_raw(inner) }, layout); // for_value_raw is unstable\n\n // SAFETY: mem_to_arc_inner return a valid pointer to uninitialized ArcInner.\n // ArcInner is repr(C), and strong and weak are the first and second fields and\n // are the same type, so `inner as *mut atomic::AtomicUsize` is strong and\n // `(inner as *mut atomic::AtomicUsize).add(1)` is weak.\n unsafe {\n let strong = inner as *mut atomic::AtomicUsize;\n strong.write(atomic::AtomicUsize::new(1));\n let weak = strong.add(1);\n weak.write(atomic::AtomicUsize::new(1));\n }\n\n inner\n }\n}\n\nimpl Arc {\n /// Allocates an `ArcInner` with sufficient space for an unsized inner value.\n #[inline]\n unsafe fn allocate_for_value(value: &T) -> *mut ArcInner {\n let ptr: *const T = value;\n // Allocate for the `ArcInner` using the given value.\n unsafe {\n Self::allocate_for_layout(\n Layout::for_value(value),\n |layout| Global.allocate(layout),\n |mem| strict::with_metadata_of(mem, ptr as *const ArcInner),\n )\n }\n }\n\n fn from_box(src: Box) -> Arc {\n unsafe {\n let value_size = mem::size_of_val(&*src);\n let ptr = Self::allocate_for_value(&*src);\n\n // Copy value as bytes\n ptr::copy_nonoverlapping(\n &*src as *const T as *const u8,\n data_ptr::(ptr, &*src) as *mut u8,\n value_size,\n );\n\n // Free the allocation without dropping its contents\n let box_ptr = Box::into_raw(src);\n let src = Box::from_raw(box_ptr as *mut mem::ManuallyDrop);\n drop(src);\n\n Self::from_ptr(ptr)\n }\n }\n}\n\nimpl Arc<[T]> {\n /// Allocates an `ArcInner<[mem::MaybeUninit]>` with the given length.\n unsafe fn allocate_for_slice(len: usize) -> *mut ArcInner<[mem::MaybeUninit]> {\n unsafe {\n Arc::allocate_for_layout(\n layout::array::(len).unwrap(),\n |layout| Global.allocate(layout),\n |mem| {\n // We create a slice just for metadata (we must use `[mem::MaybeUninit]`\n // instead of `[T]` because values behind `mem` is not valid initialized `T`s.\n // there is no size/alignment issue thanks to layout::array), and create a\n // pointer from `mem` and slice's metadata.\n //\n // We cannot use other ways here:\n // - ptr::slice_from_raw_parts_mut is best way here, but requires Rust 1.42.\n // - We cannot use slice::from_raw_parts_mut then casting to its pointer to\n // ArcInner due to provenance because the actual size of valid allocation\n // behind `mem` is `layout.size()` bytes (counters + values + padding) but the\n // allocation from the pointer from slice::from_raw_parts_mut only valid for\n // `size_of:: * len` bytes (only values).\n let meta: *const _ =\n slice::from_raw_parts(mem as *const mem::MaybeUninit, len);\n strict::with_metadata_of(mem, meta) as *mut ArcInner<[mem::MaybeUninit]>\n },\n )\n }\n }\n\n /// Constructs an `Arc<[T]>` from an iterator known to be of a certain size.\n ///\n /// Behavior is undefined should the size be wrong.\n unsafe fn from_iter_exact(iter: impl Iterator, len: usize) -> Self {\n // Panic guard while cloning T elements.\n // In the event of a panic, elements that have been written\n // into the new ArcInner will be dropped, then the memory freed.\n struct Guard {\n mem: NonNull,\n elems: *mut T,\n layout: Layout,\n n_elems: usize,\n }\n\n impl Drop for Guard {\n fn drop(&mut self) {\n unsafe {\n let slice = slice::from_raw_parts_mut(self.elems, self.n_elems);\n ptr::drop_in_place(slice);\n\n Global.deallocate(self.mem, self.layout);\n }\n }\n }\n\n unsafe {\n let ptr: *mut ArcInner<[mem::MaybeUninit]> = Arc::allocate_for_slice(len);\n\n let mem = ptr as *mut _ as *mut u8;\n let layout = Layout::for_value(&*ptr);\n\n // Pointer to first element\n let elems = (*ptr).data.as_mut_ptr() as *mut T;\n\n let mut guard = Guard { mem: NonNull::new_unchecked(mem), elems, layout, n_elems: 0 };\n\n for (i, item) in iter.enumerate() {\n ptr::write(elems.add(i), item);\n guard.n_elems += 1;\n }\n\n // All clear. Forget the guard so it doesn't free the new ArcInner.\n mem::forget(guard);\n\n Arc::from_ptr(ptr).assume_init()\n }\n }\n}\n\nimpl Clone for Arc {\n /// Makes a clone of the `Arc` pointer.\n ///\n /// This creates another pointer to the same allocation, increasing the\n /// strong reference count.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// let _ = Arc::clone(&five);\n /// ```\n #[inline]\n fn clone(&self) -> Self {\n // Using a relaxed ordering is alright here, as knowledge of the\n // original reference prevents other threads from erroneously deleting\n // the object.\n //\n // As explained in the [Boost documentation][1], Increasing the\n // reference counter can always be done with memory_order_relaxed: New\n // references to an object can only be formed from an existing\n // reference, and passing an existing reference from one thread to\n // another must already provide any required synchronization.\n //\n // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)\n let old_size = self.inner().strong.fetch_add(1, Relaxed);\n\n // However we need to guard against massive refcounts in case someone is `mem::forget`ing\n // Arcs. If we don't do this the count can overflow and users will use-after free. This\n // branch will never be taken in any realistic program. We abort because such a program is\n // incredibly degenerate, and we don't care to support it.\n //\n // This check is not 100% water-proof: we error when the refcount grows beyond `isize::MAX`.\n // But we do that check *after* having done the increment, so there is a chance here that\n // the worst already happened and we actually do overflow the `usize` counter. However, that\n // requires the counter to grow from `isize::MAX` to `usize::MAX` between the increment\n // above and the `abort` below, which seems exceedingly unlikely.\n //\n // This is a global invariant, and also applies when using a compare-exchange loop to increment\n // counters in other methods.\n // Otherwise, the counter could be brought to an almost-overflow using a compare-exchange loop,\n // and then overflow using a few `fetch_add`s.\n if old_size > MAX_REFCOUNT {\n abort();\n }\n\n unsafe { Self::from_inner(self.ptr) }\n }\n}\n\nimpl Deref for Arc {\n type Target = T;\n\n #[inline]\n fn deref(&self) -> &Self::Target {\n &self.inner().data\n }\n}\n\nimpl Arc {\n /// Makes a mutable reference into the given `Arc`.\n ///\n /// If there are other `Arc` pointers to the same allocation, then `make_mut` will\n /// [`clone`] the inner value to a new allocation to ensure unique ownership. This is also\n /// referred to as clone-on-write.\n ///\n /// However, if there are no other `Arc` pointers to this allocation, but some [`Weak`]\n /// pointers, then the [`Weak`] pointers will be dissociated and the inner value will not\n /// be cloned.\n ///\n /// See also [`get_mut`], which will fail rather than cloning the inner value\n /// or dissociating [`Weak`] pointers.\n ///\n /// [`clone`]: Clone::clone\n /// [`get_mut`]: Arc::get_mut\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let mut data = Arc::new(5);\n ///\n /// *Arc::make_mut(&mut data) += 1; // Won't clone anything\n /// let mut other_data = Arc::clone(&data); // Won't clone inner data\n /// *Arc::make_mut(&mut data) += 1; // Clones inner data\n /// *Arc::make_mut(&mut data) += 1; // Won't clone anything\n /// *Arc::make_mut(&mut other_data) *= 2; // Won't clone anything\n ///\n /// // Now `data` and `other_data` point to different allocations.\n /// assert_eq!(*data, 8);\n /// assert_eq!(*other_data, 12);\n /// ```\n ///\n /// [`Weak`] pointers will be dissociated:\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let mut data = Arc::new(75);\n /// let weak = Arc::downgrade(&data);\n ///\n /// assert_eq!(75, *data);\n /// assert_eq!(75, *weak.upgrade().unwrap());\n ///\n /// *Arc::make_mut(&mut data) += 1;\n ///\n /// assert_eq!(76, *data);\n /// assert!(weak.upgrade().is_none());\n /// ```\n #[inline]\n pub fn make_mut(this: &mut Self) -> &mut T {\n let size_of_val = mem::size_of_val::(&**this);\n\n // Note that we hold both a strong reference and a weak reference.\n // Thus, releasing our strong reference only will not, by itself, cause\n // the memory to be deallocated.\n //\n // Use Acquire to ensure that we see any writes to `weak` that happen\n // before release writes (i.e., decrements) to `strong`. Since we hold a\n // weak count, there's no chance the ArcInner itself could be\n // deallocated.\n if this.inner().strong.compare_exchange(1, 0, Acquire, Relaxed).is_err() {\n // Another strong pointer exists, so we must clone.\n *this = Arc::clone_from_ref(&**this);\n } else if this.inner().weak.load(Relaxed) != 1 {\n // Relaxed suffices in the above because this is fundamentally an\n // optimization: we are always racing with weak pointers being\n // dropped. Worst case, we end up allocated a new Arc unnecessarily.\n\n // We removed the last strong ref, but there are additional weak\n // refs remaining. We'll move the contents to a new Arc, and\n // invalidate the other weak refs.\n\n // Note that it is not possible for the read of `weak` to yield\n // usize::MAX (i.e., locked), since the weak count can only be\n // locked by a thread with a strong reference.\n\n // Materialize our own implicit weak pointer, so that it can clean\n // up the ArcInner as needed.\n let _weak = Weak { ptr: this.ptr };\n\n // Can just steal the data, all that's left is `Weak`s\n //\n // We don't need panic-protection like the above branch does, but we might as well\n // use the same mechanism.\n let mut in_progress: UniqueArcUninit = UniqueArcUninit::new(&**this);\n unsafe {\n // Initialize `in_progress` with move of **this.\n // We have to express this in terms of bytes because `T: ?Sized`; there is no\n // operation that just copies a value based on its `size_of_val()`.\n ptr::copy_nonoverlapping(\n &**this as *const T as *const u8,\n in_progress.data_ptr() as *mut u8,\n size_of_val,\n );\n\n ptr::write(this, in_progress.into_arc());\n }\n } else {\n // We were the sole reference of either kind; bump back up the\n // strong ref count.\n this.inner().strong.store(1, Release);\n }\n\n // As with `get_mut()`, the unsafety is ok because our reference was\n // either unique to begin with, or became one upon cloning the contents.\n unsafe { Self::get_mut_unchecked(this) }\n }\n}\n\nimpl Arc {\n /// If we have the only reference to `T` then unwrap it. Otherwise, clone `T` and return the\n /// clone.\n ///\n /// Assuming `arc_t` is of type `Arc`, this function is functionally equivalent to\n /// `(*arc_t).clone()`, but will avoid cloning the inner value where possible.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::ptr;\n ///\n /// use portable_atomic_util::Arc;\n ///\n /// let inner = String::from(\"test\");\n /// let ptr = inner.as_ptr();\n ///\n /// let arc = Arc::new(inner);\n /// let inner = Arc::unwrap_or_clone(arc);\n /// // The inner value was not cloned\n /// assert!(ptr::eq(ptr, inner.as_ptr()));\n ///\n /// let arc = Arc::new(inner);\n /// let arc2 = arc.clone();\n /// let inner = Arc::unwrap_or_clone(arc);\n /// // Because there were 2 references, we had to clone the inner value.\n /// assert!(!ptr::eq(ptr, inner.as_ptr()));\n /// // `arc2` is the last reference, so when we unwrap it we get back\n /// // the original `String`.\n /// let inner = Arc::unwrap_or_clone(arc2);\n /// assert!(ptr::eq(ptr, inner.as_ptr()));\n /// ```\n #[inline]\n pub fn unwrap_or_clone(this: Self) -> T {\n Self::try_unwrap(this).unwrap_or_else(|arc| (*arc).clone())\n }\n}\n\nimpl Arc {\n /// Returns a mutable reference into the given `Arc`, if there are\n /// no other `Arc` or [`Weak`] pointers to the same allocation.\n ///\n /// Returns [`None`] otherwise, because it is not safe to\n /// mutate a shared value.\n ///\n /// See also [`make_mut`][make_mut], which will [`clone`][clone]\n /// the inner value when there are other `Arc` pointers.\n ///\n /// [make_mut]: Arc::make_mut\n /// [clone]: Clone::clone\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let mut x = Arc::new(3);\n /// *Arc::get_mut(&mut x).unwrap() = 4;\n /// assert_eq!(*x, 4);\n ///\n /// let _y = Arc::clone(&x);\n /// assert!(Arc::get_mut(&mut x).is_none());\n /// ```\n #[inline]\n pub fn get_mut(this: &mut Self) -> Option<&mut T> {\n if Self::is_unique(this) {\n // This unsafety is ok because we're guaranteed that the pointer\n // returned is the *only* pointer that will ever be returned to T. Our\n // reference count is guaranteed to be 1 at this point, and we required\n // the Arc itself to be `mut`, so we're returning the only possible\n // reference to the inner data.\n unsafe { Some(Self::get_mut_unchecked(this)) }\n } else {\n None\n }\n }\n\n #[inline]\n unsafe fn get_mut_unchecked(this: &mut Self) -> &mut T {\n // We are careful to *not* create a reference covering the \"count\" fields, as\n // this would alias with concurrent access to the reference counts (e.g. by `Weak`).\n unsafe { &mut (*this.ptr.as_ptr()).data }\n }\n\n #[inline]\n fn is_unique(this: &Self) -> bool {\n // lock the weak pointer count if we appear to be the sole weak pointer\n // holder.\n //\n // The acquire label here ensures a happens-before relationship with any\n // writes to `strong` (in particular in `Weak::upgrade`) prior to decrements\n // of the `weak` count (via `Weak::drop`, which uses release). If the upgraded\n // weak ref was never dropped, the CAS here will fail so we do not care to synchronize.\n if this.inner().weak.compare_exchange(1, USIZE_MAX, Acquire, Relaxed).is_ok() {\n // This needs to be an `Acquire` to synchronize with the decrement of the `strong`\n // counter in `drop` -- the only access that happens when any but the last reference\n // is being dropped.\n let unique = this.inner().strong.load(Acquire) == 1;\n\n // The release write here synchronizes with a read in `downgrade`,\n // effectively preventing the above read of `strong` from happening\n // after the write.\n this.inner().weak.store(1, Release); // release the lock\n unique\n } else {\n false\n }\n }\n}\n\nimpl Drop for Arc {\n /// Drops the `Arc`.\n ///\n /// This will decrement the strong reference count. If the strong reference\n /// count reaches zero then the only other references (if any) are\n /// [`Weak`], so we `drop` the inner value.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// struct Foo;\n ///\n /// impl Drop for Foo {\n /// fn drop(&mut self) {\n /// println!(\"dropped!\");\n /// }\n /// }\n ///\n /// let foo = Arc::new(Foo);\n /// let foo2 = Arc::clone(&foo);\n ///\n /// drop(foo); // Doesn't print anything\n /// drop(foo2); // Prints \"dropped!\"\n /// ```\n #[inline]\n fn drop(&mut self) {\n // Because `fetch_sub` is already atomic, we do not need to synchronize\n // with other threads unless we are going to delete the object. This\n // same logic applies to the below `fetch_sub` to the `weak` count.\n if self.inner().strong.fetch_sub(1, Release) != 1 {\n return;\n }\n\n // This fence is needed to prevent reordering of use of the data and\n // deletion of the data. Because it is marked `Release`, the decreasing\n // of the reference count synchronizes with this `Acquire` fence. This\n // means that use of the data happens before decreasing the reference\n // count, which happens before this fence, which happens before the\n // deletion of the data.\n //\n // As explained in the [Boost documentation][1],\n //\n // > It is important to enforce any possible access to the object in one\n // > thread (through an existing reference) to *happen before* deleting\n // > the object in a different thread. This is achieved by a \"release\"\n // > operation after dropping a reference (any access to the object\n // > through this reference must obviously happened before), and an\n // > \"acquire\" operation before deleting the object.\n //\n // In particular, while the contents of an Arc are usually immutable, it's\n // possible to have interior writes to something like a Mutex. Since a\n // Mutex is not acquired when it is deleted, we can't rely on its\n // synchronization logic to make writes in thread A visible to a destructor\n // running in thread B.\n //\n // Also note that the Acquire fence here could probably be replaced with an\n // Acquire load, which could improve performance in highly-contended\n // situations. See [2].\n //\n // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)\n // [2]: (https://github.com/rust-lang/rust/pull/41714)\n acquire!(self.inner().strong);\n\n unsafe {\n self.drop_slow();\n }\n }\n}\n\nimpl Arc {\n /// Attempts to downcast the `Arc` to a concrete type.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::any::Any;\n ///\n /// use portable_atomic_util::Arc;\n ///\n /// fn print_if_string(value: Arc) {\n /// if let Ok(string) = value.downcast::() {\n /// println!(\"String ({}): {}\", string.len(), string);\n /// }\n /// }\n ///\n /// let my_string = \"Hello World\".to_string();\n /// print_if_string(Arc::from(Box::new(my_string) as Box));\n /// print_if_string(Arc::from(Box::new(0i8) as Box));\n /// // or with \"--cfg portable_atomic_unstable_coerce_unsized\" in RUSTFLAGS (requires Rust nightly):\n /// // print_if_string(Arc::new(my_string));\n /// // print_if_string(Arc::new(0i8));\n /// ```\n #[inline]\n pub fn downcast(self) -> Result, Self>\n where\n T: Any + Send + Sync,\n {\n if (*self).is::() {\n unsafe {\n let ptr = Arc::into_inner_non_null(self);\n Ok(Arc::from_inner(ptr.cast::>()))\n }\n } else {\n Err(self)\n }\n }\n}\n\nimpl Weak {\n /// Constructs a new `Weak`, without allocating any memory.\n /// Calling [`upgrade`] on the return value always gives [`None`].\n ///\n /// [`upgrade`]: Weak::upgrade\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Weak;\n ///\n /// let empty: Weak = Weak::new();\n /// assert!(empty.upgrade().is_none());\n /// ```\n #[inline]\n #[must_use]\n pub const fn new() -> Self {\n Self {\n ptr: unsafe {\n NonNull::new_unchecked(strict::without_provenance_mut::>(USIZE_MAX))\n },\n }\n }\n\n #[inline]\n #[must_use]\n fn new_uninit_ptr() -> NonNull> {\n unsafe {\n NonNull::new_unchecked(Self::allocate_for_layout(\n Layout::new::(),\n |layout| Global.allocate(layout),\n |ptr| ptr as *mut _,\n ))\n }\n }\n}\n\n/// Helper type to allow accessing the reference counts without\n/// making any assertions about the data field.\nstruct WeakInner<'a> {\n weak: &'a atomic::AtomicUsize,\n strong: &'a atomic::AtomicUsize,\n}\n\n// TODO: See todo comment in Weak::from_raw\nimpl Weak {\n /// Converts a raw pointer previously created by [`into_raw`] back into `Weak`.\n ///\n /// This can be used to safely get a strong reference (by calling [`upgrade`]\n /// later) or to deallocate the weak count by dropping the `Weak`.\n ///\n /// It takes ownership of one weak reference (with the exception of pointers created by [`new`],\n /// as these don't own anything; the method still works on them).\n ///\n /// # Safety\n ///\n /// The pointer must have originated from the [`into_raw`] and must still own its potential\n /// weak reference, and must point to a block of memory allocated by global allocator.\n ///\n /// It is allowed for the strong count to be 0 at the time of calling this. Nevertheless, this\n /// takes ownership of one weak reference currently represented as a raw pointer (the weak\n /// count is not modified by this operation) and therefore it must be paired with a previous\n /// call to [`into_raw`].\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::{Arc, Weak};\n ///\n /// let strong = Arc::new(\"hello\".to_owned());\n ///\n /// let raw_1 = Arc::downgrade(&strong).into_raw();\n /// let raw_2 = Arc::downgrade(&strong).into_raw();\n ///\n /// assert_eq!(2, Arc::weak_count(&strong));\n ///\n /// assert_eq!(\"hello\", &*unsafe { Weak::from_raw(raw_1) }.upgrade().unwrap());\n /// assert_eq!(1, Arc::weak_count(&strong));\n ///\n /// drop(strong);\n ///\n /// // Decrement the last weak count.\n /// assert!(unsafe { Weak::from_raw(raw_2) }.upgrade().is_none());\n /// ```\n ///\n /// [`new`]: Weak::new\n /// [`into_raw`]: Weak::into_raw\n /// [`upgrade`]: Weak::upgrade\n #[inline]\n pub unsafe fn from_raw(ptr: *const T) -> Self {\n // See Weak::as_ptr for context on how the input pointer is derived.\n\n let ptr = if is_dangling(ptr) {\n // This is a dangling Weak.\n ptr as *mut ArcInner\n } else {\n // Otherwise, we're guaranteed the pointer came from a non-dangling Weak.\n // TODO: data_offset calls align_of_val which needs to create a reference\n // to data but we cannot create a reference to data here since data in Weak\n // can be dropped concurrently from another thread. Therefore, we can\n // only support sized types that can avoid references to data\n // unless align_of_val_raw is stabilized.\n // // SAFETY: data_offset is safe to call, as ptr references a real (potentially dropped) T.\n // let offset = unsafe { data_offset(ptr) };\n let offset = data_offset_align(mem::align_of::());\n // Thus, we reverse the offset to get the whole ArcInner.\n // SAFETY: the pointer originated from a Weak, so this offset is safe.\n unsafe { strict::byte_sub(ptr as *mut T, offset) as *mut ArcInner }\n };\n\n // SAFETY: we now have recovered the original Weak pointer, so can create the Weak.\n Self { ptr: unsafe { NonNull::new_unchecked(ptr) } }\n }\n\n /// Consumes the `Weak` and turns it into a raw pointer.\n ///\n /// This converts the weak pointer into a raw pointer, while still preserving the ownership of\n /// one weak reference (the weak count is not modified by this operation). It can be turned\n /// back into the `Weak` with [`from_raw`].\n ///\n /// The same restrictions of accessing the target of the pointer as with\n /// [`as_ptr`] apply.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::{Arc, Weak};\n ///\n /// let strong = Arc::new(\"hello\".to_owned());\n /// let weak = Arc::downgrade(&strong);\n /// let raw = weak.into_raw();\n ///\n /// assert_eq!(1, Arc::weak_count(&strong));\n /// assert_eq!(\"hello\", unsafe { &*raw });\n ///\n /// drop(unsafe { Weak::from_raw(raw) });\n /// assert_eq!(0, Arc::weak_count(&strong));\n /// ```\n ///\n /// [`from_raw`]: Weak::from_raw\n /// [`as_ptr`]: Weak::as_ptr\n #[must_use = \"losing the pointer will leak memory\"]\n pub fn into_raw(self) -> *const T {\n ManuallyDrop::new(self).as_ptr()\n }\n}\n\n// TODO: See todo comment in Weak::from_raw\nimpl Weak {\n /// Returns a raw pointer to the object `T` pointed to by this `Weak`.\n ///\n /// The pointer is valid only if there are some strong references. The pointer may be dangling,\n /// unaligned or even [`null`] otherwise.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::ptr;\n ///\n /// use portable_atomic_util::Arc;\n ///\n /// let strong = Arc::new(\"hello\".to_owned());\n /// let weak = Arc::downgrade(&strong);\n /// // Both point to the same object\n /// assert!(ptr::eq(&*strong, weak.as_ptr()));\n /// // The strong here keeps it alive, so we can still access the object.\n /// assert_eq!(\"hello\", unsafe { &*weak.as_ptr() });\n ///\n /// drop(strong);\n /// // But not any more. We can do weak.as_ptr(), but accessing the pointer would lead to\n /// // undefined behavior.\n /// // assert_eq!(\"hello\", unsafe { &*weak.as_ptr() });\n /// ```\n ///\n /// [`null`]: core::ptr::null \"ptr::null\"\n #[must_use]\n pub fn as_ptr(&self) -> *const T {\n let ptr: *mut ArcInner = NonNull::as_ptr(self.ptr);\n\n if is_dangling(ptr) {\n // If the pointer is dangling, we return the sentinel directly. This cannot be\n // a valid payload address, as the payload is at least as aligned as ArcInner (usize).\n ptr as *const T\n } else {\n // TODO: See todo comment in Weak::from_raw\n // // SAFETY: if is_dangling returns false, then the pointer is dereferenceable.\n // // The payload may be dropped at this point, and we have to maintain provenance,\n // // so use raw pointer manipulation.\n // unsafe { data_ptr::(ptr, &(*ptr).data) }\n unsafe {\n let offset = data_offset_align(mem::align_of::());\n strict::byte_add(ptr, offset) as *const T\n }\n }\n }\n}\n\nimpl Weak {\n /// Attempts to upgrade the `Weak` pointer to an [`Arc`], delaying\n /// dropping of the inner value if successful.\n ///\n /// Returns [`None`] if the inner value has since been dropped.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// let weak_five = Arc::downgrade(&five);\n ///\n /// let strong_five: Option> = weak_five.upgrade();\n /// assert!(strong_five.is_some());\n ///\n /// // Destroy all strong pointers.\n /// drop(strong_five);\n /// drop(five);\n ///\n /// assert!(weak_five.upgrade().is_none());\n /// ```\n #[must_use = \"this returns a new `Arc`, \\\n without modifying the original weak pointer\"]\n pub fn upgrade(&self) -> Option> {\n #[inline]\n fn checked_increment(n: usize) -> Option {\n // Any write of 0 we can observe leaves the field in permanently zero state.\n if n == 0 {\n return None;\n }\n // See comments in `Arc::clone` for why we do this (for `mem::forget`).\n assert!(n <= MAX_REFCOUNT, \"{}\", INTERNAL_OVERFLOW_ERROR);\n Some(n + 1)\n }\n\n // We use a CAS loop to increment the strong count instead of a\n // fetch_add as this function should never take the reference count\n // from zero to one.\n //\n // Relaxed is fine for the failure case because we don't have any expectations about the new state.\n // Acquire is necessary for the success case to synchronize with `Arc::new_cyclic`, when the inner\n // value can be initialized after `Weak` references have already been created. In that case, we\n // expect to observe the fully initialized value.\n if self.inner()?.strong.fetch_update(Acquire, Relaxed, checked_increment).is_ok() {\n // SAFETY: pointer is not null, verified in checked_increment\n unsafe { Some(Arc::from_inner(self.ptr)) }\n } else {\n None\n }\n }\n\n /// Gets the number of strong (`Arc`) pointers pointing to this allocation.\n ///\n /// If `self` was created using [`Weak::new`], this will return 0.\n #[must_use]\n pub fn strong_count(&self) -> usize {\n if let Some(inner) = self.inner() { inner.strong.load(Relaxed) } else { 0 }\n }\n\n /// Gets an approximation of the number of `Weak` pointers pointing to this\n /// allocation.\n ///\n /// If `self` was created using [`Weak::new`], or if there are no remaining\n /// strong pointers, this will return 0.\n ///\n /// # Accuracy\n ///\n /// Due to implementation details, the returned value can be off by 1 in\n /// either direction when other threads are manipulating any `Arc`s or\n /// `Weak`s pointing to the same allocation.\n #[must_use]\n pub fn weak_count(&self) -> usize {\n if let Some(inner) = self.inner() {\n let weak = inner.weak.load(Acquire);\n let strong = inner.strong.load(Relaxed);\n if strong == 0 {\n 0\n } else {\n // Since we observed that there was at least one strong pointer\n // after reading the weak count, we know that the implicit weak\n // reference (present whenever any strong references are alive)\n // was still around when we observed the weak count, and can\n // therefore safely subtract it.\n weak - 1\n }\n } else {\n 0\n }\n }\n\n /// Returns `None` when the pointer is dangling and there is no allocated `ArcInner`,\n /// (i.e., when this `Weak` was created by `Weak::new`).\n #[inline]\n fn inner(&self) -> Option> {\n let ptr = self.ptr.as_ptr();\n if is_dangling(ptr) {\n None\n } else {\n // SAFETY: non-dangling Weak has a valid pointer.\n // We are careful to *not* create a reference covering the \"data\" field, as\n // the field may be mutated concurrently (for example, if the last `Arc`\n // is dropped, the data field will be dropped in-place).\n Some(unsafe { WeakInner { strong: &(*ptr).strong, weak: &(*ptr).weak } })\n }\n }\n\n /// Returns `true` if the two `Weak`s point to the same allocation similar to [`ptr::eq`], or if\n /// both don't point to any allocation (because they were created with `Weak::new()`). However,\n /// this function ignores the metadata of `dyn Trait` pointers.\n ///\n /// # Notes\n ///\n /// Since this compares pointers it means that `Weak::new()` will equal each\n /// other, even though they don't point to any allocation.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let first_rc = Arc::new(5);\n /// let first = Arc::downgrade(&first_rc);\n /// let second = Arc::downgrade(&first_rc);\n ///\n /// assert!(first.ptr_eq(&second));\n ///\n /// let third_rc = Arc::new(5);\n /// let third = Arc::downgrade(&third_rc);\n ///\n /// assert!(!first.ptr_eq(&third));\n /// ```\n ///\n /// Comparing `Weak::new`.\n ///\n /// ```\n /// use portable_atomic_util::{Arc, Weak};\n ///\n /// let first = Weak::new();\n /// let second = Weak::new();\n /// assert!(first.ptr_eq(&second));\n ///\n /// let third_rc = Arc::new(());\n /// let third = Arc::downgrade(&third_rc);\n /// assert!(!first.ptr_eq(&third));\n /// ```\n ///\n /// [`ptr::eq`]: core::ptr::eq \"ptr::eq\"\n #[inline]\n #[must_use]\n pub fn ptr_eq(&self, other: &Self) -> bool {\n ptr::eq(self.ptr.as_ptr() as *const (), other.ptr.as_ptr() as *const ())\n }\n}\n\nimpl Weak {\n /// Allocates an `ArcInner` with sufficient space for\n /// a possibly-unsized inner value where the value has the layout provided.\n ///\n /// The function `mem_to_arc_inner` is called with the data pointer\n /// and must return back a (potentially fat)-pointer for the `ArcInner`.\n unsafe fn allocate_for_layout(\n value_layout: Layout,\n allocate: impl FnOnce(Layout) -> Option>,\n mem_to_arc_inner: impl FnOnce(*mut u8) -> *mut ArcInner,\n ) -> *mut ArcInner {\n let layout = arc_inner_layout_for_value_layout(value_layout);\n\n let ptr = allocate(layout).unwrap_or_else(|| handle_alloc_error(layout));\n\n unsafe { Self::initialize_arc_inner(ptr, layout, mem_to_arc_inner) }\n }\n\n unsafe fn initialize_arc_inner(\n ptr: NonNull,\n _layout: Layout,\n mem_to_arc_inner: impl FnOnce(*mut u8) -> *mut ArcInner,\n ) -> *mut ArcInner {\n let inner: *mut ArcInner = mem_to_arc_inner(ptr.as_ptr());\n // debug_assert_eq!(unsafe { Layout::for_value_raw(inner) }, layout); // for_value_raw is unstable\n\n // SAFETY: mem_to_arc_inner return a valid pointer to uninitialized ArcInner.\n // ArcInner is repr(C), and strong and weak are the first and second fields and\n // are the same type, so `inner as *mut atomic::AtomicUsize` is strong and\n // `(inner as *mut atomic::AtomicUsize).add(1)` is weak.\n unsafe {\n let strong = inner as *mut atomic::AtomicUsize;\n strong.write(atomic::AtomicUsize::new(0));\n let weak = strong.add(1);\n weak.write(atomic::AtomicUsize::new(1));\n }\n\n inner\n }\n}\n\nimpl Clone for Weak {\n /// Makes a clone of the `Weak` pointer that points to the same allocation.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::{Arc, Weak};\n ///\n /// let weak_five = Arc::downgrade(&Arc::new(5));\n ///\n /// let _ = Weak::clone(&weak_five);\n /// ```\n #[inline]\n fn clone(&self) -> Self {\n if let Some(inner) = self.inner() {\n // See comments in Arc::clone() for why this is relaxed. This can use a\n // fetch_add (ignoring the lock) because the weak count is only locked\n // where are *no other* weak pointers in existence. (So we can't be\n // running this code in that case).\n let old_size = inner.weak.fetch_add(1, Relaxed);\n\n // See comments in Arc::clone() for why we do this (for mem::forget).\n if old_size > MAX_REFCOUNT {\n abort();\n }\n }\n\n Self { ptr: self.ptr }\n }\n}\n\nimpl Default for Weak {\n /// Constructs a new `Weak`, without allocating memory.\n /// Calling [`upgrade`] on the return value always\n /// gives [`None`].\n ///\n /// [`upgrade`]: Weak::upgrade\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Weak;\n ///\n /// let empty: Weak = Weak::default();\n /// assert!(empty.upgrade().is_none());\n /// ```\n fn default() -> Self {\n Self::new()\n }\n}\n\nimpl Drop for Weak {\n /// Drops the `Weak` pointer.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::{Arc, Weak};\n ///\n /// struct Foo;\n ///\n /// impl Drop for Foo {\n /// fn drop(&mut self) {\n /// println!(\"dropped!\");\n /// }\n /// }\n ///\n /// let foo = Arc::new(Foo);\n /// let weak_foo = Arc::downgrade(&foo);\n /// let other_weak_foo = Weak::clone(&weak_foo);\n ///\n /// drop(weak_foo); // Doesn't print anything\n /// drop(foo); // Prints \"dropped!\"\n ///\n /// assert!(other_weak_foo.upgrade().is_none());\n /// ```\n fn drop(&mut self) {\n // If we find out that we were the last weak pointer, then its time to\n // deallocate the data entirely. See the discussion in Arc::drop() about\n // the memory orderings\n //\n // It's not necessary to check for the locked state here, because the\n // weak count can only be locked if there was precisely one weak ref,\n // meaning that drop could only subsequently run ON that remaining weak\n // ref, which can only happen after the lock is released.\n let inner = if let Some(inner) = self.inner() { inner } else { return };\n\n if inner.weak.fetch_sub(1, Release) == 1 {\n acquire!(inner.weak);\n // Free the allocation without dropping T\n let ptr = self.ptr.as_ptr() as *mut ArcInner>;\n drop(unsafe { Box::from_raw(ptr) });\n }\n }\n}\n\nimpl PartialEq for Arc {\n /// Equality for two `Arc`s.\n ///\n /// Two `Arc`s are equal if their inner values are equal, even if they are\n /// stored in different allocation.\n ///\n /// If `T` also implements `Eq` (implying reflexivity of equality),\n /// two `Arc`s that point to the same allocation are always equal.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// assert_eq!(five, Arc::new(5));\n /// ```\n #[inline]\n fn eq(&self, other: &Self) -> bool {\n **self == **other\n }\n\n /// Inequality for two `Arc`s.\n ///\n /// Two `Arc`s are not equal if their inner values are not equal.\n ///\n /// If `T` also implements `Eq` (implying reflexivity of equality),\n /// two `Arc`s that point to the same value are always equal.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// assert_ne!(five, Arc::new(6));\n /// ```\n #[allow(clippy::partialeq_ne_impl)]\n #[inline]\n fn ne(&self, other: &Self) -> bool {\n **self != **other\n }\n}\n\nimpl PartialOrd for Arc {\n /// Partial comparison for two `Arc`s.\n ///\n /// The two are compared by calling `partial_cmp()` on their inner values.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::cmp::Ordering;\n ///\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// assert_eq!(Some(Ordering::Less), five.partial_cmp(&Arc::new(6)));\n /// ```\n fn partial_cmp(&self, other: &Self) -> Option {\n (**self).partial_cmp(&**other)\n }\n\n /// Less-than comparison for two `Arc`s.\n ///\n /// The two are compared by calling `<` on their inner values.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// assert!(five < Arc::new(6));\n /// ```\n fn lt(&self, other: &Self) -> bool {\n *(*self) < *(*other)\n }\n\n /// 'Less than or equal to' comparison for two `Arc`s.\n ///\n /// The two are compared by calling `<=` on their inner values.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// assert!(five <= Arc::new(5));\n /// ```\n fn le(&self, other: &Self) -> bool {\n *(*self) <= *(*other)\n }\n\n /// Greater-than comparison for two `Arc`s.\n ///\n /// The two are compared by calling `>` on their inner values.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// assert!(five > Arc::new(4));\n /// ```\n fn gt(&self, other: &Self) -> bool {\n *(*self) > *(*other)\n }\n\n /// 'Greater than or equal to' comparison for two `Arc`s.\n ///\n /// The two are compared by calling `>=` on their inner values.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// assert!(five >= Arc::new(5));\n /// ```\n fn ge(&self, other: &Self) -> bool {\n *(*self) >= *(*other)\n }\n}\nimpl Ord for Arc {\n /// Comparison for two `Arc`s.\n ///\n /// The two are compared by calling `cmp()` on their inner values.\n ///\n /// # Examples\n ///\n /// ```\n /// use std::cmp::Ordering;\n ///\n /// use portable_atomic_util::Arc;\n ///\n /// let five = Arc::new(5);\n ///\n /// assert_eq!(Ordering::Less, five.cmp(&Arc::new(6)));\n /// ```\n fn cmp(&self, other: &Self) -> Ordering {\n (**self).cmp(&**other)\n }\n}\nimpl Eq for Arc {}\n\nimpl fmt::Display for Arc {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n fmt::Display::fmt(&**self, f)\n }\n}\n\nimpl fmt::Debug for Arc {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n fmt::Debug::fmt(&**self, f)\n }\n}\n\nimpl fmt::Pointer for Arc {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n fmt::Pointer::fmt(&(&**self as *const T), f)\n }\n}\n\nimpl Default for Arc {\n /// Creates a new `Arc`, with the `Default` value for `T`.\n ///\n /// # Examples\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n ///\n /// let x: Arc = Arc::default();\n /// assert_eq!(*x, 0);\n /// ```\n fn default() -> Self {\n // TODO: https://github.com/rust-lang/rust/pull/131460 / https://github.com/rust-lang/rust/pull/132031\n Self::new(T::default())\n }\n}\n\n#[cfg(not(portable_atomic_no_min_const_generics))]\nimpl Default for Arc {\n /// Creates an empty str inside an Arc.\n ///\n /// This may or may not share an allocation with other Arcs.\n #[inline]\n fn default() -> Self {\n let arc: Arc<[u8]> = Arc::default();\n debug_assert!(core::str::from_utf8(&arc).is_ok());\n let ptr = Arc::into_inner_non_null(arc);\n unsafe { Arc::from_ptr(ptr.as_ptr() as *mut ArcInner) }\n }\n}\n\n#[cfg(not(portable_atomic_no_min_const_generics))]\nimpl Default for Arc<[T]> {\n /// Creates an empty `[T]` inside an Arc.\n ///\n /// This may or may not share an allocation with other Arcs.\n #[inline]\n fn default() -> Self {\n // TODO: we cannot use non-allocation optimization (https://github.com/rust-lang/rust/blob/1.93.0/library/alloc/src/sync.rs#L3807)\n // for now since casting Arc<[T; N]> -> Arc<[T]> requires unstable CoerceUnsized.\n let arr: [T; 0] = [];\n Arc::from(arr)\n }\n}\n\nimpl Default for Pin>\nwhere\n T: ?Sized,\n Arc: Default,\n{\n #[inline]\n fn default() -> Self {\n unsafe { Pin::new_unchecked(Arc::::default()) }\n }\n}\n\nimpl Hash for Arc {\n fn hash(&self, state: &mut H) {\n (**self).hash(state);\n }\n}\n\nimpl From for Arc {\n /// Converts a `T` into an `Arc`\n ///\n /// The conversion moves the value into a\n /// newly allocated `Arc`. It is equivalent to\n /// calling `Arc::new(t)`.\n ///\n /// # Example\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let x = 5;\n /// let arc = Arc::new(5);\n ///\n /// assert_eq!(Arc::from(x), arc);\n /// ```\n fn from(t: T) -> Self {\n Self::new(t)\n }\n}\n\n// This just outputs the input as is, but can be used like an item-level block by using it with cfg.\n// Note: This macro is items!({ }), not items! { }.\n// An extra brace is used in input to make contents rustfmt-able.\n#[cfg(not(portable_atomic_no_min_const_generics))]\nmacro_rules! items {\n ({$($tt:tt)*}) => {\n $($tt)*\n };\n}\n\n#[cfg(not(portable_atomic_no_min_const_generics))]\nitems!({\n impl From<[T; N]> for Arc<[T]> {\n /// Converts a [`[T; N]`](prim@array) into an `Arc<[T]>`.\n ///\n /// The conversion moves the array into a newly allocated `Arc`.\n ///\n /// # Example\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let original: [i32; 3] = [1, 2, 3];\n /// let shared: Arc<[i32]> = Arc::from(original);\n /// assert_eq!(&[1, 2, 3], &shared[..]);\n /// ```\n #[inline]\n fn from(v: [T; N]) -> Self {\n // Casting Arc<[T; N]> -> Arc<[T]> requires unstable CoerceUnsized, so we convert via Box.\n // Since the compiler knows the actual size and metadata, the intermediate allocation is\n // optimized and generates the same code as when using CoerceUnsized and convert Arc<[T; N]> to Arc<[T]>.\n // https://github.com/taiki-e/portable-atomic/issues/143#issuecomment-1866488569\n let v: Box<[T]> = Box::<[T; N]>::from(v);\n v.into()\n }\n }\n});\n\nimpl From<&[T]> for Arc<[T]> {\n /// Allocates a reference-counted slice and fills it by cloning `v`'s items.\n ///\n /// # Example\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let original: &[i32] = &[1, 2, 3];\n /// let shared: Arc<[i32]> = Arc::from(original);\n /// assert_eq!(&[1, 2, 3], &shared[..]);\n /// ```\n #[inline]\n fn from(v: &[T]) -> Self {\n unsafe { Self::from_iter_exact(v.iter().cloned(), v.len()) }\n }\n}\n\nimpl From<&mut [T]> for Arc<[T]> {\n /// Allocates a reference-counted slice and fills it by cloning `v`'s items.\n ///\n /// # Example\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let mut original = [1, 2, 3];\n /// let original: &mut [i32] = &mut original;\n /// let shared: Arc<[i32]> = Arc::from(original);\n /// assert_eq!(&[1, 2, 3], &shared[..]);\n /// ```\n #[inline]\n fn from(v: &mut [T]) -> Self {\n Self::from(&*v)\n }\n}\n\nimpl From<&str> for Arc {\n /// Allocates a reference-counted `str` and copies `v` into it.\n ///\n /// # Example\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let shared: Arc = Arc::from(\"eggplant\");\n /// assert_eq!(\"eggplant\", &shared[..]);\n /// ```\n #[inline]\n fn from(v: &str) -> Self {\n let arc = Arc::<[u8]>::from(v.as_bytes());\n // SAFETY: `str` has the same layout as `[u8]`.\n // https://doc.rust-lang.org/nightly/reference/type-layout.html#str-layout\n unsafe { Self::from_raw(Arc::into_raw(arc) as *const str) }\n }\n}\n\nimpl From<&mut str> for Arc {\n /// Allocates a reference-counted `str` and copies `v` into it.\n ///\n /// # Example\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let mut original = String::from(\"eggplant\");\n /// let original: &mut str = &mut original;\n /// let shared: Arc = Arc::from(original);\n /// assert_eq!(\"eggplant\", &shared[..]);\n /// ```\n #[inline]\n fn from(v: &mut str) -> Self {\n Self::from(&*v)\n }\n}\n\nimpl From for Arc {\n /// Allocates a reference-counted `str` and copies `v` into it.\n ///\n /// # Example\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let unique: String = \"eggplant\".to_owned();\n /// let shared: Arc = Arc::from(unique);\n /// assert_eq!(\"eggplant\", &shared[..]);\n /// ```\n #[inline]\n fn from(v: String) -> Self {\n Self::from(&v[..])\n }\n}\n\nimpl From> for Arc {\n /// Move a boxed object to a new, reference-counted allocation.\n ///\n /// # Example\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let unique: Box = Box::from(\"eggplant\");\n /// let shared: Arc = Arc::from(unique);\n /// assert_eq!(\"eggplant\", &shared[..]);\n /// ```\n #[inline]\n fn from(v: Box) -> Self {\n Self::from_box(v)\n }\n}\n\nimpl From> for Arc<[T]> {\n /// Allocates a reference-counted slice and moves `v`'s items into it.\n ///\n /// # Example\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let unique: Vec = vec![1, 2, 3];\n /// let shared: Arc<[i32]> = Arc::from(unique);\n /// assert_eq!(&[1, 2, 3], &shared[..]);\n /// ```\n #[inline]\n fn from(v: Vec) -> Self {\n unsafe {\n let len = v.len();\n let cap = v.capacity();\n let vec_ptr = mem::ManuallyDrop::new(v).as_mut_ptr();\n\n let mut arc = Self::new_uninit_slice(len);\n let data = Arc::get_mut_unchecked(&mut arc);\n ptr::copy_nonoverlapping(vec_ptr, data.as_mut_ptr() as *mut T, len);\n\n // Create a `Vec` with length 0, to deallocate the buffer\n // without dropping its contents or the allocator\n let _ = Vec::from_raw_parts(vec_ptr, 0, cap);\n\n arc.assume_init()\n }\n }\n}\n\nimpl<'a, B> From> for Arc\nwhere\n B: ?Sized + ToOwned,\n Arc: From<&'a B> + From,\n{\n /// Creates an atomically reference-counted pointer from a clone-on-write\n /// pointer by copying its content.\n ///\n /// # Example\n ///\n /// ```\n /// use std::borrow::Cow;\n ///\n /// use portable_atomic_util::Arc;\n ///\n /// let cow: Cow<'_, str> = Cow::Borrowed(\"eggplant\");\n /// let shared: Arc = Arc::from(cow);\n /// assert_eq!(\"eggplant\", &shared[..]);\n /// ```\n #[inline]\n fn from(cow: Cow<'a, B>) -> Self {\n match cow {\n Cow::Borrowed(s) => Self::from(s),\n Cow::Owned(s) => Self::from(s),\n }\n }\n}\n\nimpl From> for Arc<[u8]> {\n /// Converts an atomically reference-counted string slice into a byte slice.\n ///\n /// # Example\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let string: Arc = Arc::from(\"eggplant\");\n /// let bytes: Arc<[u8]> = Arc::from(string);\n /// assert_eq!(\"eggplant\".as_bytes(), bytes.as_ref());\n /// ```\n #[inline]\n fn from(rc: Arc) -> Self {\n // SAFETY: `str` has the same layout as `[u8]`.\n // https://doc.rust-lang.org/nightly/reference/type-layout.html#str-layout\n unsafe { Self::from_raw(Arc::into_raw(rc) as *const [u8]) }\n }\n}\n\n#[cfg(not(portable_atomic_no_min_const_generics))]\nitems!({\n impl TryFrom> for Arc<[T; N]> {\n type Error = Arc<[T]>;\n\n fn try_from(boxed_slice: Arc<[T]>) -> Result {\n if boxed_slice.len() == N {\n let ptr = Arc::into_inner_non_null(boxed_slice);\n Ok(unsafe { Self::from_inner(ptr.cast::>()) })\n } else {\n Err(boxed_slice)\n }\n }\n }\n});\n\nimpl FromIterator for Arc<[T]> {\n /// Takes each element in the `Iterator` and collects it into an `Arc<[T]>`.\n ///\n /// # Performance characteristics\n ///\n /// ## The general case\n ///\n /// In the general case, collecting into `Arc<[T]>` is done by first\n /// collecting into a `Vec`. That is, when writing the following:\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let evens: Arc<[u8]> = (0..10).filter(|&x| x % 2 == 0).collect();\n /// # assert_eq!(&*evens, &[0, 2, 4, 6, 8]);\n /// ```\n ///\n /// this behaves as if we wrote:\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let evens: Arc<[u8]> = (0..10).filter(|&x| x % 2 == 0)\n /// .collect::>() // The first set of allocations happens here.\n /// .into(); // A second allocation for `Arc<[T]>` happens here.\n ///\n /// assert_eq!(&*evens, &[0, 2, 4, 6, 8]);\n /// ```\n ///\n /// This will allocate as many times as needed for constructing the `Vec`\n /// and then it will allocate once for turning the `Vec` into the `Arc<[T]>`.\n ///\n /// ## Iterators of known length\n ///\n /// When your `Iterator` implements `TrustedLen` and is of an exact size,\n /// a single allocation will be made for the `Arc<[T]>`. For example:\n ///\n /// ```\n /// use portable_atomic_util::Arc;\n /// let evens: Arc<[u8]> = (0..10).collect(); // Just a single allocation happens here.\n ///\n /// assert_eq!(&*evens, &*(0..10).collect::>());\n /// ```\n fn from_iter>(iter: I) -> Self {\n iter.into_iter().collect::>().into()\n }\n}\n\nimpl borrow::Borrow for Arc {\n fn borrow(&self) -> &T {\n self\n }\n}\n\nimpl AsRef for Arc {\n fn as_ref(&self) -> &T {\n self\n }\n}\n\nimpl Unpin for Arc {}\n\n/// Gets the pointer to data within the given an `ArcInner`.\n///\n/// # Safety\n///\n/// `arc` must uphold the safety requirements for `.byte_add(data_offset)`.\n/// This is automatically satisfied if it is a pointer to a valid `ArcInner`.\nunsafe fn data_ptr(arc: *mut ArcInner, data: &T) -> *mut T {\n // SAFETY: the caller must uphold the safety contract.\n unsafe {\n let offset = data_offset::(data);\n strict::byte_add(arc, offset) as *mut T\n }\n}\n\n/// Gets the offset within an `ArcInner` for the payload behind a pointer.\nfn data_offset(ptr: &T) -> usize {\n // Align the unsized value to the end of the ArcInner.\n // Because ArcInner is repr(C), it will always be the last field in memory.\n data_offset_align(mem::align_of_val::(ptr))\n}\n\n#[inline]\nfn data_offset_align(align: usize) -> usize {\n let layout = Layout::new::>();\n layout.size() + layout::padding_needed_for(layout, align)\n}\n\n/// A unique owning pointer to an [`ArcInner`] **that does not imply the contents are initialized,**\n/// but will deallocate it (without dropping the value) when dropped.\n///\n/// This is a helper for [`Arc::make_mut()`] to ensure correct cleanup on panic.\nstruct UniqueArcUninit {\n ptr: NonNull>,\n layout_for_value: Layout,\n}\n\nimpl UniqueArcUninit {\n /// Allocates an ArcInner with layout suitable to contain `for_value` or a clone of it.\n fn new(for_value: &T) -> Self {\n let layout = Layout::for_value(for_value);\n let ptr = unsafe { Arc::allocate_for_value(for_value) };\n Self { ptr: NonNull::new(ptr).unwrap(), layout_for_value: layout }\n }\n\n /// Returns the pointer to be written into to initialize the [`Arc`].\n fn data_ptr(&mut self) -> *mut T {\n let offset = data_offset_align(self.layout_for_value.align());\n unsafe { strict::byte_add(self.ptr.as_ptr(), offset) as *mut T }\n }\n\n /// Upgrade this into a normal [`Arc`].\n ///\n /// # Safety\n ///\n /// The data must have been initialized (by writing to [`Self::data_ptr()`]).\n unsafe fn into_arc(self) -> Arc {\n let this = ManuallyDrop::new(self);\n let ptr = this.ptr.as_ptr();\n\n // SAFETY: The pointer is valid as per `UniqueArcUninit::new`, and the caller is responsible\n // for having initialized the data.\n unsafe { Arc::from_ptr(ptr) }\n }\n}\n\nimpl Drop for UniqueArcUninit {\n fn drop(&mut self) {\n // SAFETY:\n // * new() produced a pointer safe to deallocate.\n // * We own the pointer unless into_arc() was called, which forgets us.\n unsafe {\n Global.deallocate(\n self.ptr.cast::(),\n arc_inner_layout_for_value_layout(self.layout_for_value),\n );\n }\n }\n}\n\n#[cfg(not(portable_atomic_no_error_in_core))]\nuse core::error;\n#[cfg(all(portable_atomic_no_error_in_core, feature = \"std\"))]\nuse std::error;\n#[cfg(any(not(portable_atomic_no_error_in_core), feature = \"std\"))]\nimpl error::Error for Arc {\n #[allow(deprecated)]\n fn cause(&self) -> Option<&dyn error::Error> {\n error::Error::cause(&**self)\n }\n fn source(&self) -> Option<&(dyn error::Error + 'static)> {\n error::Error::source(&**self)\n }\n}\n\n#[cfg(feature = \"serde\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"serde\")))]\nmod serde_impls {\n use serde::{\n de::{Deserialize, Deserializer},\n ser::{Serialize, Serializer},\n };\n\n use super::{Arc, Box, Weak};\n\n // Refs: https://github.com/serde-rs/serde/blob/v1.0.228/serde_core/src/ser/impls.rs#L472\n impl Serialize for Arc {\n #[inline]\n fn serialize(&self, serializer: S) -> Result\n where\n S: Serializer,\n {\n (**self).serialize(serializer)\n }\n }\n\n // Refs: https://github.com/serde-rs/serde/blob/v1.0.228/serde_core/src/ser/impls.rs#L564\n impl Serialize for Weak {\n fn serialize(&self, serializer: S) -> Result\n where\n S: Serializer,\n {\n self.upgrade().serialize(serializer)\n }\n }\n\n // Refs: https://github.com/serde-rs/serde/blob/v1.0.228/serde_core/src/de/impls.rs#L2057\n impl<'de, T> Deserialize<'de> for Arc\n where\n T: ?Sized,\n Box: Deserialize<'de>,\n {\n fn deserialize(deserializer: D) -> Result\n where\n D: serde::Deserializer<'de>,\n {\n Box::deserialize(deserializer).map(Into::into)\n }\n }\n\n // Refs: https://github.com/serde-rs/serde/blob/v1.0.228/serde_core/src/de/impls.rs#L2035\n impl<'de, T> Deserialize<'de> for Weak\n where\n T: Deserialize<'de>,\n {\n fn deserialize(deserializer: D) -> Result\n where\n D: Deserializer<'de>,\n {\n Option::::deserialize(deserializer)?;\n Ok(Weak::new())\n }\n }\n}\n\n#[cfg(feature = \"std\")]\nmod std_impls {\n // TODO: Other trait implementations that are stable but we currently don't provide:\n // - alloc::ffi\n // - https://doc.rust-lang.org/nightly/alloc/sync/struct.Arc.html#impl-From%3C%26CStr%3E-for-Arc%3CCStr%3E\n // - https://doc.rust-lang.org/nightly/alloc/sync/struct.Arc.html#impl-From%3C%26mut+CStr%3E-for-Arc%3CCStr%3E\n // - https://doc.rust-lang.org/nightly/alloc/sync/struct.Arc.html#impl-From%3CCString%3E-for-Arc%3CCStr%3E\n // - https://doc.rust-lang.org/nightly/alloc/sync/struct.Arc.html#impl-Default-for-Arc%3CCStr%3E\n // - Currently, we cannot implement these since CStr layout is not stable.\n // - std::ffi\n // - https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-From%3C%26OsStr%3E-for-Arc%3COsStr%3E\n // - https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-From%3C%26mut+OsStr%3E-for-Arc%3COsStr%3E\n // - https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-From%3COsString%3E-for-Arc%3COsStr%3E\n // - Currently, we cannot implement these since OsStr layout is not stable.\n // - std::path\n // - https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-From%3C%26Path%3E-for-Arc%3CPath%3E\n // - https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-From%3C%26mut+Path%3E-for-Arc%3CPath%3E\n // - https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-From%3CPathBuf%3E-for-Arc%3CPath%3E\n // - Currently, we cannot implement these since Path layout is not stable.\n\n use std::io;\n // https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-AsFd-for-Arc%3CT%3E\n // https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-AsHandle-for-Arc%3CT%3E\n // https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-AsRawFd-for-Arc%3CT%3E\n // https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-AsSocket-for-Arc%3CT%3E\n // Note:\n // - T: ?Sized is currently only allowed on AsFd/AsHandle: https://github.com/rust-lang/rust/pull/114655#issuecomment-1977994288\n // - std doesn't implement AsRawHandle/AsRawSocket for Arc as of Rust 1.90.\n // - std::os::unix::io::AsRawFd and std::os::windows::io::{AsRawHandle, AsRawSocket} are available in all versions\n // - std::os::wasi::prelude::AsRawFd requires 1.56 (https://github.com/rust-lang/rust/commit/e555003e6d6b6d71ce5509a6b6c7a15861208d6c)\n // - std::os::unix::io::AsFd, std::os::wasi::prelude::AsFd, and std::os::windows::io::{AsHandle, AsSocket} require Rust 1.63\n // - std::os::wasi::io::AsFd requires Rust 1.65 (https://github.com/rust-lang/rust/pull/103308)\n // - std::os::fd requires Rust 1.66 (https://github.com/rust-lang/rust/pull/98368)\n // - std::os::hermit::io::AsFd requires Rust 1.69 (https://github.com/rust-lang/rust/commit/b5fb4f3d9b1b308d59cab24ef2f9bf23dad948aa)\n // - std::os::fd for HermitOS requires Rust 1.81 (https://github.com/rust-lang/rust/pull/126346)\n // - std::os::fd for Trusty requires Rust 1.87 (no std support before it, https://github.com/rust-lang/rust/commit/7f6ee12526700e037ef34912b2b0c628028d382c)\n // - std::os::solid::io::AsFd is unstable (solid_ext, https://github.com/rust-lang/rust/pull/115159)\n // Note: we don't implement unstable ones.\n #[cfg(not(portable_atomic_no_io_safety))]\n #[cfg(target_os = \"trusty\")]\n use std::os::fd;\n #[cfg(not(portable_atomic_no_io_safety))]\n #[cfg(target_os = \"hermit\")]\n use std::os::hermit::io as fd;\n #[cfg(unix)]\n use std::os::unix::io as fd;\n #[cfg(not(portable_atomic_no_io_safety))]\n #[cfg(target_os = \"wasi\")]\n use std::os::wasi::prelude as fd;\n\n use super::Arc;\n\n /// This impl allows implementing traits that require `AsRawFd` on Arc.\n /// ```\n /// # #[cfg(target_os = \"hermit\")]\n /// # use std::os::hermit::io::AsRawFd;\n /// # #[cfg(target_os = \"wasi\")]\n /// # use std::os::wasi::prelude::AsRawFd;\n /// # #[cfg(unix)]\n /// # use std::os::unix::io::AsRawFd;\n /// use std::net::UdpSocket;\n ///\n /// use portable_atomic_util::Arc;\n ///\n /// trait MyTrait: AsRawFd {}\n /// impl MyTrait for Arc {}\n /// ```\n #[cfg(any(\n unix,\n all(\n not(portable_atomic_no_io_safety),\n any(target_os = \"hermit\", target_os = \"trusty\", target_os = \"wasi\"),\n ),\n ))]\n impl fd::AsRawFd for Arc {\n #[inline]\n fn as_raw_fd(&self) -> fd::RawFd {\n (**self).as_raw_fd()\n }\n }\n /// This impl allows implementing traits that require `AsFd` on Arc.\n /// ```\n /// # #[cfg(target_os = \"hermit\")]\n /// # use std::os::hermit::io::AsFd;\n /// # #[cfg(target_os = \"wasi\")]\n /// # use std::os::wasi::prelude::AsFd;\n /// # #[cfg(unix)]\n /// # use std::os::unix::io::AsFd;\n /// use std::net::UdpSocket;\n ///\n /// use portable_atomic_util::Arc;\n ///\n /// trait MyTrait: AsFd {}\n /// impl MyTrait for Arc {}\n /// ```\n #[cfg(not(portable_atomic_no_io_safety))]\n #[cfg(any(unix, target_os = \"hermit\", target_os = \"trusty\", target_os = \"wasi\"))]\n impl fd::AsFd for Arc {\n #[inline]\n fn as_fd(&self) -> fd::BorrowedFd<'_> {\n (**self).as_fd()\n }\n }\n /// This impl allows implementing traits that require `AsHandle` on Arc.\n /// ```\n /// # use std::os::windows::io::AsHandle;\n /// use std::fs::File;\n ///\n /// use portable_atomic_util::Arc;\n ///\n /// trait MyTrait: AsHandle {}\n /// impl MyTrait for Arc {}\n /// ```\n #[cfg(not(portable_atomic_no_io_safety))]\n #[cfg(windows)]\n impl std::os::windows::io::AsHandle for Arc {\n #[inline]\n fn as_handle(&self) -> std::os::windows::io::BorrowedHandle<'_> {\n (**self).as_handle()\n }\n }\n /// This impl allows implementing traits that require `AsSocket` on Arc.\n /// ```\n /// # use std::os::windows::io::AsSocket;\n /// use std::net::UdpSocket;\n ///\n /// use portable_atomic_util::Arc;\n ///\n /// trait MyTrait: AsSocket {}\n /// impl MyTrait for Arc {}\n /// ```\n #[cfg(not(portable_atomic_no_io_safety))]\n #[cfg(windows)]\n impl std::os::windows::io::AsSocket for Arc {\n #[inline]\n fn as_socket(&self) -> std::os::windows::io::BorrowedSocket<'_> {\n (**self).as_socket()\n }\n }\n\n // https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-Read-for-Arc%3CFile%3E\n // https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-Seek-for-Arc%3CFile%3E\n // https://doc.rust-lang.org/nightly/std/sync/struct.Arc.html#impl-Write-for-Arc%3CFile%3E\n impl io::Read for Arc {\n fn read(&mut self, buf: &mut [u8]) -> io::Result {\n (&**self).read(buf)\n }\n fn read_vectored(&mut self, bufs: &mut [io::IoSliceMut<'_>]) -> io::Result {\n (&**self).read_vectored(bufs)\n }\n // fn read_buf(&mut self, cursor: io::BorrowedCursor<'_>) -> io::Result<()> {\n // (&**self).read_buf(cursor)\n // }\n // #[inline]\n // fn is_read_vectored(&self) -> bool {\n // (&**self).is_read_vectored()\n // }\n fn read_to_end(&mut self, buf: &mut alloc::vec::Vec) -> io::Result {\n (&**self).read_to_end(buf)\n }\n fn read_to_string(&mut self, buf: &mut alloc::string::String) -> io::Result {\n (&**self).read_to_string(buf)\n }\n }\n impl io::Write for Arc {\n fn write(&mut self, buf: &[u8]) -> io::Result {\n (&**self).write(buf)\n }\n fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result {\n (&**self).write_vectored(bufs)\n }\n // #[inline]\n // fn is_write_vectored(&self) -> bool {\n // (&**self).is_write_vectored()\n // }\n #[inline]\n fn flush(&mut self) -> io::Result<()> {\n (&**self).flush()\n }\n }\n impl io::Seek for Arc {\n fn seek(&mut self, pos: io::SeekFrom) -> io::Result {\n (&**self).seek(pos)\n }\n }\n // TODO: TcpStream and UnixStream: https://github.com/rust-lang/rust/pull/134190\n // impl io::Read for Arc {\n // fn read(&mut self, buf: &mut [u8]) -> io::Result {\n // (&**self).read(buf)\n // }\n // // fn read_buf(&mut self, buf: io::BorrowedCursor<'_>) -> io::Result<()> {\n // // (&**self).read_buf(buf)\n // // }\n // fn read_vectored(&mut self, bufs: &mut [io::IoSliceMut<'_>]) -> io::Result {\n // (&**self).read_vectored(bufs)\n // }\n // // #[inline]\n // // fn is_read_vectored(&self) -> bool {\n // // (&**self).is_read_vectored()\n // // }\n // }\n // impl io::Write for Arc {\n // fn write(&mut self, buf: &[u8]) -> io::Result {\n // (&**self).write(buf)\n // }\n // fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result {\n // (&**self).write_vectored(bufs)\n // }\n // // #[inline]\n // // fn is_write_vectored(&self) -> bool {\n // // (&**self).is_write_vectored()\n // // }\n // #[inline]\n // fn flush(&mut self) -> io::Result<()> {\n // (&**self).flush()\n // }\n // }\n // #[cfg(unix)]\n // impl io::Read for Arc {\n // fn read(&mut self, buf: &mut [u8]) -> io::Result {\n // (&**self).read(buf)\n // }\n // // fn read_buf(&mut self, buf: io::BorrowedCursor<'_>) -> io::Result<()> {\n // // (&**self).read_buf(buf)\n // // }\n // fn read_vectored(&mut self, bufs: &mut [io::IoSliceMut<'_>]) -> io::Result {\n // (&**self).read_vectored(bufs)\n // }\n // // #[inline]\n // // fn is_read_vectored(&self) -> bool {\n // // (&**self).is_read_vectored()\n // // }\n // }\n // #[cfg(unix)]\n // impl io::Write for Arc {\n // fn write(&mut self, buf: &[u8]) -> io::Result {\n // (&**self).write(buf)\n // }\n // fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result {\n // (&**self).write_vectored(bufs)\n // }\n // // #[inline]\n // // fn is_write_vectored(&self) -> bool {\n // // (&**self).is_write_vectored()\n // // }\n // #[inline]\n // fn flush(&mut self) -> io::Result<()> {\n // (&**self).flush()\n // }\n // }\n}\n\nuse self::clone::CloneToUninit;\nmod clone {\n use core::{\n mem::{self, MaybeUninit},\n ptr, slice,\n };\n\n use super::strict;\n\n // Based on unstable core::clone::CloneToUninit.\n // This trait is private and cannot be implemented for types outside of `portable-atomic-util`.\n #[doc(hidden)] // private API\n #[allow(unknown_lints, unnameable_types)] // Not public API. unnameable_types is available on Rust 1.79+\n pub unsafe trait CloneToUninit {\n unsafe fn clone_to_uninit(&self, dest: *mut u8);\n }\n unsafe impl CloneToUninit for T {\n #[inline]\n unsafe fn clone_to_uninit(&self, dest: *mut u8) {\n // SAFETY: we're calling a specialization with the same contract\n unsafe { clone_one(self, dest as *mut T) }\n }\n }\n unsafe impl CloneToUninit for [T] {\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n unsafe fn clone_to_uninit(&self, dest: *mut u8) {\n let dest: *mut [T] = strict::with_metadata_of(dest, self);\n // SAFETY: we're calling a specialization with the same contract\n unsafe { clone_slice(self, dest) }\n }\n }\n unsafe impl CloneToUninit for str {\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n unsafe fn clone_to_uninit(&self, dest: *mut u8) {\n // SAFETY: str is just a [u8] with UTF-8 invariant\n unsafe { self.as_bytes().clone_to_uninit(dest) }\n }\n }\n // Note: Currently, we cannot implement this for CStr/OsStr/Path since theirs layout is not stable.\n\n #[inline]\n unsafe fn clone_one(src: &T, dst: *mut T) {\n // SAFETY: The safety conditions of clone_to_uninit() are a superset of those of\n // ptr::write().\n unsafe {\n // We hope the optimizer will figure out to create the cloned value in-place,\n // skipping ever storing it on the stack and the copy to the destination.\n ptr::write(dst, src.clone());\n }\n }\n #[inline]\n #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]\n unsafe fn clone_slice(src: &[T], dst: *mut [T]) {\n let len = src.len();\n\n // SAFETY: The produced `&mut` is valid because:\n // * The caller is obligated to provide a pointer which is valid for writes.\n // * All bytes pointed to are in MaybeUninit, so we don't care about the memory's\n // initialization status.\n let uninit_ref = unsafe { &mut *(dst as *mut [MaybeUninit]) };\n\n // This is the most likely mistake to make, so check it as a debug assertion.\n debug_assert_eq!(\n len,\n uninit_ref.len(), // <*const [T]>::len is unstable\n \"clone_to_uninit() source and destination must have equal lengths\",\n );\n\n // Copy the elements\n let mut initializing = InitializingSlice::from_fully_uninit(uninit_ref);\n for element_ref in src {\n // If the clone() panics, `initializing` will take care of the cleanup.\n initializing.push(element_ref.clone());\n }\n // If we reach here, then the entire slice is initialized, and we've satisfied our\n // responsibilities to the caller. Disarm the cleanup guard by forgetting it.\n mem::forget(initializing);\n }\n\n /// Ownership of a collection of values stored in a non-owned `[MaybeUninit]`, some of which\n /// are not yet initialized. This is sort of like a `Vec` that doesn't own its allocation.\n /// Its responsibility is to provide cleanup on unwind by dropping the values that *are*\n /// initialized, unless disarmed by forgetting.\n ///\n /// This is a helper for `impl CloneToUninit for [T]`.\n struct InitializingSlice<'a, T> {\n data: &'a mut [MaybeUninit],\n /// Number of elements of `*self.data` that are initialized.\n initialized_len: usize,\n }\n impl<'a, T> InitializingSlice<'a, T> {\n #[inline]\n fn from_fully_uninit(data: &'a mut [MaybeUninit]) -> Self {\n Self { data, initialized_len: 0 }\n }\n /// Push a value onto the end of the initialized part of the slice.\n ///\n /// # Panics\n ///\n /// Panics if the slice is already fully initialized.\n #[inline]\n fn push(&mut self, value: T) {\n self.data[self.initialized_len] = MaybeUninit::new(value);\n self.initialized_len += 1;\n }\n }\n impl Drop for InitializingSlice<'_, T> {\n #[cold] // will only be invoked on unwind\n fn drop(&mut self) {\n let initialized_slice = unsafe {\n slice::from_raw_parts_mut(self.data.as_mut_ptr() as *mut T, self.initialized_len)\n };\n // SAFETY:\n // * the pointer is valid because it was made from a mutable reference\n // * `initialized_len` counts the initialized elements as an invariant of this type,\n // so each of the pointed-to elements is initialized and may be dropped.\n unsafe {\n ptr::drop_in_place::<[T]>(initialized_slice);\n }\n }\n }\n}\n\nmod layout {\n use core::{alloc::Layout, cmp};\n\n use super::{ISIZE_MAX, USIZE_MAX};\n\n // Based on unstable Layout::padding_needed_for.\n #[inline]\n #[must_use]\n pub(super) fn padding_needed_for(layout: Layout, align: usize) -> usize {\n // FIXME: Can we just change the type on this to `Alignment`?\n if !align.is_power_of_two() {\n return USIZE_MAX;\n }\n let len_rounded_up = size_rounded_up_to_custom_align(layout, align);\n // SAFETY: Cannot overflow because the rounded-up value is never less\n len_rounded_up.wrapping_sub(layout.size()) // can use unchecked_sub\n }\n\n /// Returns the smallest multiple of `align` greater than or equal to `self.size()`.\n ///\n /// This can return at most `Alignment::MAX` (aka `isize::MAX + 1`)\n /// because the original size is at most `isize::MAX`.\n #[inline]\n fn size_rounded_up_to_custom_align(layout: Layout, align: usize) -> usize {\n // Rounded up value is:\n // size_rounded_up = (size + align - 1) & !(align - 1);\n //\n // The arithmetic we do here can never overflow:\n //\n // 1. align is guaranteed to be > 0, so align - 1 is always\n // valid.\n //\n // 2. size is at most `isize::MAX`, so adding `align - 1` (which is at\n // most `isize::MAX`) can never overflow a `usize`.\n //\n // 3. masking by the alignment can remove at most `align - 1`,\n // which is what we just added, thus the value we return is never\n // less than the original `size`.\n //\n // (Size 0 Align MAX is already aligned, so stays the same, but things like\n // Size 1 Align MAX or Size isize::MAX Align 2 round up to `isize::MAX + 1`.)\n let align_m1 = align.wrapping_sub(1);\n layout.size().wrapping_add(align_m1) & !align_m1\n }\n\n // Based on Layout::pad_to_align stabilized in Rust 1.44.\n #[inline]\n #[must_use]\n pub(super) fn pad_to_align(layout: Layout) -> Layout {\n // This cannot overflow. Quoting from the invariant of Layout:\n // > `size`, when rounded up to the nearest multiple of `align`,\n // > must not overflow isize (i.e., the rounded value must be\n // > less than or equal to `isize::MAX`)\n let new_size = size_rounded_up_to_custom_align(layout, layout.align());\n\n // SAFETY: padded size is guaranteed to not exceed `isize::MAX`.\n unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }\n }\n\n // Based on Layout::extend stabilized in Rust 1.44.\n #[inline]\n pub(super) fn extend(layout: Layout, next: Layout) -> Option<(Layout, usize)> {\n let new_align = cmp::max(layout.align(), next.align());\n let offset = size_rounded_up_to_custom_align(layout, next.align());\n\n // SAFETY: `offset` is at most `isize::MAX + 1` (such as from aligning\n // to `Alignment::MAX`) and `next.size` is at most `isize::MAX` (from the\n // `Layout` type invariant). Thus the largest possible `new_size` is\n // `isize::MAX + 1 + isize::MAX`, which is `usize::MAX`, and cannot overflow.\n let new_size = offset.wrapping_add(next.size()); // can use unchecked_add\n\n let layout = Layout::from_size_align(new_size, new_align).ok()?;\n Some((layout, offset))\n }\n\n // Based on Layout::array stabilized in Rust 1.44.\n #[inline]\n pub(super) fn array(n: usize) -> Option {\n #[inline(always)]\n const fn max_size_for_align(align: usize) -> usize {\n // (power-of-two implies align != 0.)\n\n // Rounded up size is:\n // size_rounded_up = (size + align - 1) & !(align - 1);\n //\n // We know from above that align != 0. If adding (align - 1)\n // does not overflow, then rounding up will be fine.\n //\n // Conversely, &-masking with !(align - 1) will subtract off\n // only low-order-bits. Thus if overflow occurs with the sum,\n // the &-mask cannot subtract enough to undo that overflow.\n //\n // Above implies that checking for summation overflow is both\n // necessary and sufficient.\n\n // SAFETY: the maximum possible alignment is `isize::MAX + 1`,\n // so the subtraction cannot overflow.\n (ISIZE_MAX as usize + 1).wrapping_sub(align)\n }\n\n #[inline]\n fn inner(element_layout: Layout, n: usize) -> Option {\n let element_size = element_layout.size();\n let align = element_layout.align();\n\n // We need to check two things about the size:\n // - That the total size won't overflow a `usize`, and\n // - That the total size still fits in an `isize`.\n // By using division we can check them both with a single threshold.\n // That'd usually be a bad idea, but thankfully here the element size\n // and alignment are constants, so the compiler will fold all of it.\n if element_size != 0 && n > max_size_for_align(align) / element_size {\n return None;\n }\n\n // SAFETY: We just checked that we won't overflow `usize` when we multiply.\n // This is a useless hint inside this function, but after inlining this helps\n // deduplicate checks for whether the overall capacity is zero (e.g., in `RawVec`'s\n // allocation path) before/after this multiplication.\n let array_size = element_size.wrapping_mul(n); // can use unchecked_mul\n\n // SAFETY: We just checked above that the `array_size` will not\n // exceed `isize::MAX` even when rounded up to the alignment.\n // And `Alignment` guarantees it's a power of two.\n unsafe { Some(Layout::from_size_align_unchecked(array_size, align)) }\n }\n\n // Reduce the amount of code we need to monomorphize per `T`.\n inner(Layout::new::(), n)\n }\n}\n\n#[cfg(feature = \"std\")]\nuse std::process::abort;\n#[cfg(not(feature = \"std\"))]\n#[cold]\nfn abort() -> ! {\n struct Abort;\n impl Drop for Abort {\n fn drop(&mut self) {\n panic!();\n }\n }\n\n let _abort = Abort;\n panic!(\"abort\")\n}\n\nfn is_dangling(ptr: *const T) -> bool {\n (ptr as *const ()).addr() == USIZE_MAX\n}\n\n// Based on unstable alloc::alloc::Global.\n//\n// Note: unlike alloc::alloc::Global that returns NonNull<[u8]>,\n// this returns NonNull.\nstruct Global;\n#[allow(clippy::unused_self)]\nimpl Global {\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n fn alloc_impl(&self, layout: Layout, zeroed: bool) -> Option> {\n // Layout::dangling is unstable\n #[inline]\n #[must_use]\n fn dangling(layout: Layout) -> NonNull {\n // SAFETY: align is guaranteed to be non-zero\n unsafe { NonNull::new_unchecked(strict::without_provenance_mut::(layout.align())) }\n }\n\n match layout.size() {\n 0 => Some(dangling(layout)),\n // SAFETY: `layout` is non-zero in size,\n _size => unsafe {\n let raw_ptr = if zeroed {\n alloc::alloc::alloc_zeroed(layout)\n } else {\n alloc::alloc::alloc(layout)\n };\n NonNull::new(raw_ptr)\n },\n }\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n fn allocate(self, layout: Layout) -> Option> {\n self.alloc_impl(layout, false)\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n fn allocate_zeroed(self, layout: Layout) -> Option> {\n self.alloc_impl(layout, true)\n }\n #[inline]\n #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces\n unsafe fn deallocate(self, ptr: NonNull, layout: Layout) {\n if layout.size() != 0 {\n // SAFETY:\n // * We have checked that `layout` is non-zero in size.\n // * The caller is obligated to provide a layout that \"fits\", and in this case,\n // \"fit\" always means a layout that is equal to the original, because our\n // `allocate()`, `grow()`, and `shrink()` implementations never returns a larger\n // allocation than requested.\n // * Other conditions must be upheld by the caller, as per `Allocator::deallocate()`'s\n // safety documentation.\n unsafe { alloc::alloc::dealloc(ptr.as_ptr(), layout) }\n }\n }\n}\n" }, { "package": "portable-atomic-util@0.2.8", "sourcePath": "src/lib.rs", "sha256": "4cedeff1ec801c7365d9c033354f5632f7f25c46bf3d48f8330ede559dc579a6", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*!\n\n\n\nSynchronization primitives built with [portable-atomic].\n\n- Provide `Arc`. (optional, requires the `std` or `alloc` feature)\n- Provide `task::Wake`. (optional, requires the `std` or `alloc` feature)\n\n\nSee [portable-atomic#1] for other primitives being considered for addition to this crate.\n\nThis crate was originally [part of the portable-atomic repository](https://github.com/taiki-e/portable-atomic/tree/cbbee0c0d202a5944f7d66aaafaac6ed76e6f599/portable-atomic-util) and was extracted into its own repository.\n\n## Optional features\n\n- **`std`**
\n Use `std`.\n\n Note:\n - This implicitly enables the `alloc` feature.\n\n- **`alloc`**
\n Use `alloc`.\n\n- **`serde`**
\n Implement `serde::{Serialize, Deserialize}` for `Arc`.\n\n Note:\n - The MSRV when this feature is enabled is the one coming from serde, as of now it's Rust 1.56\n\n\n\n[portable-atomic]: https://github.com/taiki-e/portable-atomic\n[portable-atomic#1]: https://github.com/taiki-e/portable-atomic/issues/1\n\n## Optional cfg\n\nOne of the ways to enable cfg is to set [rustflags in the cargo config](https://doc.rust-lang.org/cargo/reference/config.html#targettriplerustflags):\n\n```toml\n# .cargo/config.toml\n[target.]\nrustflags = [\"--cfg\", \"portable_atomic_unstable_coerce_unsized\"]\n```\n\nOr set environment variable:\n\n```sh\nRUSTFLAGS=\"--cfg portable_atomic_unstable_coerce_unsized\" cargo ...\n```\n\n- **`--cfg portable_atomic_unstable_coerce_unsized`**
\n Support coercing of `Arc` to `Arc` as in `std::sync::Arc`.\n\n \n\n This cfg requires Rust nightly because this coercing requires [unstable `CoerceUnsized` trait](https://doc.rust-lang.org/nightly/core/ops/trait.CoerceUnsized.html).\n\n See [this issue comment](https://github.com/taiki-e/portable-atomic/issues/143#issuecomment-1866488569) for another known workaround.\n\n **Note:** This cfg is unstable and outside of the normal semver guarantees and minor or patch versions of portable-atomic-util may make breaking changes to them at any time.\n\n## Related Projects\n\n- [portable-atomic]: Portable atomic types including support for 128-bit atomics, atomic float, etc.\n- [atomic-maybe-uninit]: Atomic operations on potentially uninitialized integers.\n- [atomic-memcpy]: Byte-wise atomic memcpy.\n\n[atomic-maybe-uninit]: https://github.com/taiki-e/atomic-maybe-uninit\n[atomic-memcpy]: https://github.com/taiki-e/atomic-memcpy\n\n\n*/\n\n#![no_std]\n#![doc(test(\n no_crate_inject,\n attr(allow(\n dead_code,\n unused_variables,\n clippy::undocumented_unsafe_blocks,\n clippy::unused_trait_names,\n ))\n))]\n#![cfg_attr(not(portable_atomic_no_unsafe_op_in_unsafe_fn), warn(unsafe_op_in_unsafe_fn))] // unsafe_op_in_unsafe_fn requires Rust 1.52\n#![cfg_attr(portable_atomic_no_unsafe_op_in_unsafe_fn, allow(unused_unsafe))]\n#![warn(\n // Lints that may help when writing public library.\n missing_debug_implementations,\n missing_docs,\n clippy::alloc_instead_of_core,\n clippy::exhaustive_enums,\n clippy::exhaustive_structs,\n clippy::impl_trait_in_params,\n clippy::std_instead_of_alloc,\n clippy::std_instead_of_core,\n // clippy::missing_inline_in_public_items,\n)]\n#![cfg_attr(portable_atomic_no_strict_provenance, allow(unstable_name_collisions))]\n#![allow(clippy::inline_always)]\n// docs.rs only (cfg is enabled by docs.rs, not build script)\n#![cfg_attr(docsrs, feature(doc_cfg))]\n#![cfg_attr(docsrs, doc(auto_cfg = false))]\n// Enable custom unsized coercions if the user explicitly opts-in to unstable cfg\n#![cfg_attr(portable_atomic_unstable_coerce_unsized, feature(coerce_unsized, unsize))]\n\n#[cfg(feature = \"alloc\")]\nextern crate alloc;\n#[cfg(feature = \"std\")]\nextern crate std;\n\n#[macro_use]\nmod utils;\n\n#[cfg(feature = \"alloc\")]\nmod arc;\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(any(feature = \"alloc\", feature = \"std\"))))]\npub use self::arc::{Arc, Weak};\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(any(feature = \"alloc\", feature = \"std\"))))]\npub mod task;\n" }, { "package": "portable-atomic-util@0.2.8", "sourcePath": "src/task.rs", "sha256": "fc6f5a52f300d428f07b915aea711b638d49b00e4b5b804dcff00937e7fda2c5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Types and Traits for working with asynchronous tasks.\n\n// This module is based on alloc::task::Wake.\n//\n// The code has been adjusted to work with stable Rust.\n//\n// Source: https://github.com/rust-lang/rust/blob/1.84.0/library/alloc/src/task.rs.\n//\n// Copyright & License of the original code:\n// - https://github.com/rust-lang/rust/blob/1.84.0/COPYRIGHT\n// - https://github.com/rust-lang/rust/blob/1.84.0/LICENSE-APACHE\n// - https://github.com/rust-lang/rust/blob/1.84.0/LICENSE-MIT\n\nuse core::{\n mem::ManuallyDrop,\n task::{RawWaker, RawWakerVTable, Waker},\n};\n\nuse crate::Arc;\n\n/// The implementation of waking a task on an executor.\n///\n/// This is an equivalent to [`std::task::Wake`], but using [`portable_atomic_util::Arc`](crate::Arc)\n/// as a reference-counted pointer. See the documentation for [`std::task::Wake`] for more details.\n///\n/// **Note:** Unlike `std::task::Wake`, all methods take `this:` instead of `self:`.\n/// This is because using `portable_atomic_util::Arc` as a receiver requires the\n/// [unstable `arbitrary_self_types` feature](https://github.com/rust-lang/rust/issues/44874).\n///\n/// # Examples\n///\n/// A basic `block_on` function that takes a future and runs it to completion on\n/// the current thread.\n///\n/// **Note:** This example trades correctness for simplicity. In order to prevent\n/// deadlocks, production-grade implementations will also need to handle\n/// intermediate calls to `thread::unpark` as well as nested invocations.\n///\n/// ```\n/// use std::{\n/// future::Future,\n/// task::{Context, Poll},\n/// thread::{self, Thread},\n/// };\n///\n/// use portable_atomic_util::{Arc, task::Wake};\n///\n/// /// A waker that wakes up the current thread when called.\n/// struct ThreadWaker(Thread);\n///\n/// impl Wake for ThreadWaker {\n/// fn wake(this: Arc) {\n/// this.0.unpark();\n/// }\n/// }\n///\n/// /// Run a future to completion on the current thread.\n/// fn block_on(fut: impl Future) -> T {\n/// // Pin the future so it can be polled.\n/// let mut fut = Box::pin(fut);\n///\n/// // Create a new context to be passed to the future.\n/// let t = thread::current();\n/// let waker = Arc::new(ThreadWaker(t)).into();\n/// let mut cx = Context::from_waker(&waker);\n///\n/// // Run the future to completion.\n/// loop {\n/// match fut.as_mut().poll(&mut cx) {\n/// Poll::Ready(res) => return res,\n/// Poll::Pending => thread::park(),\n/// }\n/// }\n/// }\n///\n/// block_on(async {\n/// println!(\"Hi from inside a future!\");\n/// });\n/// ```\npub trait Wake {\n /// Wake this task.\n fn wake(this: Arc);\n\n /// Wake this task without consuming the waker.\n ///\n /// If an executor supports a cheaper way to wake without consuming the\n /// waker, it should override this method. By default, it clones the\n /// [`Arc`] and calls [`wake`] on the clone.\n ///\n /// [`wake`]: Wake::wake\n fn wake_by_ref(this: &Arc) {\n Self::wake(this.clone());\n }\n}\nimpl From> for Waker {\n /// Use a [`Wake`]-able type as a `Waker`.\n ///\n /// No heap allocations or atomic operations are used for this conversion.\n fn from(waker: Arc) -> Self {\n // SAFETY: This is safe because raw_waker safely constructs\n // a RawWaker from Arc.\n unsafe { Self::from_raw(raw_waker(waker)) }\n }\n}\nimpl From> for RawWaker {\n /// Use a `Wake`-able type as a `RawWaker`.\n ///\n /// No heap allocations or atomic operations are used for this conversion.\n fn from(waker: Arc) -> Self {\n raw_waker(waker)\n }\n}\n\n// NB: This private function for constructing a RawWaker is used, rather than\n// inlining this into the `From> for RawWaker` impl, to ensure that\n// the safety of `From> for Waker` does not depend on the correct\n// trait dispatch - instead both impls call this function directly and\n// explicitly.\n#[inline(always)]\nfn raw_waker(waker: Arc) -> RawWaker {\n // Increment the reference count of the arc to clone it.\n //\n // The #[inline(always)] is to ensure that raw_waker and clone_waker are\n // always generated in the same code generation unit as one another, and\n // therefore that the structurally identical const-promoted RawWakerVTable\n // within both functions is deduplicated at LLVM IR code generation time.\n // This allows optimizing Waker::will_wake to a single pointer comparison of\n // the vtable pointers, rather than comparing all four function pointers\n // within the vtables.\n #[inline(always)]\n unsafe fn clone_waker(waker: *const ()) -> RawWaker {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { Arc::increment_strong_count(waker as *const W) }\n RawWaker::new(\n waker,\n &RawWakerVTable::new(clone_waker::, wake::, wake_by_ref::, drop_waker::),\n )\n }\n\n // Wake by value, moving the Arc into the Wake::wake function\n unsafe fn wake(waker: *const ()) {\n // SAFETY: the caller must uphold the safety contract.\n let waker = unsafe { Arc::from_raw(waker as *const W) };\n ::wake(waker);\n }\n\n // Wake by reference, wrap the waker in ManuallyDrop to avoid dropping it\n unsafe fn wake_by_ref(waker: *const ()) {\n // SAFETY: the caller must uphold the safety contract.\n let waker = unsafe { ManuallyDrop::new(Arc::from_raw(waker as *const W)) };\n ::wake_by_ref(&waker);\n }\n\n // Decrement the reference count of the Arc on drop\n unsafe fn drop_waker(waker: *const ()) {\n // SAFETY: the caller must uphold the safety contract.\n unsafe { Arc::decrement_strong_count(waker as *const W) }\n }\n\n RawWaker::new(\n Arc::into_raw(waker) as *const (),\n &RawWakerVTable::new(clone_waker::, wake::, wake_by_ref::, drop_waker::),\n )\n}\n" }, { "package": "portable-atomic-util@0.2.8", "sourcePath": "src/utils.rs", "sha256": "afe133c869f3a833a8a3d35afeb96e4caffc39da5c5bc063501b0293ae0344db", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![cfg_attr(not(feature = \"alloc\"), allow(dead_code, unused_macros, unused_imports))]\n\n// rustfmt-compatible cfg_select/cfg_if alternative\n// Note: This macro is cfg_sel!({ }), not cfg_sel! { }.\n// An extra brace is used in input to make contents rustfmt-able.\nmacro_rules! cfg_sel {\n ({#[cfg(else)] { $($output:tt)* }}) => {\n $($output)*\n };\n ({\n #[cfg($cfg:meta)]\n { $($output:tt)* }\n $($( $rest:tt )+)?\n }) => {\n #[cfg($cfg)]\n cfg_sel! {{#[cfg(else)] { $($output)* }}}\n $(\n #[cfg(not($cfg))]\n cfg_sel! {{ $($rest)+ }}\n )?\n };\n}\n\n// This module provides core::ptr strict_provenance/exposed_provenance polyfill for pre-1.84 rustc.\npub(crate) mod ptr {\n cfg_sel!({\n #[cfg(not(portable_atomic_no_strict_provenance))]\n {\n pub(crate) use core::ptr::without_provenance_mut;\n }\n #[cfg(else)]\n {\n #[inline(always)]\n #[must_use]\n pub(crate) const fn without_provenance_mut(addr: usize) -> *mut T {\n // An int-to-pointer transmute currently has exactly the intended semantics: it creates a\n // pointer without provenance. Note that this is *not* a stable guarantee about transmute\n // semantics, it relies on sysroot crates having special status.\n // SAFETY: every valid integer is also a valid pointer (as long as you don't dereference that\n // pointer).\n #[cfg(miri)]\n unsafe {\n core::mem::transmute(addr)\n }\n // const transmute requires Rust 1.56.\n // Using transmute doesn't work with CHERI: https://github.com/kent-weak-memory/rust/blob/0c0ca909de877f889629057e1ddf139527446d75/library/core/src/ptr/mod.rs#L607\n #[cfg(not(miri))]\n {\n addr as *mut T\n }\n }\n\n pub(crate) trait PtrExt: Copy {\n #[must_use]\n fn addr(self) -> usize;\n }\n impl PtrExt for *const T {\n #[inline(always)]\n #[must_use]\n fn addr(self) -> usize {\n // A pointer-to-integer transmute currently has exactly the right semantics: it returns the\n // address without exposing the provenance. Note that this is *not* a stable guarantee about\n // transmute semantics, it relies on sysroot crates having special status.\n // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the\n // provenance).\n #[cfg(miri)]\n unsafe {\n core::mem::transmute(self as *const ())\n }\n // Using transmute doesn't work with CHERI: https://github.com/kent-weak-memory/rust/blob/0c0ca909de877f889629057e1ddf139527446d75/library/core/src/ptr/mut_ptr.rs#L210\n #[cfg(not(miri))]\n {\n self as *const () as usize\n }\n }\n }\n }\n });\n\n /// Creates a new pointer with the metadata of `other`.\n #[inline]\n #[must_use]\n pub(crate) fn with_metadata_of(this: *mut T, mut other: *const U) -> *mut U {\n let target = &mut other as *mut *const U as *mut *const u8;\n\n // SAFETY: In case of a thin pointer, this operations is identical\n // to a simple assignment. In case of a fat pointer, with the current\n // fat pointer layout implementation, the first field of such a\n // pointer is always the data pointer, which is likewise assigned.\n unsafe { *target = this as *const u8 }\n other as *mut U\n }\n\n // Based on ::byte_add stabilized in Rust 1.75.\n #[inline]\n #[must_use]\n pub(crate) unsafe fn byte_add(ptr: *mut T, count: usize) -> *mut T {\n // SAFETY: the caller must uphold the safety contract for `add`.\n unsafe { with_metadata_of((ptr as *mut u8).add(count), ptr) }\n }\n\n // Based on ::byte_sub stabilized in Rust 1.75.\n #[inline]\n #[must_use]\n pub(crate) unsafe fn byte_sub(ptr: *mut T, count: usize) -> *mut T {\n // SAFETY: the caller must uphold the safety contract for `sub`.\n unsafe { with_metadata_of((ptr as *mut u8).sub(count), ptr) }\n }\n}\n" }, { "package": "portable-atomic-util@0.2.8", "sourcePath": "tests/arc.rs", "sha256": "19963c36b6781078a5882809f4bc7a6c5866931ade681bd983177f05c9c44105", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n#![cfg(any(feature = \"std\", feature = \"alloc\"))]\n#![allow(clippy::undocumented_unsafe_blocks)]\n\nuse std::{borrow::Cow, panic};\n\nuse portable_atomic_util::{Arc, Weak};\n\n#[derive(Debug, PartialEq)]\n#[repr(align(128))]\nstruct Aligned(u32);\n\n// https://github.com/taiki-e/portable-atomic/issues/37\n#[test]\nfn over_aligned() {\n let value = Arc::new(Aligned(128));\n let ptr = Arc::into_raw(value);\n // SAFETY: `ptr` should always be a valid `Aligned`.\n assert_eq!(unsafe { (*ptr).0 }, 128);\n // SAFETY: `ptr` is a valid reference to an `Arc`.\n let value = unsafe { Arc::from_raw(ptr) };\n assert_eq!(value.0, 128);\n}\n\n#[test]\nfn default() {\n let v = Arc::<[Aligned]>::default();\n assert_eq!(v[..], []);\n let v = Arc::<[()]>::default();\n assert_eq!(v[..], []);\n let v = Arc::::default();\n assert_eq!(&v[..], \"\");\n}\n\n#[test]\nfn cow_from() {\n let o = Cow::Owned(\"abc\".to_owned());\n let b = Cow::Borrowed(\"def\");\n let o: Arc = Arc::from(o);\n let b: Arc = Arc::from(b);\n assert_eq!(&*o, \"abc\");\n assert_eq!(&*b, \"def\");\n}\n\n#[test]\nfn make_mut_unsized() {\n let mut v: Arc<[i32]> = Arc::from([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);\n Arc::make_mut(&mut v)[0] += 10;\n assert_eq!(&*v, [11, 2, 3, 4, 5, 6, 7, 8, 9, 10]);\n let v1 = Arc::clone(&v);\n let v2 = Arc::make_mut(&mut v);\n v2[1] += 10;\n assert_eq!(&*v1, [11, 2, 3, 4, 5, 6, 7, 8, 9, 10]);\n assert_eq!(&*v2, [11, 12, 3, 4, 5, 6, 7, 8, 9, 10]);\n assert_eq!(&*v, [11, 12, 3, 4, 5, 6, 7, 8, 9, 10]);\n drop(v1);\n let w = Arc::downgrade(&v);\n Arc::make_mut(&mut v)[2] += 10;\n assert_eq!(&*v, [11, 12, 13, 4, 5, 6, 7, 8, 9, 10]);\n assert!(w.upgrade().is_none());\n}\n\n#[test]\nfn make_mut_clone_panic() {\n struct C(#[allow(dead_code)] Box);\n impl Clone for C {\n fn clone(&self) -> Self {\n panic!()\n }\n }\n let mut v: Arc<[C]> = Arc::from([C(Box::new(1)), C(Box::new(2))]);\n let _v = Arc::make_mut(&mut v);\n let v1 = Arc::clone(&v);\n if !is_panic_abort() {\n panic::catch_unwind(panic::AssertUnwindSafe(|| {\n let _v = Arc::make_mut(&mut v);\n }))\n .unwrap_err();\n }\n drop(v1);\n}\n\n/// Test that a panic from a destructor does not leak the allocation.\n#[test]\nfn panic_no_leak() {\n // use std::alloc::{AllocError, Allocator, Global, Layout};\n use std::panic::{AssertUnwindSafe, catch_unwind};\n // use std::ptr::NonNull;\n\n // TODO: no custom allocator support\n // struct AllocCount(Cell);\n // unsafe impl Allocator for AllocCount {\n // fn allocate(&self, layout: Layout) -> Result, AllocError> {\n // self.0.set(self.0.get() + 1);\n // Global.allocate(layout)\n // }\n // unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) {\n // self.0.set(self.0.get() - 1);\n // unsafe { Global.deallocate(ptr, layout) }\n // }\n // }\n\n struct PanicOnDrop;\n impl Drop for PanicOnDrop {\n fn drop(&mut self) {\n panic!(\"PanicOnDrop\");\n }\n }\n\n if !is_panic_abort() {\n // let alloc = AllocCount(Cell::new(0));\n // let rc = Arc::new_in(PanicOnDrop, &alloc);\n let rc = Arc::new(PanicOnDrop);\n // assert_eq!(alloc.0.get(), 1);\n\n let panic_message = catch_unwind(AssertUnwindSafe(|| drop(rc))).unwrap_err();\n assert_eq!(*panic_message.downcast_ref::<&'static str>().unwrap(), \"PanicOnDrop\");\n // assert_eq!(alloc.0.get(), 0);\n }\n}\n\n#[test]\nfn weak_dangling() {\n let w = Weak::::new();\n let p = Weak::into_raw(w);\n let w = unsafe { Weak::from_raw(p) };\n let w2 = Weak::clone(&w);\n assert!(w.upgrade().is_none());\n assert!(w2.upgrade().is_none());\n}\n\n// For -C panic=abort -Z panic_abort_tests: https://github.com/rust-lang/rust/issues/67650\nfn is_panic_abort() -> bool {\n !matches!(build_context::PANIC, \"unwind\" | \"\") // cfg(panic) requires Rust 1.60\n}\n\n// https://github.com/rust-lang/rust/blob/1.84.0/library/alloc/src/sync/tests.rs\n#[allow(clippy::many_single_char_names)]\nmod alloc_tests {\n use std::{\n convert::TryInto as _,\n sync::{Mutex, mpsc::channel},\n thread,\n };\n\n use portable_atomic::{\n AtomicBool, AtomicUsize,\n Ordering::{Acquire, SeqCst},\n };\n use portable_atomic_util::{Arc, Weak};\n\n struct Canary(*mut AtomicUsize);\n\n impl Drop for Canary {\n fn drop(&mut self) {\n unsafe {\n match *self {\n Canary(c) => {\n (*c).fetch_add(1, SeqCst);\n }\n }\n }\n }\n }\n\n #[test]\n #[cfg_attr(target_os = \"emscripten\", ignore = \"thread::spawn doesn't work on emscripten\")]\n fn manually_share_arc() {\n let v = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10];\n let arc_v = Arc::new(v);\n\n let (tx, rx) = channel();\n\n let _t = thread::spawn(move || {\n let arc_v: Arc> = rx.recv().unwrap();\n assert_eq!((*arc_v)[3], 4);\n });\n\n tx.send(arc_v.clone()).unwrap();\n\n assert_eq!((*arc_v)[2], 3);\n assert_eq!((*arc_v)[4], 5);\n }\n\n #[test]\n fn test_arc_get_mut() {\n let mut x = Arc::new(3);\n *Arc::get_mut(&mut x).unwrap() = 4;\n assert_eq!(*x, 4);\n let y = x.clone();\n assert!(Arc::get_mut(&mut x).is_none());\n drop(y);\n assert!(Arc::get_mut(&mut x).is_some());\n let _w = Arc::downgrade(&x);\n assert!(Arc::get_mut(&mut x).is_none());\n }\n\n #[test]\n fn weak_counts() {\n assert_eq!(Weak::weak_count(&Weak::::new()), 0);\n assert_eq!(Weak::strong_count(&Weak::::new()), 0);\n\n let a = Arc::new(0);\n let w = Arc::downgrade(&a);\n assert_eq!(Weak::strong_count(&w), 1);\n assert_eq!(Weak::weak_count(&w), 1);\n let w2 = w.clone();\n assert_eq!(Weak::strong_count(&w), 1);\n assert_eq!(Weak::weak_count(&w), 2);\n assert_eq!(Weak::strong_count(&w2), 1);\n assert_eq!(Weak::weak_count(&w2), 2);\n drop(w);\n assert_eq!(Weak::strong_count(&w2), 1);\n assert_eq!(Weak::weak_count(&w2), 1);\n let a2 = a.clone();\n assert_eq!(Weak::strong_count(&w2), 2);\n assert_eq!(Weak::weak_count(&w2), 1);\n drop(a2);\n drop(a);\n assert_eq!(Weak::strong_count(&w2), 0);\n assert_eq!(Weak::weak_count(&w2), 0);\n drop(w2);\n }\n\n #[test]\n fn try_unwrap() {\n let x = Arc::new(3);\n assert_eq!(Arc::try_unwrap(x), Ok(3));\n let x = Arc::new(4);\n let _y = x.clone();\n assert_eq!(Arc::try_unwrap(x), Err(Arc::new(4)));\n let x = Arc::new(5);\n let _w = Arc::downgrade(&x);\n assert_eq!(Arc::try_unwrap(x), Ok(5));\n }\n\n #[test]\n fn into_inner() {\n for _ in 0..100\n // ^ Increase chances of hitting potential race conditions\n {\n let x = Arc::new(3);\n let y = Arc::clone(&x);\n let r_thread = std::thread::spawn(|| Arc::into_inner(x));\n let s_thread = std::thread::spawn(|| Arc::into_inner(y));\n let r = r_thread.join().expect(\"r_thread panicked\");\n let s = s_thread.join().expect(\"s_thread panicked\");\n assert!(\n matches!((r, s), (None, Some(3)) | (Some(3), None)),\n \"assertion failed: unexpected result `{:?}`\\\n \\n expected `(None, Some(3))` or `(Some(3), None)`\",\n (r, s),\n );\n }\n\n let x = Arc::new(3);\n assert_eq!(Arc::into_inner(x), Some(3));\n\n let x = Arc::new(4);\n let y = Arc::clone(&x);\n assert_eq!(Arc::into_inner(x), None);\n assert_eq!(Arc::into_inner(y), Some(4));\n\n let x = Arc::new(5);\n let _w = Arc::downgrade(&x);\n assert_eq!(Arc::into_inner(x), Some(5));\n }\n\n #[test]\n fn into_from_raw() {\n let x = Arc::new(Box::new(\"hello\"));\n let y = x.clone();\n\n let x_ptr = Arc::into_raw(x);\n drop(y);\n unsafe {\n assert_eq!(**x_ptr, \"hello\");\n\n let x = Arc::from_raw(x_ptr);\n assert_eq!(**x, \"hello\");\n\n assert_eq!(Arc::try_unwrap(x).map(|x| *x), Ok(\"hello\"));\n }\n }\n\n #[test]\n fn test_into_from_raw_unsized() {\n use std::fmt::Display;\n\n let arc: Arc = Arc::from(\"foo\");\n\n let ptr = Arc::into_raw(arc.clone());\n let arc2 = unsafe { Arc::from_raw(ptr) };\n\n assert_eq!(unsafe { &*ptr }, \"foo\");\n assert_eq!(arc, arc2);\n\n #[cfg(portable_atomic_unstable_coerce_unsized)]\n let arc: Arc = Arc::new(123);\n // TODO: This is a workaround in case CoerceUnsized is not available - remove this once it is no longer needed\n #[cfg(not(portable_atomic_unstable_coerce_unsized))]\n let arc: Arc = Arc::from(Box::new(123) as Box);\n\n let ptr = Arc::into_raw(arc.clone());\n let arc2 = unsafe { Arc::from_raw(ptr) };\n\n assert_eq!(unsafe { &*ptr }.to_string(), \"123\");\n assert_eq!(arc2.to_string(), \"123\");\n }\n\n #[test]\n fn into_from_weak_raw() {\n let x = Arc::new(Box::new(\"hello\"));\n let y = Arc::downgrade(&x);\n\n let y_ptr = Weak::into_raw(y);\n unsafe {\n assert_eq!(**y_ptr, \"hello\");\n\n let y = Weak::from_raw(y_ptr);\n let y_up = Weak::upgrade(&y).unwrap();\n assert_eq!(**y_up, \"hello\");\n drop(y_up);\n\n assert_eq!(Arc::try_unwrap(x).map(|x| *x), Ok(\"hello\"));\n }\n }\n\n // TODO: See Weak::from_raw\n // #[test]\n // fn test_into_from_weak_raw_unsized() {\n // use std::fmt::Display;\n\n // let arc: Arc = Arc::from(\"foo\");\n // let weak: Weak = Arc::downgrade(&arc);\n\n // let ptr = Weak::into_raw(weak.clone());\n // let weak2 = unsafe { Weak::from_raw(ptr) };\n\n // assert_eq!(unsafe { &*ptr }, \"foo\");\n // assert!(weak.ptr_eq(&weak2));\n\n // // TODO: CoerceUnsized is needed to cast to Arc\n // // (may be possible to support this with portable_atomic_unstable_coerce_unsized cfg option)\n // // let arc: Arc = Arc::new(123);\n // let arc: Arc = Arc::from(Box::new(123) as Box);\n // let weak: Weak = Arc::downgrade(&arc);\n\n // let ptr = Weak::into_raw(weak.clone());\n // let weak2 = unsafe { Weak::from_raw(ptr) };\n\n // assert_eq!(unsafe { &*ptr }.to_string(), \"123\");\n // assert!(weak.ptr_eq(&weak2));\n // }\n\n #[test]\n fn test_cow_arc_clone_make_mut() {\n let mut cow0 = Arc::new(75);\n let mut cow1 = cow0.clone();\n let mut cow2 = cow1.clone();\n\n assert_eq!(75, *Arc::make_mut(&mut cow0));\n assert_eq!(75, *Arc::make_mut(&mut cow1));\n assert_eq!(75, *Arc::make_mut(&mut cow2));\n\n *Arc::make_mut(&mut cow0) += 1;\n *Arc::make_mut(&mut cow1) += 2;\n *Arc::make_mut(&mut cow2) += 3;\n\n assert_eq!(76, *cow0);\n assert_eq!(77, *cow1);\n assert_eq!(78, *cow2);\n\n // none should point to the same backing memory\n assert_ne!(*cow0, *cow1);\n assert_ne!(*cow0, *cow2);\n assert_ne!(*cow1, *cow2);\n }\n\n #[test]\n fn test_cow_arc_clone_unique2() {\n let mut cow0 = Arc::new(75);\n let cow1 = cow0.clone();\n let cow2 = cow1.clone();\n\n assert_eq!(75, *cow0);\n assert_eq!(75, *cow1);\n assert_eq!(75, *cow2);\n\n *Arc::make_mut(&mut cow0) += 1;\n assert_eq!(76, *cow0);\n assert_eq!(75, *cow1);\n assert_eq!(75, *cow2);\n\n // cow1 and cow2 should share the same contents\n // cow0 should have a unique reference\n assert_ne!(*cow0, *cow1);\n assert_ne!(*cow0, *cow2);\n assert_eq!(*cow1, *cow2);\n }\n\n #[test]\n fn test_cow_arc_clone_weak() {\n let mut cow0 = Arc::new(75);\n let cow1_weak = Arc::downgrade(&cow0);\n\n assert_eq!(75, *cow0);\n assert_eq!(75, *cow1_weak.upgrade().unwrap());\n\n *Arc::make_mut(&mut cow0) += 1;\n\n assert_eq!(76, *cow0);\n assert!(cow1_weak.upgrade().is_none());\n }\n\n #[test]\n fn test_live() {\n let x = Arc::new(5);\n let y = Arc::downgrade(&x);\n assert!(y.upgrade().is_some());\n }\n\n #[test]\n fn test_dead() {\n let x = Arc::new(5);\n let y = Arc::downgrade(&x);\n drop(x);\n assert!(y.upgrade().is_none());\n }\n\n #[test]\n fn weak_self_cyclic() {\n struct Cycle {\n x: Mutex>>,\n }\n\n let a = Arc::new(Cycle { x: Mutex::new(None) });\n let b = Arc::downgrade(&a.clone());\n *a.x.lock().unwrap() = Some(b);\n\n // hopefully we don't double-free (or leak)...\n }\n\n #[test]\n fn drop_arc() {\n let mut canary = AtomicUsize::new(0);\n let x = Arc::new(Canary(&mut canary as *mut AtomicUsize));\n drop(x);\n assert_eq!(canary.load(Acquire), 1);\n }\n\n #[test]\n fn drop_arc_weak() {\n let mut canary = AtomicUsize::new(0);\n let arc = Arc::new(Canary(&mut canary as *mut AtomicUsize));\n let arc_weak = Arc::downgrade(&arc);\n assert_eq!(canary.load(Acquire), 0);\n drop(arc);\n assert_eq!(canary.load(Acquire), 1);\n drop(arc_weak);\n }\n\n #[test]\n fn test_strong_count() {\n let a = Arc::new(0);\n assert_eq!(Arc::strong_count(&a), 1);\n let w = Arc::downgrade(&a);\n assert_eq!(Arc::strong_count(&a), 1);\n let b = w.upgrade().expect(\"\");\n assert_eq!(Arc::strong_count(&b), 2);\n assert_eq!(Arc::strong_count(&a), 2);\n drop(w);\n drop(a);\n assert_eq!(Arc::strong_count(&b), 1);\n let c = b.clone();\n assert_eq!(Arc::strong_count(&b), 2);\n assert_eq!(Arc::strong_count(&c), 2);\n }\n\n #[test]\n fn test_weak_count() {\n let a = Arc::new(0);\n assert_eq!(Arc::strong_count(&a), 1);\n assert_eq!(Arc::weak_count(&a), 0);\n let w = Arc::downgrade(&a);\n assert_eq!(Arc::strong_count(&a), 1);\n assert_eq!(Arc::weak_count(&a), 1);\n let x = w.clone();\n assert_eq!(Arc::weak_count(&a), 2);\n drop(w);\n drop(x);\n assert_eq!(Arc::strong_count(&a), 1);\n assert_eq!(Arc::weak_count(&a), 0);\n let c = a.clone();\n assert_eq!(Arc::strong_count(&a), 2);\n assert_eq!(Arc::weak_count(&a), 0);\n let d = Arc::downgrade(&c);\n assert_eq!(Arc::weak_count(&c), 1);\n assert_eq!(Arc::strong_count(&c), 2);\n\n drop(a);\n drop(c);\n drop(d);\n }\n\n #[test]\n fn show_arc() {\n let a = Arc::new(5);\n assert_eq!(format!(\"{:?}\", a), \"5\");\n }\n\n // Make sure deriving works with Arc\n #[derive(Eq, Ord, PartialEq, PartialOrd, Clone, Debug, Default)]\n struct _Foo {\n inner: Arc,\n }\n\n #[test]\n fn test_unsized() {\n #[cfg(portable_atomic_unstable_coerce_unsized)]\n let x: Arc<[i32]> = Arc::new([1, 2, 3]);\n // TODO: This is a workaround in case CoerceUnsized is not available - remove this once it is no longer needed\n #[cfg(not(portable_atomic_unstable_coerce_unsized))]\n let x: Arc<[i32]> = Arc::from(Box::new([1, 2, 3]) as Box<[i32]>);\n assert_eq!(format!(\"{:?}\", x), \"[1, 2, 3]\");\n let y = Arc::downgrade(&x.clone());\n drop(x);\n assert!(y.upgrade().is_none());\n }\n\n #[test]\n fn test_maybe_thin_unsized() {\n // If/when custom thin DSTs exist, this test should be updated to use one\n use std::ffi::{CStr, CString};\n\n let x: Arc = Arc::from(CString::new(\"swordfish\").unwrap().into_boxed_c_str());\n assert_eq!(format!(\"{:?}\", x), \"\\\"swordfish\\\"\");\n let y: Weak = Arc::downgrade(&x);\n drop(x);\n\n // At this point, the weak points to a dropped DST\n assert!(y.upgrade().is_none());\n // But we still need to be able to get the alloc layout to drop.\n // CStr has no drop glue, but custom DSTs might, and need to work.\n drop(y);\n }\n\n #[test]\n fn test_from_owned() {\n let foo = 123;\n let foo_arc = Arc::from(foo);\n assert_eq!(123, *foo_arc);\n }\n\n #[test]\n fn test_new_weak() {\n let foo: Weak = Weak::new();\n assert!(foo.upgrade().is_none());\n }\n\n #[test]\n fn test_ptr_eq() {\n let five = Arc::new(5);\n let same_five = five.clone();\n let other_five = Arc::new(5);\n\n assert!(Arc::ptr_eq(&five, &same_five));\n assert!(!Arc::ptr_eq(&five, &other_five));\n }\n\n #[test]\n #[cfg_attr(target_os = \"emscripten\", ignore = \"thread::spawn doesn't work on emscripten\")]\n fn test_weak_count_locked() {\n let mut a = Arc::new(AtomicBool::new(false));\n let a2 = a.clone();\n let t = thread::spawn(move || {\n // Miri is too slow\n let count = if cfg!(miri) { 1_000 } else { 1_000_000 };\n for _i in 0..count {\n Arc::get_mut(&mut a);\n }\n a.store(true, SeqCst);\n });\n\n while !a2.load(SeqCst) {\n let n = Arc::weak_count(&a2);\n assert!(n < 2, \"bad weak count: {}\", n);\n #[cfg(miri)] // Miri's scheduler does not guarantee liveness, and thus needs this hint.\n std::hint::spin_loop();\n }\n t.join().unwrap();\n }\n\n #[test]\n fn test_from_str() {\n let r: Arc = Arc::from(\"foo\");\n\n assert_eq!(&r[..], \"foo\");\n }\n\n #[test]\n fn test_copy_from_slice() {\n let s: &[u32] = &[1, 2, 3];\n let r: Arc<[u32]> = Arc::from(s);\n\n assert_eq!(&r[..], [1, 2, 3]);\n }\n\n #[test]\n fn test_clone_from_slice() {\n #[derive(Clone, Debug, Eq, PartialEq)]\n struct X(u32);\n\n let s: &[X] = &[X(1), X(2), X(3)];\n let r: Arc<[X]> = Arc::from(s);\n\n assert_eq!(&r[..], s);\n }\n\n #[test]\n #[should_panic = \"test_clone_from_slice_panic\"]\n fn test_clone_from_slice_panic() {\n struct Fail(u32, String);\n\n impl Clone for Fail {\n fn clone(&self) -> Fail {\n if self.0 == 2 {\n panic!(\"test_clone_from_slice_panic\");\n }\n Fail(self.0, self.1.clone())\n }\n }\n\n let s: &[Fail] =\n &[Fail(0, \"foo\".to_owned()), Fail(1, \"bar\".to_owned()), Fail(2, \"baz\".to_owned())];\n\n // Should panic, but not cause memory corruption\n let _r: Arc<[Fail]> = Arc::from(s);\n }\n\n #[test]\n fn test_from_box() {\n let b: Box = Box::new(123);\n let r: Arc = Arc::from(b);\n\n assert_eq!(*r, 123);\n }\n\n #[test]\n fn test_from_box_str() {\n let s = String::from(\"foo\").into_boxed_str();\n let r: Arc = Arc::from(s);\n\n assert_eq!(&r[..], \"foo\");\n }\n\n #[test]\n fn test_from_box_slice() {\n let s = vec![1, 2, 3].into_boxed_slice();\n let r: Arc<[u32]> = Arc::from(s);\n\n assert_eq!(&r[..], [1, 2, 3]);\n }\n\n #[test]\n fn test_from_box_trait() {\n use std::fmt::Display;\n\n let b: Box = Box::new(123);\n let r: Arc = Arc::from(b);\n\n assert_eq!(r.to_string(), \"123\");\n }\n\n #[test]\n fn test_from_box_trait_zero_sized() {\n use std::fmt::Debug;\n\n let b: Box = Box::new(());\n let r: Arc = Arc::from(b);\n\n assert_eq!(format!(\"{:?}\", r), \"()\");\n }\n\n #[test]\n fn test_from_vec() {\n let v = vec![1, 2, 3];\n let r: Arc<[u32]> = Arc::from(v);\n\n assert_eq!(&r[..], [1, 2, 3]);\n }\n\n #[test]\n fn test_downcast() {\n use std::any::Any;\n\n #[cfg(portable_atomic_unstable_coerce_unsized)]\n let r1: Arc = Arc::new(i32::MAX);\n // TODO: This is a workaround in case CoerceUnsized is not available - remove this once it is no longer needed\n #[cfg(not(portable_atomic_unstable_coerce_unsized))]\n let r1: Arc =\n Arc::from(Box::new(i32::MAX) as Box);\n\n #[cfg(portable_atomic_unstable_coerce_unsized)]\n let r2: Arc = Arc::new(\"abc\");\n // TODO: This is a workaround in case CoerceUnsized is not available - remove this once it is no longer needed\n #[cfg(not(portable_atomic_unstable_coerce_unsized))]\n let r2: Arc =\n Arc::from(Box::new(\"abc\") as Box);\n\n assert!(r1.clone().downcast::().is_err());\n\n let r1i32 = r1.downcast::();\n assert!(r1i32.is_ok());\n assert_eq!(r1i32.unwrap(), Arc::new(i32::MAX));\n\n assert!(r2.clone().downcast::().is_err());\n\n let r2str = r2.downcast::<&'static str>();\n assert!(r2str.is_ok());\n assert_eq!(r2str.unwrap(), Arc::new(\"abc\"));\n }\n\n #[test]\n fn test_array_from_slice() {\n let v = vec![1, 2, 3];\n let r: Arc<[u32]> = Arc::from(v);\n\n let a: Result, _> = r.clone().try_into();\n assert!(a.is_ok());\n\n let a: Result, _> = r.clone().try_into();\n assert!(a.is_err());\n }\n\n #[test]\n fn test_arc_cyclic_with_zero_refs() {\n struct ZeroRefs {\n inner: Weak,\n }\n let zero_refs = Arc::new_cyclic(|inner| {\n assert_eq!(inner.strong_count(), 0);\n assert!(inner.upgrade().is_none());\n ZeroRefs { inner: Weak::new() }\n });\n\n assert_eq!(Arc::strong_count(&zero_refs), 1);\n assert_eq!(Arc::weak_count(&zero_refs), 0);\n assert_eq!(zero_refs.inner.strong_count(), 0);\n assert_eq!(zero_refs.inner.weak_count(), 0);\n }\n\n #[test]\n fn test_arc_new_cyclic_one_ref() {\n struct OneRef {\n inner: Weak,\n }\n let one_ref = Arc::new_cyclic(|inner| {\n assert_eq!(inner.strong_count(), 0);\n assert!(inner.upgrade().is_none());\n OneRef { inner: inner.clone() }\n });\n\n assert_eq!(Arc::strong_count(&one_ref), 1);\n assert_eq!(Arc::weak_count(&one_ref), 1);\n\n let one_ref2 = Weak::upgrade(&one_ref.inner).unwrap();\n assert!(Arc::ptr_eq(&one_ref, &one_ref2));\n\n assert_eq!(Arc::strong_count(&one_ref), 2);\n assert_eq!(Arc::weak_count(&one_ref), 1);\n }\n\n #[test]\n fn test_arc_cyclic_two_refs() {\n struct TwoRefs {\n inner1: Weak,\n inner2: Weak,\n }\n let two_refs = Arc::new_cyclic(|inner| {\n assert_eq!(inner.strong_count(), 0);\n assert!(inner.upgrade().is_none());\n\n let inner1 = inner.clone();\n let inner2 = inner1.clone();\n\n TwoRefs { inner1, inner2 }\n });\n\n assert_eq!(Arc::strong_count(&two_refs), 1);\n assert_eq!(Arc::weak_count(&two_refs), 2);\n\n let two_refs1 = Weak::upgrade(&two_refs.inner1).unwrap();\n assert!(Arc::ptr_eq(&two_refs, &two_refs1));\n\n let two_refs2 = Weak::upgrade(&two_refs.inner2).unwrap();\n assert!(Arc::ptr_eq(&two_refs, &two_refs2));\n\n assert_eq!(Arc::strong_count(&two_refs), 3);\n assert_eq!(Arc::weak_count(&two_refs), 2);\n }\n\n /// Test for Arc::drop bug (https://github.com/rust-lang/rust/issues/55005)\n #[test]\n #[cfg(miri)] // relies on Stacked Borrows in Miri\n fn arc_drop_dereferenceable_race() {\n // The bug seems to take up to 700 iterations to reproduce with most seeds (tested 0-9).\n for _ in 0..750 {\n let arc_1 = Arc::new(());\n let arc_2 = arc_1.clone();\n let thread = thread::spawn(|| drop(arc_2));\n // Spin a bit; makes the race more likely to appear\n let mut i = 0;\n while i < 256 {\n i += 1;\n }\n drop(arc_1);\n thread.join().unwrap();\n }\n }\n}\n" }, { "package": "proc-macro2@1.0.107", "sourcePath": "README.md", "sha256": "8eebdcf2f9a5034693589a9982adf06c2aac5c9f078c07a44400d293ed5b4a8e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# proc-macro2\n\n[\"github\"](https://github.com/dtolnay/proc-macro2)\n[\"crates.io\"](https://crates.io/crates/proc-macro2)\n[\"docs.rs\"](https://docs.rs/proc-macro2)\n[\"build](https://github.com/dtolnay/proc-macro2/actions?query=branch%3Amaster)\n\nA wrapper around the procedural macro API of the compiler's `proc_macro` crate.\nThis library serves two purposes:\n\n- **Bring proc-macro-like functionality to other contexts like build.rs and\n main.rs.** Types from `proc_macro` are entirely specific to procedural macros\n and cannot ever exist in code outside of a procedural macro. Meanwhile\n `proc_macro2` types may exist anywhere including non-macro code. By developing\n foundational libraries like [syn] and [quote] against `proc_macro2` rather\n than `proc_macro`, the procedural macro ecosystem becomes easily applicable to\n many other use cases and we avoid reimplementing non-macro equivalents of\n those libraries.\n\n- **Make procedural macros unit testable.** As a consequence of being specific\n to procedural macros, nothing that uses `proc_macro` can be executed from a\n unit test. In order for helper libraries or components of a macro to be\n testable in isolation, they must be implemented using `proc_macro2`.\n\n[syn]: https://github.com/dtolnay/syn\n[quote]: https://github.com/dtolnay/quote\n\n## Usage\n\n```toml\n[dependencies]\nproc-macro2 = \"1.0\"\n```\n\nThe skeleton of a typical procedural macro typically looks like this:\n\n```rust\nextern crate proc_macro;\n\n#[proc_macro_derive(MyDerive)]\npub fn my_derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream {\n let input = proc_macro2::TokenStream::from(input);\n\n let output: proc_macro2::TokenStream = {\n /* transform input */\n };\n\n proc_macro::TokenStream::from(output)\n}\n```\n\nIf parsing with [Syn], you'll use [`parse_macro_input!`] instead to propagate\nparse errors correctly back to the compiler when parsing fails.\n\n[`parse_macro_input!`]: https://docs.rs/syn/3/syn/macro.parse_macro_input.html\n\n## Unstable features\n\nThe default feature set of proc-macro2 tracks the most recent stable compiler\nAPI. Functionality in `proc_macro` that is not yet stable is not exposed by\nproc-macro2 by default.\n\nTo opt into the additional APIs available in the most recent nightly compiler,\nthe `procmacro2_semver_exempt` config flag must be passed to rustc. We will\npolyfill those nightly-only APIs back to Rust 1.71.0. As these are unstable APIs\nthat track the nightly compiler, minor versions of proc-macro2 may make breaking\nchanges to them at any time.\n\n```\nRUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build\n```\n\nNote that this must not only be done for your crate, but for any crate that\ndepends on your crate. This infectious nature is intentional, as it serves as a\nreminder that you are outside of the normal semver guarantees.\n\nSemver exempt methods are marked as such in the proc-macro2 documentation.\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this crate by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "quote@1.0.47", "sourcePath": "README.md", "sha256": "9d6af721967088b8263c0544c0c3c459206998252dd5632a3bdf87cb27ec92be", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Rust Quasi-Quoting\n==================\n\n[\"github\"](https://github.com/dtolnay/quote)\n[\"crates.io\"](https://crates.io/crates/quote)\n[\"docs.rs\"](https://docs.rs/quote)\n[\"build](https://github.com/dtolnay/quote/actions?query=branch%3Amaster)\n\nThis crate provides the [`quote!`] macro for turning Rust syntax tree data\nstructures into tokens of source code.\n\n[`quote!`]: https://docs.rs/quote/1.0/quote/macro.quote.html\n\nProcedural macros in Rust receive a stream of tokens as input, execute arbitrary\nRust code to determine how to manipulate those tokens, and produce a stream of\ntokens to hand back to the compiler to compile into the caller's crate.\nQuasi-quoting is a solution to one piece of that — producing tokens to\nreturn to the compiler.\n\nThe idea of quasi-quoting is that we write *code* that we treat as *data*.\nWithin the `quote!` macro, we can write what looks like code to our text editor\nor IDE. We get all the benefits of the editor's brace matching, syntax\nhighlighting, indentation, and maybe autocompletion. But rather than compiling\nthat as code into the current crate, we can treat it as data, pass it around,\nmutate it, and eventually hand it back to the compiler as tokens to compile into\nthe macro caller's crate.\n\nThis crate is motivated by the procedural macro use case, but is a\ngeneral-purpose Rust quasi-quoting library and is not specific to procedural\nmacros.\n\n```toml\n[dependencies]\nquote = \"1.0\"\n```\n\n[*Release notes*](https://github.com/dtolnay/quote/releases)\n\n
\n\n## Syntax\n\nThe quote crate provides a [`quote!`] macro within which you can write Rust code\nthat gets packaged into a [`TokenStream`] and can be treated as data. You should\nthink of `TokenStream` as representing a fragment of Rust source code.\n\n[`TokenStream`]: https://docs.rs/proc-macro2/1.0/proc_macro2/struct.TokenStream.html\n\nWithin the `quote!` macro, interpolation is done with `#var`. Any type\nimplementing the [`quote::ToTokens`] trait can be interpolated. This includes\nmost Rust primitive types as well as most of the syntax tree types from [`syn`].\n\n[`quote::ToTokens`]: https://docs.rs/quote/1.0/quote/trait.ToTokens.html\n[`syn`]: https://github.com/dtolnay/syn\n\n```rust\nlet tokens = quote! {\n struct SerializeWith #generics #where_clause {\n value: &'a #field_ty,\n phantom: core::marker::PhantomData<#item_ty>,\n }\n\n impl #generics serde::Serialize for SerializeWith #generics #where_clause {\n fn serialize(&self, serializer: S) -> Result\n where\n S: serde::Serializer,\n {\n #path(self.value, serializer)\n }\n }\n\n SerializeWith {\n value: #value,\n phantom: core::marker::PhantomData::<#item_ty>,\n }\n};\n```\n\n
\n\n## Repetition\n\nRepetition is done using `#(...)*` or `#(...),*` similar to `macro_rules!`. This\niterates through the elements of any variable interpolated within the repetition\nand inserts a copy of the repetition body for each one. The variables in an\ninterpolation may be a `Vec`, slice, `BTreeSet`, or any `Iterator`.\n\n- `#(#var)*` — no separators\n- `#(#var),*` — the character before the asterisk is used as a separator\n- `#( struct #var; )*` — the repetition can contain other things\n- `#( #k => println!(\"{}\", #v), )*` — even multiple interpolations\n\nNote that there is a difference between `#(#var ,)*` and `#(#var),*`—the latter\ndoes not produce a trailing comma. This matches the behavior of delimiters in\n`macro_rules!`.\n\n
\n\n## Returning tokens to the compiler\n\nThe `quote!` macro evaluates to an expression of type\n`proc_macro2::TokenStream`. Meanwhile Rust procedural macros are expected to\nreturn the type `proc_macro::TokenStream`.\n\nThe difference between the two types is that `proc_macro` types are entirely\nspecific to procedural macros and cannot ever exist in code outside of a\nprocedural macro, while `proc_macro2` types may exist anywhere including tests\nand non-macro code like main.rs and build.rs. This is why even the procedural\nmacro ecosystem is largely built around `proc_macro2`, because that ensures the\nlibraries are unit testable and accessible in non-macro contexts.\n\nThere is a [`From`]-conversion in both directions so returning the output of\n`quote!` from a procedural macro usually looks like `tokens.into()` or\n`proc_macro::TokenStream::from(tokens)`.\n\n[`From`]: https://doc.rust-lang.org/std/convert/trait.From.html\n\n
\n\n## Examples\n\n### Combining quoted fragments\n\nUsually you don't end up constructing an entire final `TokenStream` in one\npiece. Different parts may come from different helper functions. The tokens\nproduced by `quote!` themselves implement `ToTokens` and so can be interpolated\ninto later `quote!` invocations to build up a final result.\n\n```rust\nlet type_definition = quote! {...};\nlet methods = quote! {...};\n\nlet tokens = quote! {\n #type_definition\n #methods\n};\n```\n\n### Constructing identifiers\n\nSuppose we have an identifier `ident` which came from somewhere in a macro\ninput and we need to modify it in some way for the macro output. Let's consider\nprepending the identifier with an underscore.\n\nSimply interpolating the identifier next to an underscore will not have the\nbehavior of concatenating them. The underscore and the identifier will continue\nto be two separate tokens as if you had written `_ x`.\n\n```rust\n// incorrect\nquote! {\n let mut _#ident = 0;\n}\n```\n\nThe solution is to build a new identifier token with the correct value. As this\nis such a common case, the `format_ident!` macro provides a convenient utility\nfor doing so correctly.\n\n```rust\nlet varname = format_ident!(\"_{}\", ident);\nquote! {\n let mut #varname = 0;\n}\n```\n\nAlternatively, the APIs provided by Syn and proc-macro2 can be used to directly\nbuild the identifier. This is roughly equivalent to the above, but will not\nhandle `ident` being a raw identifier.\n\n```rust\nlet concatenated = format!(\"_{}\", ident);\nlet varname = syn::Ident::new(&concatenated, ident.span());\nquote! {\n let mut #varname = 0;\n}\n```\n\n### Making method calls\n\nLet's say our macro requires some type specified in the macro input to have a\nconstructor called `new`. We have the type in a variable called `field_type` of\ntype `syn::Type` and want to invoke the constructor.\n\n```rust\n// incorrect\nquote! {\n let value = #field_type::new();\n}\n```\n\nThis works only sometimes. If `field_type` is `String`, the expanded code\ncontains `String::new()` which is fine. But if `field_type` is something like\n`Vec` then the expanded code is `Vec::new()` which is invalid syntax.\nOrdinarily in handwritten Rust we would write `Vec::::new()` but for macros\noften the following is more convenient.\n\n```rust\nquote! {\n let value = <#field_type>::new();\n}\n```\n\nThis expands to `>::new()` which behaves correctly.\n\nA similar pattern is appropriate for trait methods.\n\n```rust\nquote! {\n let value = <#field_type as core::default::Default>::default();\n}\n```\n\n
\n\n## Hygiene\n\nAny interpolated tokens preserve the `Span` information provided by their\n`ToTokens` implementation. Tokens that originate within a `quote!` invocation\nare spanned with [`Span::call_site()`].\n\n[`Span::call_site()`]: https://docs.rs/proc-macro2/1.0/proc_macro2/struct.Span.html#method.call_site\n\nA different span can be provided explicitly through the [`quote_spanned!`]\nmacro.\n\n[`quote_spanned!`]: https://docs.rs/quote/1.0/quote/macro.quote_spanned.html\n\n
\n\n## Non-macro code generators\n\nWhen using `quote` in a build.rs or main.rs and writing the output out to a\nfile, consider having the code generator pass the tokens through [prettyplease]\nbefore writing. This way if an error occurs in the generated code it is\nconvenient for a human to read and debug.\n\nBe aware that no kind of hygiene or span information is retained when tokens are\nwritten to a file; the conversion from tokens to source code is lossy.\n\nExample usage in build.rs:\n\n```rust\nlet output = quote! { ... };\nlet syntax_tree = syn::parse2(output).unwrap();\nlet formatted = prettyplease::unparse(&syntax_tree);\n\nlet out_dir = env::var_os(\"OUT_DIR\").unwrap();\nlet dest_path = Path::new(&out_dir).join(\"out.rs\");\nfs::write(dest_path, formatted).unwrap();\n```\n\n[prettyplease]: https://github.com/dtolnay/prettyplease\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this crate by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "radium@0.7.0", "sourcePath": "LICENSE.txt", "sha256": "13f4cc9fbc8d4a447b28aa84019c10ad4abf4b5f6919db061bf6690ccc23bc02", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "MIT License\n\nCopyright (c) 2019 kneecaw (Nika Layzell)\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "package": "rctree@0.5.0", "sourcePath": "README.md", "sha256": "8b89a6f85cd86ba8525d96a9fe29d9907e3c8f8e000eda366987bd590dbfa4a3", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# rctree\n![Build Status](https://github.com/RazrFalcon/rctree/workflows/rctree/badge.svg)\n[![Crates.io](https://img.shields.io/crates/v/rctree.svg)](https://crates.io/crates/rctree)\n[![Documentation](https://docs.rs/rctree/badge.svg)](https://docs.rs/rctree)\n[![Rust 1.22+](https://img.shields.io/badge/rust-1.22+-orange.svg)](https://www.rust-lang.org)\n![](https://img.shields.io/badge/unsafe-forbidden-brightgreen.svg)\n\n*rctree* is a \"DOM-like\" tree implemented using reference counting.\n\n### Origin\n\nThis is a fork of the [rust-forest](https://github.com/SimonSapin/rust-forest) rctree.\n\n### Details\n\n\"DOM-like\" here means that data structures can be used to represent\nthe parsed content of an HTML or XML document,\nlike [*the* DOM](https://dom.spec.whatwg.org/) does,\nbut don't necessarily have the exact same API as the DOM.\nThat is:\n\n* A tree is made up of nodes.\n* Each node has zero or more *child* nodes, which are ordered.\n* Each node has a no more than one *parent*, the node that it is a *child* of.\n* A node without a *parent* is called a *root*.\n* As a consequence, each node may also have *siblings*: its *parent*'s other *children*, if any.\n* From any given node, access to its\n parent, previous sibling, next sibling, first child, and last child (if any)\n can take no more than *O(1)* time.\n* Each node also has data associated to it,\n which for the purpose of this project is purely generic.\n For an HTML document, the data would be either the text of a text node,\n or the name and attributes of an element node.\n* The tree is mutable:\n nodes (with their sub-trees) can be inserted or removed anywhere in the tree.\n\nThe lifetime of nodes is managed through *reference counting*.\nTo avoid reference cycles which would cause memory leaks, the tree is *asymmetric*:\neach node holds optional *strong references* to its next sibling and first child,\nbut only optional *weak references* to its parent, previous sibling, and last child.\n\nNodes are destroyed as soon as there is no strong reference left to them.\nThe structure is such that holding a reference to the root\nis sufficient to keep the entire tree alive.\nHowever, if for example the only reference that exists from outside the tree\nis one that you use to traverse it,\nyou will not be able to go back \"up\" the tree to ancestors and previous siblings after going \"down\",\nas those nodes will have been destroyed.\n\nWeak references to destroyed nodes are treated as if they were not set at all.\n(E.g. a node can become a root when its parent is destroyed.)\n\nSince nodes are *aliased* (have multiple references to them),\n[`RefCell`](http://doc.rust-lang.org/std/cell/index.html) is used for interior mutability.\n\nAdvantages:\n\n* A single `Node` user-visible type to manipulate the tree, with methods.\n* Memory is freed as soon as it becomes unused (if parts of the tree are removed).\n\nDisadvantages:\n\n* The tree can only be accessed from the thread is was created in.\n* Any tree manipulation, including read-only traversals,\n requires incrementing and decrementing reference counts,\n which causes run-time overhead.\n* Nodes are allocated individually, which may cause memory fragmentation and hurt performance.\n\n### License\n\n*rctree* is licensed under the **MIT**.\n" }, { "package": "regex@1.13.1", "sourcePath": "README.md", "sha256": "ce1a8faf72cf8f0e28a28658c59033b02726e3caecd9a3d358b4c6f6cd3e6c24", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "regex\n=====\nThis crate provides routines for searching strings for matches of a [regular\nexpression] (aka \"regex\"). The regex syntax supported by this crate is similar\nto other regex engines, but it lacks several features that are not known how to\nimplement efficiently. This includes, but is not limited to, look-around and\nbackreferences. In exchange, all regex searches in this crate have worst case\n`O(m * n)` time complexity, where `m` is proportional to the size of the regex\nand `n` is proportional to the size of the string being searched.\n\n[regular expression]: https://en.wikipedia.org/wiki/Regular_expression\n\n[![Build status](https://github.com/rust-lang/regex/workflows/ci/badge.svg)](https://github.com/rust-lang/regex/actions)\n[![Crates.io](https://img.shields.io/crates/v/regex.svg)](https://crates.io/crates/regex)\n\n### Documentation\n\n[Module documentation with examples](https://docs.rs/regex).\nThe module documentation also includes a comprehensive description of the\nsyntax supported.\n\nDocumentation with examples for the various matching functions and iterators\ncan be found on the\n[`Regex` type](https://docs.rs/regex/*/regex/struct.Regex.html).\n\n### Usage\n\nTo bring this crate into your repository, either add `regex` to your\n`Cargo.toml`, or run `cargo add regex`.\n\nHere's a simple example that matches a date in YYYY-MM-DD format and prints the\nyear, month and day:\n\n```rust\nuse regex::Regex;\n\nfn main() {\n let re = Regex::new(r\"(?x)\n(?P\\d{4}) # the year\n-\n(?P\\d{2}) # the month\n-\n(?P\\d{2}) # the day\n\").unwrap();\n\n let caps = re.captures(\"2010-03-14\").unwrap();\n assert_eq!(\"2010\", &caps[\"year\"]);\n assert_eq!(\"03\", &caps[\"month\"]);\n assert_eq!(\"14\", &caps[\"day\"]);\n}\n```\n\nIf you have lots of dates in text that you'd like to iterate over, then it's\neasy to adapt the above example with an iterator:\n\n```rust\nuse regex::Regex;\n\nfn main() {\n let re = Regex::new(r\"(\\d{4})-(\\d{2})-(\\d{2})\").unwrap();\n let hay = \"On 2010-03-14, foo happened. On 2014-10-14, bar happened.\";\n\n let mut dates = vec![];\n for (_, [year, month, day]) in re.captures_iter(hay).map(|c| c.extract()) {\n dates.push((year, month, day));\n }\n assert_eq!(dates, vec![\n (\"2010\", \"03\", \"14\"),\n (\"2014\", \"10\", \"14\"),\n ]);\n}\n```\n\n### Usage: Avoid compiling the same regex in a loop\n\nIt is an anti-pattern to compile the same regular expression in a loop since\ncompilation is typically expensive. (It takes anywhere from a few microseconds\nto a few **milliseconds** depending on the size of the regex.) Not only is\ncompilation itself expensive, but this also prevents optimizations that reuse\nallocations internally to the matching engines.\n\nIn Rust, it can sometimes be a pain to pass regular expressions around if\nthey're used from inside a helper function. Instead, we recommend using\n[`std::sync::LazyLock`], or the [`once_cell`] crate,\nif you can't use the standard library.\n\nThis example shows how to use `std::sync::LazyLock`:\n\n```rust\nuse std::sync::LazyLock;\n\nuse regex::Regex;\n\nfn some_helper_function(haystack: &str) -> bool {\n static RE: LazyLock = LazyLock::new(|| Regex::new(r\"...\").unwrap());\n RE.is_match(haystack)\n}\n\nfn main() {\n assert!(some_helper_function(\"abc\"));\n assert!(!some_helper_function(\"ac\"));\n}\n```\n\nSpecifically, in this example, the regex will be compiled when it is used for\nthe first time. On subsequent uses, it will reuse the previous compilation.\n\nThe [`regex!`] macro can also be used, which handles lazy compilation.\n\n[`std::sync::LazyLock`]: https://doc.rust-lang.org/std/sync/struct.LazyLock.html\n[`once_cell`]: https://crates.io/crates/once_cell\n[`regex!`]: https://docs.rs/regex/*/regex/macro.regex.html\n\n### Usage: match regular expressions on `&[u8]`\n\nThe main API of this crate (`regex::Regex`) requires the caller to pass a\n`&str` for searching. In Rust, an `&str` is required to be valid UTF-8, which\nmeans the main API can't be used for searching arbitrary bytes.\n\nTo match on arbitrary bytes, use the `regex::bytes::Regex` API. The API is\nidentical to the main API, except that it takes an `&[u8]` to search on instead\nof an `&str`. The `&[u8]` APIs also permit disabling Unicode mode in the regex\neven when the pattern would match invalid UTF-8. For example, `(?-u:.)` is\nnot allowed in `regex::Regex` but is allowed in `regex::bytes::Regex` since\n`(?-u:.)` matches any byte except for `\\n`. Conversely, `.` will match the\nUTF-8 encoding of any Unicode scalar value except for `\\n`.\n\nThis example shows how to find all null-terminated strings in a slice of bytes:\n\n```rust\nuse regex::bytes::Regex;\n\nlet re = Regex::new(r\"(?-u)(?[^\\x00]+)\\x00\").unwrap();\nlet text = b\"foo\\xFFbar\\x00baz\\x00\";\n\n// Extract all of the strings without the null terminator from each match.\n// The unwrap is OK here since a match requires the `cstr` capture to match.\nlet cstrs: Vec<&[u8]> =\n re.captures_iter(text)\n .map(|c| c.name(\"cstr\").unwrap().as_bytes())\n .collect();\nassert_eq!(vec![&b\"foo\\xFFbar\"[..], &b\"baz\"[..]], cstrs);\n```\n\nNotice here that the `[^\\x00]+` will match any *byte* except for `NUL`,\nincluding bytes like `\\xFF` which are not valid UTF-8. When using the main API,\n`[^\\x00]+` would instead match any valid UTF-8 sequence except for `NUL`.\n\n### Usage: match multiple regular expressions simultaneously\n\nThis demonstrates how to use a `RegexSet` to match multiple (possibly\noverlapping) regular expressions in a single scan of the search text:\n\n```rust\nuse regex::RegexSet;\n\nlet set = RegexSet::new(&[\n r\"\\w+\",\n r\"\\d+\",\n r\"\\pL+\",\n r\"foo\",\n r\"bar\",\n r\"barfoo\",\n r\"foobar\",\n]).unwrap();\n\n// Iterate over and collect all of the matches.\nlet matches: Vec<_> = set.matches(\"foobar\").into_iter().collect();\nassert_eq!(matches, vec![0, 2, 3, 4, 6]);\n\n// You can also test whether a particular regex matched:\nlet matches = set.matches(\"foobar\");\nassert!(!matches.matched(5));\nassert!(matches.matched(6));\n```\n\n\n### Usage: regex internals as a library\n\nThe [`regex-automata` directory](./regex-automata/) contains a crate that\nexposes all the internal matching engines used by the `regex` crate. The\nidea is that the `regex` crate exposes a simple API for 99% of use cases, but\n`regex-automata` exposes oodles of customizable behaviors.\n\n[Documentation for `regex-automata`.](https://docs.rs/regex-automata)\n\n\n### Usage: a regular expression parser\n\nThis repository contains a crate that provides a well tested regular expression\nparser, abstract syntax and a high-level intermediate representation for\nconvenient analysis. It provides no facilities for compilation or execution.\nThis may be useful if you're implementing your own regex engine or otherwise\nneed to do analysis on the syntax of a regular expression. It is otherwise not\nrecommended for general use.\n\n[Documentation for `regex-syntax`.](https://docs.rs/regex-syntax)\n\n\n### Crate features\n\nThis crate comes with several features that permit tweaking the trade-off\nbetween binary size, compilation time and runtime performance. Users of this\ncrate can selectively disable Unicode tables, or choose from a variety of\noptimizations performed by this crate to disable.\n\nWhen all of these features are disabled, runtime match performance may be much\nworse, but if you're matching on short strings, or if high performance isn't\nnecessary, then such a configuration is perfectly serviceable. To disable\nall such features, use the following `Cargo.toml` dependency configuration:\n\n```toml\n[dependencies.regex]\nversion = \"1.3\"\ndefault-features = false\n# Unless you have a specific reason not to, it's good sense to enable standard\n# library support. It enables several optimizations and avoids spin locks. It\n# also shouldn't meaningfully impact compile times or binary size.\nfeatures = [\"std\"]\n```\n\nThis will reduce the dependency tree of `regex` down to two crates:\n`regex-syntax` and `regex-automata`.\n\nThe full set of features one can disable are\n[in the \"Crate features\" section of the documentation](https://docs.rs/regex/1.*/#crate-features).\n\n\n### Performance\n\nOne of the goals of this crate is for the regex engine to be \"fast.\" What that\nis a somewhat nebulous goal, it is usually interpreted in one of two ways.\nFirst, it means that all searches take worst case `O(m * n)` time, where\n`m` is proportional to `len(regex)` and `n` is proportional to `len(haystack)`.\nSecond, it means that even aside from the time complexity constraint, regex\nsearches are \"fast\" in practice.\n\nWhile the first interpretation is pretty unambiguous, the second one remains\nnebulous. While nebulous, it guides this crate's architecture and the sorts of\nthe trade-offs it makes. For example, here are some general architectural\nstatements that follow as a result of the goal to be \"fast\":\n\n* When given the choice between faster regex searches and faster _Rust compile\ntimes_, this crate will generally choose faster regex searches.\n* When given the choice between faster regex searches and faster _regex compile\ntimes_, this crate will generally choose faster regex searches. That is, it is\ngenerally acceptable for `Regex::new` to get a little slower if it means that\nsearches get faster. (This is a somewhat delicate balance to strike, because\nthe speed of `Regex::new` needs to remain somewhat reasonable. But this is why\none should avoid re-compiling the same regex over and over again.)\n* When given the choice between faster regex searches and simpler API\ndesign, this crate will generally choose faster regex searches. For example,\nif one didn't care about performance, we could like get rid of both of\nthe `Regex::is_match` and `Regex::find` APIs and instead just rely on\n`Regex::captures`.\n\nThere are perhaps more ways that being \"fast\" influences things.\n\nWhile this repository used to provide its own benchmark suite, it has since\nbeen moved to [rebar](https://github.com/BurntSushi/rebar). The benchmarks are\nquite extensive, and there are many more than what is shown in rebar's README\n(which is just limited to a \"curated\" set meant to compare performance between\nregex engines). To run all of this crate's benchmarks, first start by cloning\nand installing `rebar`:\n\n```text\n$ git clone https://github.com/BurntSushi/rebar\n$ cd rebar\n$ cargo install --path ./\n```\n\nThen build the benchmark harness for just this crate:\n\n```text\n$ rebar build -e '^rust/regex$'\n```\n\nRun all benchmarks for this crate as tests (each benchmark is executed once to\nensure it works):\n\n```text\n$ rebar measure -e '^rust/regex$' -t\n```\n\nRecord measurements for all benchmarks and save them to a CSV file:\n\n```text\n$ rebar measure -e '^rust/regex$' | tee results.csv\n```\n\nExplore benchmark timings:\n\n```text\n$ rebar cmp results.csv\n```\n\nSee the `rebar` documentation for more details on how it works and how to\ncompare results with other regex engines.\n\n\n### Hacking\n\nThe `regex` crate is, for the most part, a pretty thin wrapper around the\n[`meta::Regex`](https://docs.rs/regex-automata/latest/regex_automata/meta/struct.Regex.html)\nfrom the\n[`regex-automata` crate](https://docs.rs/regex-automata/latest/regex_automata/).\nTherefore, if you're looking to work on the internals of this crate, you'll\nlikely either want to look in `regex-syntax` (for parsing) or `regex-automata`\n(for construction of finite automata and the search routines).\n\nMy [blog on regex internals](https://burntsushi.net/regex-internals/)\ngoes into more depth.\n\n\n### Minimum Rust version policy\n\nThis crate's minimum supported `rustc` version is `1.65.0`.\n\nThe policy is that the minimum Rust version required to use this crate can be\nincreased in minor version updates. For example, if regex 1.0 requires Rust\n1.20.0, then regex 1.0.z for all values of `z` will also require Rust 1.20.0 or\nnewer. However, regex 1.y for `y > 0` may require a newer minimum version of\nRust.\n\n\n### License\n\nThis project is licensed under either of\n\n * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or\n https://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or\n https://opensource.org/licenses/MIT)\n\nat your option.\n\nThe data in `regex-syntax/src/unicode_tables/` is licensed under the Unicode\nLicense Agreement\n([LICENSE-UNICODE](https://www.unicode.org/copyright.html#License)).\n" }, { "package": "resvg@0.34.1", "sourcePath": "LICENSE.txt", "sha256": "fab3dd6bdab226f1c08630b1dd917e11fcb4ec5e1e020e2c16f83a0a13863e85", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Mozilla Public License Version 2.0\n==================================\n\n1. Definitions\n--------------\n\n1.1. \"Contributor\"\n means each individual or legal entity that creates, contributes to\n the creation of, or owns Covered Software.\n\n1.2. \"Contributor Version\"\n means the combination of the Contributions of others (if any) used\n by a Contributor and that particular Contributor's Contribution.\n\n1.3. \"Contribution\"\n means Covered Software of a particular Contributor.\n\n1.4. \"Covered Software\"\n means Source Code Form to which the initial Contributor has attached\n the notice in Exhibit A, the Executable Form of such Source Code\n Form, and Modifications of such Source Code Form, in each case\n including portions thereof.\n\n1.5. \"Incompatible With Secondary Licenses\"\n means\n\n (a) that the initial Contributor has attached the notice described\n in Exhibit B to the Covered Software; or\n\n (b) that the Covered Software was made available under the terms of\n version 1.1 or earlier of the License, but not also under the\n terms of a Secondary License.\n\n1.6. \"Executable Form\"\n means any form of the work other than Source Code Form.\n\n1.7. \"Larger Work\"\n means a work that combines Covered Software with other material, in \n a separate file or files, that is not Covered Software.\n\n1.8. \"License\"\n means this document.\n\n1.9. \"Licensable\"\n means having the right to grant, to the maximum extent possible,\n whether at the time of the initial grant or subsequently, any and\n all of the rights conveyed by this License.\n\n1.10. \"Modifications\"\n means any of the following:\n\n (a) any file in Source Code Form that results from an addition to,\n deletion from, or modification of the contents of Covered\n Software; or\n\n (b) any new file in Source Code Form that contains any Covered\n Software.\n\n1.11. \"Patent Claims\" of a Contributor\n means any patent claim(s), including without limitation, method,\n process, and apparatus claims, in any patent Licensable by such\n Contributor that would be infringed, but for the grant of the\n License, by the making, using, selling, offering for sale, having\n made, import, or transfer of either its Contributions or its\n Contributor Version.\n\n1.12. \"Secondary License\"\n means either the GNU General Public License, Version 2.0, the GNU\n Lesser General Public License, Version 2.1, the GNU Affero General\n Public License, Version 3.0, or any later versions of those\n licenses.\n\n1.13. \"Source Code Form\"\n means the form of the work preferred for making modifications.\n\n1.14. \"You\" (or \"Your\")\n means an individual or a legal entity exercising rights under this\n License. For legal entities, \"You\" includes any entity that\n controls, is controlled by, or is under common control with You. For\n purposes of this definition, \"control\" means (a) the power, direct\n or indirect, to cause the direction or management of such entity,\n whether by contract or otherwise, or (b) ownership of more than\n fifty percent (50%) of the outstanding shares or beneficial\n ownership of such entity.\n\n2. License Grants and Conditions\n--------------------------------\n\n2.1. Grants\n\nEach Contributor hereby grants You a world-wide, royalty-free,\nnon-exclusive license:\n\n(a) under intellectual property rights (other than patent or trademark)\n Licensable by such Contributor to use, reproduce, make available,\n modify, display, perform, distribute, and otherwise exploit its\n Contributions, either on an unmodified basis, with Modifications, or\n as part of a Larger Work; and\n\n(b) under Patent Claims of such Contributor to make, use, sell, offer\n for sale, have made, import, and otherwise transfer either its\n Contributions or its Contributor Version.\n\n2.2. Effective Date\n\nThe licenses granted in Section 2.1 with respect to any Contribution\nbecome effective for each Contribution on the date the Contributor first\ndistributes such Contribution.\n\n2.3. Limitations on Grant Scope\n\nThe licenses granted in this Section 2 are the only rights granted under\nthis License. No additional rights or licenses will be implied from the\ndistribution or licensing of Covered Software under this License.\nNotwithstanding Section 2.1(b) above, no patent license is granted by a\nContributor:\n\n(a) for any code that a Contributor has removed from Covered Software;\n or\n\n(b) for infringements caused by: (i) Your and any other third party's\n modifications of Covered Software, or (ii) the combination of its\n Contributions with other software (except as part of its Contributor\n Version); or\n\n(c) under Patent Claims infringed by Covered Software in the absence of\n its Contributions.\n\nThis License does not grant any rights in the trademarks, service marks,\nor logos of any Contributor (except as may be necessary to comply with\nthe notice requirements in Section 3.4).\n\n2.4. Subsequent Licenses\n\nNo Contributor makes additional grants as a result of Your choice to\ndistribute the Covered Software under a subsequent version of this\nLicense (see Section 10.2) or under the terms of a Secondary License (if\npermitted under the terms of Section 3.3).\n\n2.5. Representation\n\nEach Contributor represents that the Contributor believes its\nContributions are its original creation(s) or it has sufficient rights\nto grant the rights to its Contributions conveyed by this License.\n\n2.6. Fair Use\n\nThis License is not intended to limit any rights You have under\napplicable copyright doctrines of fair use, fair dealing, or other\nequivalents.\n\n2.7. Conditions\n\nSections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted\nin Section 2.1.\n\n3. Responsibilities\n-------------------\n\n3.1. Distribution of Source Form\n\nAll distribution of Covered Software in Source Code Form, including any\nModifications that You create or to which You contribute, must be under\nthe terms of this License. You must inform recipients that the Source\nCode Form of the Covered Software is governed by the terms of this\nLicense, and how they can obtain a copy of this License. You may not\nattempt to alter or restrict the recipients' rights in the Source Code\nForm.\n\n3.2. Distribution of Executable Form\n\nIf You distribute Covered Software in Executable Form then:\n\n(a) such Covered Software must also be made available in Source Code\n Form, as described in Section 3.1, and You must inform recipients of\n the Executable Form how they can obtain a copy of such Source Code\n Form by reasonable means in a timely manner, at a charge no more\n than the cost of distribution to the recipient; and\n\n(b) You may distribute such Executable Form under the terms of this\n License, or sublicense it under different terms, provided that the\n license for the Executable Form does not attempt to limit or alter\n the recipients' rights in the Source Code Form under this License.\n\n3.3. Distribution of a Larger Work\n\nYou may create and distribute a Larger Work under terms of Your choice,\nprovided that You also comply with the requirements of this License for\nthe Covered Software. If the Larger Work is a combination of Covered\nSoftware with a work governed by one or more Secondary Licenses, and the\nCovered Software is not Incompatible With Secondary Licenses, this\nLicense permits You to additionally distribute such Covered Software\nunder the terms of such Secondary License(s), so that the recipient of\nthe Larger Work may, at their option, further distribute the Covered\nSoftware under the terms of either this License or such Secondary\nLicense(s).\n\n3.4. Notices\n\nYou may not remove or alter the substance of any license notices\n(including copyright notices, patent notices, disclaimers of warranty,\nor limitations of liability) contained within the Source Code Form of\nthe Covered Software, except that You may alter any license notices to\nthe extent required to remedy known factual inaccuracies.\n\n3.5. Application of Additional Terms\n\nYou may choose to offer, and to charge a fee for, warranty, support,\nindemnity or liability obligations to one or more recipients of Covered\nSoftware. However, You may do so only on Your own behalf, and not on\nbehalf of any Contributor. You must make it absolutely clear that any\nsuch warranty, support, indemnity, or liability obligation is offered by\nYou alone, and You hereby agree to indemnify every Contributor for any\nliability incurred by such Contributor as a result of warranty, support,\nindemnity or liability terms You offer. You may include additional\ndisclaimers of warranty and limitations of liability specific to any\njurisdiction.\n\n4. Inability to Comply Due to Statute or Regulation\n---------------------------------------------------\n\nIf it is impossible for You to comply with any of the terms of this\nLicense with respect to some or all of the Covered Software due to\nstatute, judicial order, or regulation then You must: (a) comply with\nthe terms of this License to the maximum extent possible; and (b)\ndescribe the limitations and the code they affect. Such description must\nbe placed in a text file included with all distributions of the Covered\nSoftware under this License. Except to the extent prohibited by statute\nor regulation, such description must be sufficiently detailed for a\nrecipient of ordinary skill to be able to understand it.\n\n5. Termination\n--------------\n\n5.1. The rights granted under this License will terminate automatically\nif You fail to comply with any of its terms. However, if You become\ncompliant, then the rights granted under this License from a particular\nContributor are reinstated (a) provisionally, unless and until such\nContributor explicitly and finally terminates Your grants, and (b) on an\nongoing basis, if such Contributor fails to notify You of the\nnon-compliance by some reasonable means prior to 60 days after You have\ncome back into compliance. Moreover, Your grants from a particular\nContributor are reinstated on an ongoing basis if such Contributor\nnotifies You of the non-compliance by some reasonable means, this is the\nfirst time You have received notice of non-compliance with this License\nfrom such Contributor, and You become compliant prior to 30 days after\nYour receipt of the notice.\n\n5.2. If You initiate litigation against any entity by asserting a patent\ninfringement claim (excluding declaratory judgment actions,\ncounter-claims, and cross-claims) alleging that a Contributor Version\ndirectly or indirectly infringes any patent, then the rights granted to\nYou by any and all Contributors for the Covered Software under Section\n2.1 of this License shall terminate.\n\n5.3. In the event of termination under Sections 5.1 or 5.2 above, all\nend user license agreements (excluding distributors and resellers) which\nhave been validly granted by You or Your distributors under this License\nprior to termination shall survive termination.\n\n************************************************************************\n* *\n* 6. Disclaimer of Warranty *\n* ------------------------- *\n* *\n* Covered Software is provided under this License on an \"as is\" *\n* basis, without warranty of any kind, either expressed, implied, or *\n* statutory, including, without limitation, warranties that the *\n* Covered Software is free of defects, merchantable, fit for a *\n* particular purpose or non-infringing. The entire risk as to the *\n* quality and performance of the Covered Software is with You. *\n* Should any Covered Software prove defective in any respect, You *\n* (not any Contributor) assume the cost of any necessary servicing, *\n* repair, or correction. This disclaimer of warranty constitutes an *\n* essential part of this License. No use of any Covered Software is *\n* authorized under this License except under this disclaimer. *\n* *\n************************************************************************\n\n************************************************************************\n* *\n* 7. Limitation of Liability *\n* -------------------------- *\n* *\n* Under no circumstances and under no legal theory, whether tort *\n* (including negligence), contract, or otherwise, shall any *\n* Contributor, or anyone who distributes Covered Software as *\n* permitted above, be liable to You for any direct, indirect, *\n* special, incidental, or consequential damages of any character *\n* including, without limitation, damages for lost profits, loss of *\n* goodwill, work stoppage, computer failure or malfunction, or any *\n* and all other commercial damages or losses, even if such party *\n* shall have been informed of the possibility of such damages. This *\n* limitation of liability shall not apply to liability for death or *\n* personal injury resulting from such party's negligence to the *\n* extent applicable law prohibits such limitation. Some *\n* jurisdictions do not allow the exclusion or limitation of *\n* incidental or consequential damages, so this exclusion and *\n* limitation may not apply to You. *\n* *\n************************************************************************\n\n8. Litigation\n-------------\n\nAny litigation relating to this License may be brought only in the\ncourts of a jurisdiction where the defendant maintains its principal\nplace of business and such litigation shall be governed by laws of that\njurisdiction, without reference to its conflict-of-law provisions.\nNothing in this Section shall prevent a party's ability to bring\ncross-claims or counter-claims.\n\n9. Miscellaneous\n----------------\n\nThis License represents the complete agreement concerning the subject\nmatter hereof. If any provision of this License is held to be\nunenforceable, such provision shall be reformed only to the extent\nnecessary to make it enforceable. Any law or regulation which provides\nthat the language of a contract shall be construed against the drafter\nshall not be used to construe this License against a Contributor.\n\n10. Versions of the License\n---------------------------\n\n10.1. New Versions\n\nMozilla Foundation is the license steward. Except as provided in Section\n10.3, no one other than the license steward has the right to modify or\npublish new versions of this License. Each version will be given a\ndistinguishing version number.\n\n10.2. Effect of New Versions\n\nYou may distribute the Covered Software under the terms of the version\nof the License under which You originally received the Covered Software,\nor under the terms of any subsequent version published by the license\nsteward.\n\n10.3. Modified Versions\n\nIf you create software not governed by this License, and you want to\ncreate a new license for such software, you may create and use a\nmodified version of this License if you rename the license and remove\nany references to the name of the license steward (except to note that\nsuch modified license differs from this License).\n\n10.4. Distributing Source Code Form that is Incompatible With Secondary\nLicenses\n\nIf You choose to distribute Source Code Form that is Incompatible With\nSecondary Licenses under the terms of this version of the License, the\nnotice described in Exhibit B of this License must be attached.\n\nExhibit A - Source Code Form License Notice\n-------------------------------------------\n\n This Source Code Form is subject to the terms of the Mozilla Public\n License, v. 2.0. If a copy of the MPL was not distributed with this\n file, You can obtain one at http://mozilla.org/MPL/2.0/.\n\nIf it is not possible or desirable to put the notice in a particular\nfile, then You may include the notice in a location (such as a LICENSE\nfile in a relevant directory) where a recipient would be likely to look\nfor such a notice.\n\nYou may add additional accurate notices of copyright ownership.\n\nExhibit B - \"Incompatible With Secondary Licenses\" Notice\n---------------------------------------------------------\n\n This Source Code Form is \"Incompatible With Secondary Licenses\", as\n defined by the Mozilla Public License, v. 2.0.\n" }, { "package": "resvg@0.34.1", "sourcePath": "src/filter/iir_blur.rs", "sha256": "b4f295e31b547ec9b3f83a6b875d0440ba7311f94e8e9f0ac6ad1eeb9ee804ac", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// This Source Code Form is subject to the terms of the Mozilla Public\n// License, v. 2.0. If a copy of the MPL was not distributed with this\n// file, You can obtain one at http://mozilla.org/MPL/2.0/.\n\n// An IIR blur.\n//\n// Based on http://www.getreuer.info/home/gaussianiir\n//\n// Licensed under 'Simplified BSD License'.\n//\n//\n// Implements the fast Gaussian convolution algorithm of Alvarez and Mazorra,\n// where the Gaussian is approximated by a cascade of first-order infinite\n// impulsive response (IIR) filters. Boundaries are handled with half-sample\n// symmetric extension.\n//\n// Gaussian convolution is approached as approximating the heat equation and\n// each timestep is performed with an efficient recursive computation. Using\n// more steps yields a more accurate approximation of the Gaussian. A\n// reasonable default value for `numsteps` is 4.\n//\n// Reference:\n// Alvarez, Mazorra, \"Signal and Image Restoration using Shock Filters and\n// Anisotropic Diffusion,\" SIAM J. on Numerical Analysis, vol. 31, no. 2,\n// pp. 590-605, 1994.\n\n// TODO: Blurs right and bottom sides twice for some reason.\n\nuse super::ImageRefMut;\nuse rgb::ComponentSlice;\n\nstruct BlurData {\n width: usize,\n height: usize,\n sigma_x: f64,\n sigma_y: f64,\n steps: usize,\n}\n\n/// Applies an IIR blur.\n///\n/// Input image pixels should have a **premultiplied alpha**.\n///\n/// A negative or zero `sigma_x`/`sigma_y` will disable the blur along that axis.\n///\n/// # Allocations\n///\n/// This method will allocate a 2x `src` buffer.\npub fn apply(sigma_x: f64, sigma_y: f64, src: ImageRefMut) {\n let buf_size = (src.width * src.height) as usize;\n let mut buf = vec![0.0; buf_size];\n let buf = &mut buf;\n\n let d = BlurData {\n width: src.width as usize,\n height: src.height as usize,\n sigma_x,\n sigma_y,\n steps: 4,\n };\n\n let data = src.data.as_mut_slice();\n gaussian_channel(data, &d, 0, buf);\n gaussian_channel(data, &d, 1, buf);\n gaussian_channel(data, &d, 2, buf);\n gaussian_channel(data, &d, 3, buf);\n}\n\nfn gaussian_channel(data: &mut [u8], d: &BlurData, channel: usize, buf: &mut Vec) {\n for i in 0..data.len() / 4 {\n buf[i] = data[i * 4 + channel] as f64 / 255.0;\n }\n\n gaussianiir2d(d, buf);\n\n for i in 0..data.len() / 4 {\n data[i * 4 + channel] = (buf[i] * 255.0) as u8;\n }\n}\n\nfn gaussianiir2d(d: &BlurData, buf: &mut Vec) {\n // Filter horizontally along each row.\n let (lambda_x, dnu_x) = if d.sigma_x > 0.0 {\n let (lambda, dnu) = gen_coefficients(d.sigma_x, d.steps);\n\n for y in 0..d.height {\n for _ in 0..d.steps {\n let idx = d.width * y;\n\n // Filter rightwards.\n for x in 1..d.width {\n buf[idx + x] += dnu * buf[idx + x - 1];\n }\n\n let mut x = d.width - 1;\n\n // Filter leftwards.\n while x > 0 {\n buf[idx + x - 1] += dnu * buf[idx + x];\n x -= 1;\n }\n }\n }\n\n (lambda, dnu)\n } else {\n (1.0, 1.0)\n };\n\n // Filter vertically along each column.\n let (lambda_y, dnu_y) = if d.sigma_y > 0.0 {\n let (lambda, dnu) = gen_coefficients(d.sigma_y, d.steps);\n for x in 0..d.width {\n for _ in 0..d.steps {\n let idx = x;\n\n // Filter downwards.\n let mut y = d.width;\n while y < buf.len() {\n buf[idx + y] += dnu * buf[idx + y - d.width];\n y += d.width;\n }\n\n y = buf.len() - d.width;\n\n // Filter upwards.\n while y > 0 {\n buf[idx + y - d.width] += dnu * buf[idx + y];\n y -= d.width;\n }\n }\n }\n\n (lambda, dnu)\n } else {\n (1.0, 1.0)\n };\n\n let post_scale =\n ((dnu_x * dnu_y).sqrt() / (lambda_x * lambda_y).sqrt()).powi(2 * d.steps as i32);\n buf.iter_mut().for_each(|v| *v *= post_scale);\n}\n\nfn gen_coefficients(sigma: f64, steps: usize) -> (f64, f64) {\n let lambda = (sigma * sigma) / (2.0 * steps as f64);\n let dnu = (1.0 + 2.0 * lambda - (1.0 + 4.0 * lambda).sqrt()) / (2.0 * lambda);\n (lambda, dnu)\n}\n" }, { "package": "resvg@0.34.1", "sourcePath": "tests/fonts/Amiri-LICENSE-OFL.txt", "sha256": "1f91c5ef4a18a747526f9c527d8e39337d31d3f37a8472c89fbdfe874c2d0d69", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Copyright (c) 2010-2016, Khaled Hosny ()\r\n\r\nThis Font Software is licensed under the SIL Open Font License, Version 1.1.\r\nThis license is copied below, and is also available with a FAQ at:\r\nhttp://scripts.sil.org/OFL\r\n\r\n\r\n-----------------------------------------------------------\r\nSIL OPEN FONT LICENSE Version 1.1 - 26 February 2007\r\n-----------------------------------------------------------\r\n\r\nPREAMBLE\r\nThe goals of the Open Font License (OFL) are to stimulate worldwide\r\ndevelopment of collaborative font projects, to support the font creation\r\nefforts of academic and linguistic communities, and to provide a free and\r\nopen framework in which fonts may be shared and improved in partnership\r\nwith others.\r\n\r\nThe OFL allows the licensed fonts to be used, studied, modified and\r\nredistributed freely as long as they are not sold by themselves. The\r\nfonts, including any derivative works, can be bundled, embedded, \r\nredistributed and/or sold with any software provided that any reserved\r\nnames are not used by derivative works. The fonts and derivatives,\r\nhowever, cannot be released under any other type of license. The\r\nrequirement for fonts to remain under this license does not apply\r\nto any document created using the fonts or their derivatives.\r\n\r\nDEFINITIONS\r\n\"Font Software\" refers to the set of files released by the Copyright\r\nHolder(s) under this license and clearly marked as such. This may\r\ninclude source files, build scripts and documentation.\r\n\r\n\"Reserved Font Name\" refers to any names specified as such after the\r\ncopyright statement(s).\r\n\r\n\"Original Version\" refers to the collection of Font Software components as\r\ndistributed by the Copyright Holder(s).\r\n\r\n\"Modified Version\" refers to any derivative made by adding to, deleting,\r\nor substituting -- in part or in whole -- any of the components of the\r\nOriginal Version, by changing formats or by porting the Font Software to a\r\nnew environment.\r\n\r\n\"Author\" refers to any designer, engineer, programmer, technical\r\nwriter or other person who contributed to the Font Software.\r\n\r\nPERMISSION & CONDITIONS\r\nPermission is hereby granted, free of charge, to any person obtaining\r\na copy of the Font Software, to use, study, copy, merge, embed, modify,\r\nredistribute, and sell modified and unmodified copies of the Font\r\nSoftware, subject to the following conditions:\r\n\r\n1) Neither the Font Software nor any of its individual components,\r\nin Original or Modified Versions, may be sold by itself.\r\n\r\n2) Original or Modified Versions of the Font Software may be bundled,\r\nredistributed and/or sold with any software, provided that each copy\r\ncontains the above copyright notice and this license. These can be\r\nincluded either as stand-alone text files, human-readable headers or\r\nin the appropriate machine-readable metadata fields within text or\r\nbinary files as long as those fields can be easily viewed by the user.\r\n\r\n3) No Modified Version of the Font Software may use the Reserved Font\r\nName(s) unless explicit written permission is granted by the corresponding\r\nCopyright Holder. This restriction only applies to the primary font name as\r\npresented to the users.\r\n\r\n4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font\r\nSoftware shall not be used to promote, endorse or advertise any\r\nModified Version, except to acknowledge the contribution(s) of the\r\nCopyright Holder(s) and the Author(s) or with their explicit written\r\npermission.\r\n\r\n5) The Font Software, modified or unmodified, in part or in whole,\r\nmust be distributed entirely under this license, and must not be\r\ndistributed under any other license. The requirement for fonts to\r\nremain under this license does not apply to any document created\r\nusing the Font Software.\r\n\r\nTERMINATION\r\nThis license becomes null and void if any of the above conditions are\r\nnot met.\r\n\r\nDISCLAIMER\r\nTHE FONT SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\r\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF\r\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT\r\nOF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE\r\nCOPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\r\nINCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL\r\nDAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\r\nFROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM\r\nOTHER DEALINGS IN THE FONT SOFTWARE.\r\n" }, { "package": "resvg@0.34.1", "sourcePath": "tests/fonts/MPLUS1p-LICENSE-OFL.txt", "sha256": "7f18ec1ebb6b50e3ed0f74b2c61f25b8d7cd69e43f4de66e991bcfd3c419a8bb", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "This Font Software is licensed under the SIL Open Font License, Version 1.1.\r\nThis license is copied below, and is also available with a FAQ at:\r\nhttp://scripts.sil.org/OFL\r\n\r\n\r\n-----------------------------------------------------------\r\nSIL OPEN FONT LICENSE Version 1.1 - 26 February 2007\r\n-----------------------------------------------------------\r\n\r\nPREAMBLE\r\nThe goals of the Open Font License (OFL) are to stimulate worldwide\r\ndevelopment of collaborative font projects, to support the font creation\r\nefforts of academic and linguistic communities, and to provide a free and\r\nopen framework in which fonts may be shared and improved in partnership\r\nwith others.\r\n\r\nThe OFL allows the licensed fonts to be used, studied, modified and\r\nredistributed freely as long as they are not sold by themselves. The\r\nfonts, including any derivative works, can be bundled, embedded, \r\nredistributed and/or sold with any software provided that any reserved\r\nnames are not used by derivative works. The fonts and derivatives,\r\nhowever, cannot be released under any other type of license. The\r\nrequirement for fonts to remain under this license does not apply\r\nto any document created using the fonts or their derivatives.\r\n\r\nDEFINITIONS\r\n\"Font Software\" refers to the set of files released by the Copyright\r\nHolder(s) under this license and clearly marked as such. This may\r\ninclude source files, build scripts and documentation.\r\n\r\n\"Reserved Font Name\" refers to any names specified as such after the\r\ncopyright statement(s).\r\n\r\n\"Original Version\" refers to the collection of Font Software components as\r\ndistributed by the Copyright Holder(s).\r\n\r\n\"Modified Version\" refers to any derivative made by adding to, deleting,\r\nor substituting -- in part or in whole -- any of the components of the\r\nOriginal Version, by changing formats or by porting the Font Software to a\r\nnew environment.\r\n\r\n\"Author\" refers to any designer, engineer, programmer, technical\r\nwriter or other person who contributed to the Font Software.\r\n\r\nPERMISSION & CONDITIONS\r\nPermission is hereby granted, free of charge, to any person obtaining\r\na copy of the Font Software, to use, study, copy, merge, embed, modify,\r\nredistribute, and sell modified and unmodified copies of the Font\r\nSoftware, subject to the following conditions:\r\n\r\n1) Neither the Font Software nor any of its individual components,\r\nin Original or Modified Versions, may be sold by itself.\r\n\r\n2) Original or Modified Versions of the Font Software may be bundled,\r\nredistributed and/or sold with any software, provided that each copy\r\ncontains the above copyright notice and this license. These can be\r\nincluded either as stand-alone text files, human-readable headers or\r\nin the appropriate machine-readable metadata fields within text or\r\nbinary files as long as those fields can be easily viewed by the user.\r\n\r\n3) No Modified Version of the Font Software may use the Reserved Font\r\nName(s) unless explicit written permission is granted by the corresponding\r\nCopyright Holder. This restriction only applies to the primary font name as\r\npresented to the users.\r\n\r\n4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font\r\nSoftware shall not be used to promote, endorse or advertise any\r\nModified Version, except to acknowledge the contribution(s) of the\r\nCopyright Holder(s) and the Author(s) or with their explicit written\r\npermission.\r\n\r\n5) The Font Software, modified or unmodified, in part or in whole,\r\nmust be distributed entirely under this license, and must not be\r\ndistributed under any other license. The requirement for fonts to\r\nremain under this license does not apply to any document created\r\nusing the Font Software.\r\n\r\nTERMINATION\r\nThis license becomes null and void if any of the above conditions are\r\nnot met.\r\n\r\nDISCLAIMER\r\nTHE FONT SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\r\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF\r\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT\r\nOF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE\r\nCOPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\r\nINCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL\r\nDAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\r\nFROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM\r\nOTHER DEALINGS IN THE FONT SOFTWARE.\r\n" }, { "package": "resvg@0.34.1", "sourcePath": "tests/fonts/Noto-LICENSE-OFL.txt", "sha256": "6a73f9541c2de74158c0e7cf6b0a58ef774f5a780bf191f2d7ec9cc53efe2bf2", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "This Font Software is licensed under the SIL Open Font License,\nVersion 1.1.\n\nThis license is copied below, and is also available with a FAQ at:\nhttp://scripts.sil.org/OFL\n\n-----------------------------------------------------------\nSIL OPEN FONT LICENSE Version 1.1 - 26 February 2007\n-----------------------------------------------------------\n\nPREAMBLE\nThe goals of the Open Font License (OFL) are to stimulate worldwide\ndevelopment of collaborative font projects, to support the font\ncreation efforts of academic and linguistic communities, and to\nprovide a free and open framework in which fonts may be shared and\nimproved in partnership with others.\n\nThe OFL allows the licensed fonts to be used, studied, modified and\nredistributed freely as long as they are not sold by themselves. The\nfonts, including any derivative works, can be bundled, embedded,\nredistributed and/or sold with any software provided that any reserved\nnames are not used by derivative works. The fonts and derivatives,\nhowever, cannot be released under any other type of license. The\nrequirement for fonts to remain under this license does not apply to\nany document created using the fonts or their derivatives.\n\nDEFINITIONS\n\"Font Software\" refers to the set of files released by the Copyright\nHolder(s) under this license and clearly marked as such. This may\ninclude source files, build scripts and documentation.\n\n\"Reserved Font Name\" refers to any names specified as such after the\ncopyright statement(s).\n\n\"Original Version\" refers to the collection of Font Software\ncomponents as distributed by the Copyright Holder(s).\n\n\"Modified Version\" refers to any derivative made by adding to,\ndeleting, or substituting -- in part or in whole -- any of the\ncomponents of the Original Version, by changing formats or by porting\nthe Font Software to a new environment.\n\n\"Author\" refers to any designer, engineer, programmer, technical\nwriter or other person who contributed to the Font Software.\n\nPERMISSION & CONDITIONS\nPermission is hereby granted, free of charge, to any person obtaining\na copy of the Font Software, to use, study, copy, merge, embed,\nmodify, redistribute, and sell modified and unmodified copies of the\nFont Software, subject to the following conditions:\n\n1) Neither the Font Software nor any of its individual components, in\nOriginal or Modified Versions, may be sold by itself.\n\n2) Original or Modified Versions of the Font Software may be bundled,\nredistributed and/or sold with any software, provided that each copy\ncontains the above copyright notice and this license. These can be\nincluded either as stand-alone text files, human-readable headers or\nin the appropriate machine-readable metadata fields within text or\nbinary files as long as those fields can be easily viewed by the user.\n\n3) No Modified Version of the Font Software may use the Reserved Font\nName(s) unless explicit written permission is granted by the\ncorresponding Copyright Holder. This restriction only applies to the\nprimary font name as presented to the users.\n\n4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font\nSoftware shall not be used to promote, endorse or advertise any\nModified Version, except to acknowledge the contribution(s) of the\nCopyright Holder(s) and the Author(s) or with their explicit written\npermission.\n\n5) The Font Software, modified or unmodified, in part or in whole,\nmust be distributed entirely under this license, and must not be\ndistributed under any other license. The requirement for fonts to\nremain under this license does not apply to any document created using\nthe Font Software.\n\nTERMINATION\nThis license becomes null and void if any of the above conditions are\nnot met.\n\nDISCLAIMER\nTHE FONT SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT\nOF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE\nCOPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\nINCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL\nDAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\nFROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM\nOTHER DEALINGS IN THE FONT SOFTWARE.\n" }, { "package": "resvg@0.34.1", "sourcePath": "tests/fonts/SedgwickAveDisplay-LICENSE-OFL.txt", "sha256": "7727a792e40376a1a6ef139e2b94a8199692de5a2ea48a3d72e7161bf672753d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Copyright 2017 The Sedgwick Ave Project Authors (https://github.com/googlefonts/sedgwickave)\r\n\r\nThis Font Software is licensed under the SIL Open Font License, Version 1.1.\r\nThis license is copied below, and is also available with a FAQ at:\r\nhttp://scripts.sil.org/OFL\r\n\r\n\r\n-----------------------------------------------------------\r\nSIL OPEN FONT LICENSE Version 1.1 - 26 February 2007\r\n-----------------------------------------------------------\r\n\r\nPREAMBLE\r\nThe goals of the Open Font License (OFL) are to stimulate worldwide\r\ndevelopment of collaborative font projects, to support the font creation\r\nefforts of academic and linguistic communities, and to provide a free and\r\nopen framework in which fonts may be shared and improved in partnership\r\nwith others.\r\n\r\nThe OFL allows the licensed fonts to be used, studied, modified and\r\nredistributed freely as long as they are not sold by themselves. The\r\nfonts, including any derivative works, can be bundled, embedded, \r\nredistributed and/or sold with any software provided that any reserved\r\nnames are not used by derivative works. The fonts and derivatives,\r\nhowever, cannot be released under any other type of license. The\r\nrequirement for fonts to remain under this license does not apply\r\nto any document created using the fonts or their derivatives.\r\n\r\nDEFINITIONS\r\n\"Font Software\" refers to the set of files released by the Copyright\r\nHolder(s) under this license and clearly marked as such. This may\r\ninclude source files, build scripts and documentation.\r\n\r\n\"Reserved Font Name\" refers to any names specified as such after the\r\ncopyright statement(s).\r\n\r\n\"Original Version\" refers to the collection of Font Software components as\r\ndistributed by the Copyright Holder(s).\r\n\r\n\"Modified Version\" refers to any derivative made by adding to, deleting,\r\nor substituting -- in part or in whole -- any of the components of the\r\nOriginal Version, by changing formats or by porting the Font Software to a\r\nnew environment.\r\n\r\n\"Author\" refers to any designer, engineer, programmer, technical\r\nwriter or other person who contributed to the Font Software.\r\n\r\nPERMISSION & CONDITIONS\r\nPermission is hereby granted, free of charge, to any person obtaining\r\na copy of the Font Software, to use, study, copy, merge, embed, modify,\r\nredistribute, and sell modified and unmodified copies of the Font\r\nSoftware, subject to the following conditions:\r\n\r\n1) Neither the Font Software nor any of its individual components,\r\nin Original or Modified Versions, may be sold by itself.\r\n\r\n2) Original or Modified Versions of the Font Software may be bundled,\r\nredistributed and/or sold with any software, provided that each copy\r\ncontains the above copyright notice and this license. These can be\r\nincluded either as stand-alone text files, human-readable headers or\r\nin the appropriate machine-readable metadata fields within text or\r\nbinary files as long as those fields can be easily viewed by the user.\r\n\r\n3) No Modified Version of the Font Software may use the Reserved Font\r\nName(s) unless explicit written permission is granted by the corresponding\r\nCopyright Holder. This restriction only applies to the primary font name as\r\npresented to the users.\r\n\r\n4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font\r\nSoftware shall not be used to promote, endorse or advertise any\r\nModified Version, except to acknowledge the contribution(s) of the\r\nCopyright Holder(s) and the Author(s) or with their explicit written\r\npermission.\r\n\r\n5) The Font Software, modified or unmodified, in part or in whole,\r\nmust be distributed entirely under this license, and must not be\r\ndistributed under any other license. The requirement for fonts to\r\nremain under this license does not apply to any document created\r\nusing the Font Software.\r\n\r\nTERMINATION\r\nThis license becomes null and void if any of the above conditions are\r\nnot met.\r\n\r\nDISCLAIMER\r\nTHE FONT SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\r\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF\r\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT\r\nOF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE\r\nCOPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\r\nINCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL\r\nDAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\r\nFROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM\r\nOTHER DEALINGS IN THE FONT SOFTWARE.\r\n" }, { "package": "resvg@0.34.1", "sourcePath": "tests/fonts/SourceSansPro-LICENSE-OFL.md", "sha256": "e1d1fdcefb57985226c06fa9500d29caacfa2b1f8b8c74db911f13c6cad186a1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Copyright 2010-2018 Adobe (http://www.adobe.com/), with Reserved Font Name 'Source'. All Rights Reserved. Source is a trademark of Adobe in the United States and/or other countries.\r\n\r\nThis Font Software is licensed under the SIL Open Font License, Version 1.1.\r\n\r\nThis license is copied below, and is also available with a FAQ at: http://scripts.sil.org/OFL\r\n\r\n\r\n-----------------------------------------------------------\r\nSIL OPEN FONT LICENSE Version 1.1 - 26 February 2007\r\n-----------------------------------------------------------\r\n\r\nPREAMBLE\r\nThe goals of the Open Font License (OFL) are to stimulate worldwide\r\ndevelopment of collaborative font projects, to support the font creation\r\nefforts of academic and linguistic communities, and to provide a free and\r\nopen framework in which fonts may be shared and improved in partnership\r\nwith others.\r\n\r\nThe OFL allows the licensed fonts to be used, studied, modified and\r\nredistributed freely as long as they are not sold by themselves. The\r\nfonts, including any derivative works, can be bundled, embedded, \r\nredistributed and/or sold with any software provided that any reserved\r\nnames are not used by derivative works. The fonts and derivatives,\r\nhowever, cannot be released under any other type of license. The\r\nrequirement for fonts to remain under this license does not apply\r\nto any document created using the fonts or their derivatives.\r\n\r\nDEFINITIONS\r\n\"Font Software\" refers to the set of files released by the Copyright\r\nHolder(s) under this license and clearly marked as such. This may\r\ninclude source files, build scripts and documentation.\r\n\r\n\"Reserved Font Name\" refers to any names specified as such after the\r\ncopyright statement(s).\r\n\r\n\"Original Version\" refers to the collection of Font Software components as\r\ndistributed by the Copyright Holder(s).\r\n\r\n\"Modified Version\" refers to any derivative made by adding to, deleting,\r\nor substituting -- in part or in whole -- any of the components of the\r\nOriginal Version, by changing formats or by porting the Font Software to a\r\nnew environment.\r\n\r\n\"Author\" refers to any designer, engineer, programmer, technical\r\nwriter or other person who contributed to the Font Software.\r\n\r\nPERMISSION & CONDITIONS\r\nPermission is hereby granted, free of charge, to any person obtaining\r\na copy of the Font Software, to use, study, copy, merge, embed, modify,\r\nredistribute, and sell modified and unmodified copies of the Font\r\nSoftware, subject to the following conditions:\r\n\r\n1) Neither the Font Software nor any of its individual components,\r\nin Original or Modified Versions, may be sold by itself.\r\n\r\n2) Original or Modified Versions of the Font Software may be bundled,\r\nredistributed and/or sold with any software, provided that each copy\r\ncontains the above copyright notice and this license. These can be\r\nincluded either as stand-alone text files, human-readable headers or\r\nin the appropriate machine-readable metadata fields within text or\r\nbinary files as long as those fields can be easily viewed by the user.\r\n\r\n3) No Modified Version of the Font Software may use the Reserved Font\r\nName(s) unless explicit written permission is granted by the corresponding\r\nCopyright Holder. This restriction only applies to the primary font name as\r\npresented to the users.\r\n\r\n4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font\r\nSoftware shall not be used to promote, endorse or advertise any\r\nModified Version, except to acknowledge the contribution(s) of the\r\nCopyright Holder(s) and the Author(s) or with their explicit written\r\npermission.\r\n\r\n5) The Font Software, modified or unmodified, in part or in whole,\r\nmust be distributed entirely under this license, and must not be\r\ndistributed under any other license. The requirement for fonts to\r\nremain under this license does not apply to any document created\r\nusing the Font Software.\r\n\r\nTERMINATION\r\nThis license becomes null and void if any of the above conditions are\r\nnot met.\r\n\r\nDISCLAIMER\r\nTHE FONT SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\r\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF\r\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT\r\nOF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE\r\nCOPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\r\nINCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL\r\nDAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\r\nFROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM\r\nOTHER DEALINGS IN THE FONT SOFTWARE.\r\n" }, { "package": "resvg@0.34.1", "sourcePath": "tests/fonts/Yellowtail-LICENSE-Apache2.txt", "sha256": "3ddf9be5c28fe27dad143a5dc76eea25222ad1dd68934a047064e56ed2fa40c5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "\r\n Apache License\r\n Version 2.0, January 2004\r\n http://www.apache.org/licenses/\r\n\r\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\r\n\r\n 1. Definitions.\r\n\r\n \"License\" shall mean the terms and conditions for use, reproduction,\r\n and distribution as defined by Sections 1 through 9 of this document.\r\n\r\n \"Licensor\" shall mean the copyright owner or entity authorized by\r\n the copyright owner that is granting the License.\r\n\r\n \"Legal Entity\" shall mean the union of the acting entity and all\r\n other entities that control, are controlled by, or are under common\r\n control with that entity. For the purposes of this definition,\r\n \"control\" means (i) the power, direct or indirect, to cause the\r\n direction or management of such entity, whether by contract or\r\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\r\n outstanding shares, or (iii) beneficial ownership of such entity.\r\n\r\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\r\n exercising permissions granted by this License.\r\n\r\n \"Source\" form shall mean the preferred form for making modifications,\r\n including but not limited to software source code, documentation\r\n source, and configuration files.\r\n\r\n \"Object\" form shall mean any form resulting from mechanical\r\n transformation or translation of a Source form, including but\r\n not limited to compiled object code, generated documentation,\r\n and conversions to other media types.\r\n\r\n \"Work\" shall mean the work of authorship, whether in Source or\r\n Object form, made available under the License, as indicated by a\r\n copyright notice that is included in or attached to the work\r\n (an example is provided in the Appendix below).\r\n\r\n \"Derivative Works\" shall mean any work, whether in Source or Object\r\n form, that is based on (or derived from) the Work and for which the\r\n editorial revisions, annotations, elaborations, or other modifications\r\n represent, as a whole, an original work of authorship. For the purposes\r\n of this License, Derivative Works shall not include works that remain\r\n separable from, or merely link (or bind by name) to the interfaces of,\r\n the Work and Derivative Works thereof.\r\n\r\n \"Contribution\" shall mean any work of authorship, including\r\n the original version of the Work and any modifications or additions\r\n to that Work or Derivative Works thereof, that is intentionally\r\n submitted to Licensor for inclusion in the Work by the copyright owner\r\n or by an individual or Legal Entity authorized to submit on behalf of\r\n the copyright owner. For the purposes of this definition, \"submitted\"\r\n means any form of electronic, verbal, or written communication sent\r\n to the Licensor or its representatives, including but not limited to\r\n communication on electronic mailing lists, source code control systems,\r\n and issue tracking systems that are managed by, or on behalf of, the\r\n Licensor for the purpose of discussing and improving the Work, but\r\n excluding communication that is conspicuously marked or otherwise\r\n designated in writing by the copyright owner as \"Not a Contribution.\"\r\n\r\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\r\n on behalf of whom a Contribution has been received by Licensor and\r\n subsequently incorporated within the Work.\r\n\r\n 2. Grant of Copyright License. Subject to the terms and conditions of\r\n this License, each Contributor hereby grants to You a perpetual,\r\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\r\n copyright license to reproduce, prepare Derivative Works of,\r\n publicly display, publicly perform, sublicense, and distribute the\r\n Work and such Derivative Works in Source or Object form.\r\n\r\n 3. Grant of Patent License. Subject to the terms and conditions of\r\n this License, each Contributor hereby grants to You a perpetual,\r\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\r\n (except as stated in this section) patent license to make, have made,\r\n use, offer to sell, sell, import, and otherwise transfer the Work,\r\n where such license applies only to those patent claims licensable\r\n by such Contributor that are necessarily infringed by their\r\n Contribution(s) alone or by combination of their Contribution(s)\r\n with the Work to which such Contribution(s) was submitted. If You\r\n institute patent litigation against any entity (including a\r\n cross-claim or counterclaim in a lawsuit) alleging that the Work\r\n or a Contribution incorporated within the Work constitutes direct\r\n or contributory patent infringement, then any patent licenses\r\n granted to You under this License for that Work shall terminate\r\n as of the date such litigation is filed.\r\n\r\n 4. Redistribution. You may reproduce and distribute copies of the\r\n Work or Derivative Works thereof in any medium, with or without\r\n modifications, and in Source or Object form, provided that You\r\n meet the following conditions:\r\n\r\n (a) You must give any other recipients of the Work or\r\n Derivative Works a copy of this License; and\r\n\r\n (b) You must cause any modified files to carry prominent notices\r\n stating that You changed the files; and\r\n\r\n (c) You must retain, in the Source form of any Derivative Works\r\n that You distribute, all copyright, patent, trademark, and\r\n attribution notices from the Source form of the Work,\r\n excluding those notices that do not pertain to any part of\r\n the Derivative Works; and\r\n\r\n (d) If the Work includes a \"NOTICE\" text file as part of its\r\n distribution, then any Derivative Works that You distribute must\r\n include a readable copy of the attribution notices contained\r\n within such NOTICE file, excluding those notices that do not\r\n pertain to any part of the Derivative Works, in at least one\r\n of the following places: within a NOTICE text file distributed\r\n as part of the Derivative Works; within the Source form or\r\n documentation, if provided along with the Derivative Works; or,\r\n within a display generated by the Derivative Works, if and\r\n wherever such third-party notices normally appear. The contents\r\n of the NOTICE file are for informational purposes only and\r\n do not modify the License. You may add Your own attribution\r\n notices within Derivative Works that You distribute, alongside\r\n or as an addendum to the NOTICE text from the Work, provided\r\n that such additional attribution notices cannot be construed\r\n as modifying the License.\r\n\r\n You may add Your own copyright statement to Your modifications and\r\n may provide additional or different license terms and conditions\r\n for use, reproduction, or distribution of Your modifications, or\r\n for any such Derivative Works as a whole, provided Your use,\r\n reproduction, and distribution of the Work otherwise complies with\r\n the conditions stated in this License.\r\n\r\n 5. Submission of Contributions. Unless You explicitly state otherwise,\r\n any Contribution intentionally submitted for inclusion in the Work\r\n by You to the Licensor shall be under the terms and conditions of\r\n this License, without any additional terms or conditions.\r\n Notwithstanding the above, nothing herein shall supersede or modify\r\n the terms of any separate license agreement you may have executed\r\n with Licensor regarding such Contributions.\r\n\r\n 6. Trademarks. This License does not grant permission to use the trade\r\n names, trademarks, service marks, or product names of the Licensor,\r\n except as required for reasonable and customary use in describing the\r\n origin of the Work and reproducing the content of the NOTICE file.\r\n\r\n 7. Disclaimer of Warranty. Unless required by applicable law or\r\n agreed to in writing, Licensor provides the Work (and each\r\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\r\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\r\n implied, including, without limitation, any warranties or conditions\r\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\r\n PARTICULAR PURPOSE. You are solely responsible for determining the\r\n appropriateness of using or redistributing the Work and assume any\r\n risks associated with Your exercise of permissions under this License.\r\n\r\n 8. Limitation of Liability. In no event and under no legal theory,\r\n whether in tort (including negligence), contract, or otherwise,\r\n unless required by applicable law (such as deliberate and grossly\r\n negligent acts) or agreed to in writing, shall any Contributor be\r\n liable to You for damages, including any direct, indirect, special,\r\n incidental, or consequential damages of any character arising as a\r\n result of this License or out of the use or inability to use the\r\n Work (including but not limited to damages for loss of goodwill,\r\n work stoppage, computer failure or malfunction, or any and all\r\n other commercial damages or losses), even if such Contributor\r\n has been advised of the possibility of such damages.\r\n\r\n 9. Accepting Warranty or Additional Liability. While redistributing\r\n the Work or Derivative Works thereof, You may choose to offer,\r\n and charge a fee for, acceptance of support, warranty, indemnity,\r\n or other liability obligations and/or rights consistent with this\r\n License. However, in accepting such obligations, You may act only\r\n on Your own behalf and on Your sole responsibility, not on behalf\r\n of any other Contributor, and only if You agree to indemnify,\r\n defend, and hold each Contributor harmless for any liability\r\n incurred by, or claims asserted against, such Contributor by reason\r\n of your accepting any such warranty or additional liability.\r\n\r\n END OF TERMS AND CONDITIONS\r\n\r\n APPENDIX: How to apply the Apache License to your work.\r\n\r\n To apply the Apache License to your work, attach the following\r\n boilerplate notice, with the fields enclosed by brackets \"[]\"\r\n replaced with your own identifying information. (Don't include\r\n the brackets!) The text should be enclosed in the appropriate\r\n comment syntax for the file format. We also recommend that a\r\n file or class name and description of purpose be included on the\r\n same \"printed page\" as the copyright notice for easier\r\n identification within third-party archives.\r\n\r\n Copyright [yyyy] [name of copyright owner]\r\n\r\n Licensed under the Apache License, Version 2.0 (the \"License\");\r\n you may not use this file except in compliance with the License.\r\n You may obtain a copy of the License at\r\n\r\n http://www.apache.org/licenses/LICENSE-2.0\r\n\r\n Unless required by applicable law or agreed to in writing, software\r\n distributed under the License is distributed on an \"AS IS\" BASIS,\r\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n See the License for the specific language governing permissions and\r\n limitations under the License.\r\n" }, { "package": "resvg-js@1.0.0", "sourcePath": "README.md", "sha256": "e3e35220f5dd57ef197086ea5de17cb46e5df78a50501d3e5fa2bc0526942f90", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# resvg-js\r\n\r\n\"GitHub\r\n\"@resvg/resvg-js\r\n\"@resvg/resvg-js\r\n[![Rust 1.65+](https://img.shields.io/badge/rust-1.65+-orange.svg)](https://www.rust-lang.org)\r\n\r\n> resvg-js is a high-performance SVG renderer and toolkit, powered by Rust based [resvg](https://github.com/RazrFalcon/resvg/), with Node.js backend using [napi-rs](https://github.com/napi-rs/napi-rs), also a pure WebAssembly backend.\r\n\r\nPlease use all lowercase `resvg-js` when referencing project names.\r\n\r\n**💖 🙏[Sponsoring me](#sponsor)**\r\n\r\n> Your sponsorship means a lot to me. It will help me sustain my projects actively and make more of my ideas come true. Much appreciated!\r\n\r\n## Features\r\n\r\n- Fast, safe and zero dependencies, with correct output.\r\n- Convert SVG to PNG, includes cropping, scaling and setting the background color.\r\n- Support system fonts and custom fonts in SVG text.\r\n- `v2`: Gets the width and height of the SVG and the generated PNG.\r\n- `v2`: Support for outputting simplified SVG strings, such as converting shapes(rect, circle, etc) to ``.\r\n- `v2`: Support WebAssembly.\r\n- `v2`: Support to get SVG bounding box and crop according to bounding box.\r\n- `v2`: Support for loading images of external links in ``.\r\n- No need for node-gyp and postinstall, the `.node` file has been compiled for you.\r\n- Cross-platform support, including [Apple M Chips](https://www.apple.com/newsroom/2020/11/apple-unleashes-m1/).\r\n- Support for running as native addons in Deno.\r\n\r\n## Sponsor\r\n\r\nAlipay(支付宝)\r\n\r\n\r\n\r\n## Installation\r\n\r\n### Node.js\r\n\r\n```shell\r\nnpm i @resvg/resvg-js\r\n```\r\n\r\n### Browser(Wasm)\r\n\r\n```html\r\n\r\n```\r\n\r\n## Docs\r\n\r\n- Node.js see [index.d.ts](./index.d.ts)\r\n- Wasm see [wasm/index.d.ts](./wasm/index.d.ts)\r\n\r\n## Example\r\n\r\n### [Node.js Example](example/index.js)\r\n\r\nThis example will load Source Han Serif, and then render the SVG to PNG.\r\n\r\n```shell\r\nnode example/index.js\r\n\r\nLoaded 1 font faces in 0ms.\r\nFont './example/SourceHanSerifCN-Light-subset.ttf':0 found in 0.006ms.\r\n✨ Done in 55.65491008758545 ms\r\n```\r\n\r\n### [Deno Example](example/index-deno.js)\r\n\r\n```shell\r\ndeno run --unstable --allow-read --allow-write --allow-ffi example/index-deno.js\r\n\r\n[2022-11-16T15:03:29Z DEBUG resvg_js::fonts] Loaded 1 font faces in 0.067ms.\r\n[2022-11-16T15:03:29Z DEBUG resvg_js::fonts] Font './example/SourceHanSerifCN-Light-subset.ttf':0 found in 0.001ms.\r\nOriginal SVG Size: 1324 x 687\r\nOutput PNG Size : 1200 x 623\r\n✨ Done in 66 ms\r\n```\r\n\r\n| SVG | PNG |\r\n| ---------------------------------------- | -------------------------------------------- |\r\n| | |\r\n\r\n## Usage\r\n\r\n### Node.js\r\n\r\n```js\r\nconst { promises } = require('fs')\r\nconst { join } = require('path')\r\nconst { Resvg } = require('@resvg/resvg-js')\r\n\r\nasync function main() {\r\n const svg = await promises.readFile(join(__dirname, './text.svg'))\r\n const opts = {\r\n background: 'rgba(238, 235, 230, .9)',\r\n fitTo: {\r\n mode: 'width',\r\n value: 1200,\r\n },\r\n font: {\r\n fontFiles: ['./example/SourceHanSerifCN-Light-subset.ttf'], // Load custom fonts.\r\n loadSystemFonts: false, // It will be faster to disable loading system fonts.\r\n // defaultFontFamily: 'Source Han Serif CN Light', // You can omit this.\r\n },\r\n }\r\n const resvg = new Resvg(svg, opts)\r\n const pngData = resvg.render()\r\n const pngBuffer = pngData.asPng()\r\n\r\n console.info('Original SVG Size:', `${resvg.width} x ${resvg.height}`)\r\n console.info('Output PNG Size :', `${pngData.width} x ${pngData.height}`)\r\n\r\n await promises.writeFile(join(__dirname, './text-out.png'), pngBuffer)\r\n}\r\n\r\nmain()\r\n```\r\n\r\n### Bun\r\n\r\nStarting with [Bun 0.8.1](https://github.com/oven-sh/bun/issues/172#issuecomment-1692930718), resvg-js can be run directly in Bun without any modification to the JS files, and is fully compatible with the syntax in Node.js.\r\n\r\n```shell\r\nbun example/index.js\r\n```\r\n\r\n### Deno\r\n\r\nStarting with [Deno 1.26.1](https://github.com/denoland/deno/releases/tag/v1.26.1), there is support for running Native Addons directly from Node.js.\r\nThis allows for performance that is close to that found in Node.js.\r\n\r\n```shell\r\ndeno run --unstable --allow-read --allow-write --allow-ffi example/index-deno.js\r\n```\r\n\r\n```js\r\nimport * as path from 'https://deno.land/std@0.159.0/path/mod.ts'\r\nimport { Resvg } from 'npm:@resvg/resvg-js'\r\nconst __dirname = path.dirname(path.fromFileUrl(import.meta.url))\r\n\r\nconst svg = await Deno.readFile(path.join(__dirname, './text.svg'))\r\nconst opts = {\r\n fitTo: {\r\n mode: 'width',\r\n value: 1200,\r\n },\r\n}\r\n\r\nconst t = performance.now()\r\nconst resvg = new Resvg(svg, opts)\r\nconst pngData = resvg.render()\r\nconst pngBuffer = pngData.asPng()\r\nconsole.info('Original SVG Size:', `${resvg.width} x ${resvg.height}`)\r\nconsole.info('Output PNG Size :', `${pngData.width} x ${pngData.height}`)\r\nconsole.info('✨ Done in', performance.now() - t, 'ms')\r\n\r\nawait Deno.writeFile(path.join(__dirname, './text-out-deno.png'), pngBuffer)\r\n```\r\n\r\n### WebAssembly\r\n\r\nThis package also ships a pure WebAssembly artifact built with `wasm-bindgen` to run in browsers.\r\n\r\n#### Browser\r\n\r\n```html\r\n\r\n\r\n```\r\n\r\nSee [playground](wasm/index.html), it is also possible to [call Wasm in Node.js](example/wasm-node.js), but it is slower.\r\n\r\n## Sample Benchmark\r\n\r\n```shell\r\nnpm i benny@3.x sharp@0.x @types/sharp svg2img@0.x\r\nnpm run bench\r\n```\r\n\r\n```shell\r\nRunning \"resize width\" suite...\r\n resvg-js(Rust):\r\n 12 ops/s\r\n\r\n sharp:\r\n 9 ops/s\r\n\r\n skr-canvas(Rust):\r\n 7 ops/s\r\n\r\n svg2img(canvg and node-canvas):\r\n 6 ops/s\r\n```\r\n\r\n## Support matrix\r\n\r\n| | Node.js 12 | Node.js 14 | Node.js 16 | Node.js 18 | Node.js 20 | npm |\r\n| ---------------- | ---------- | ---------- | ---------- | ---------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\r\n| Windows x64 | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-win32-x64-msvc.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-win32-x64-msvc) |\r\n| Windows x32 | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-win32-ia32-msvc.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-win32-ia32-msvc) |\r\n| Windows arm64 | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-win32-arm64-msvc.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-win32-arm64-msvc) |\r\n| macOS x64 | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-darwin-x64.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-darwin-x64) |\r\n| macOS arm64(M1) | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-darwin-arm64.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-darwin-arm64) |\r\n| Linux x64 gnu | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-linux-x64-gnu.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-linux-x64-gnu) |\r\n| Linux x64 musl | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-linux-x64-musl.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-linux-x64-musl) |\r\n| Linux arm gnu | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-linux-arm-gnueabihf.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-linux-arm-gnueabihf) |\r\n| Linux arm64 gnu | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-linux-arm64-gnu.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-linux-arm64-gnu) |\r\n| Linux arm64 musl | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-linux-arm64-musl.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-linux-arm64-musl) |\r\n| Android arm64 | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-android-arm64.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-android-arm64) |\r\n| Android armv7 | ✓ | ✓ | ✓ | ✓ | ✓ | [![npm version](https://img.shields.io/npm/v/@resvg/resvg-js-android-arm-eabi.svg?sanitize=true)](https://www.npmjs.com/package/@resvg/resvg-js-android-arm-eabi) |\r\n\r\n## Test or Contributing\r\n\r\n- Install latest `Rust`\r\n- Install `Node.js@10+` which fully supported `Node-API`\r\n- Install `wasm-pack`\r\n\r\n ```bash\r\n curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh\r\n ```\r\n\r\n Normally `wasm-pack` will install `wasm-bindgen` automatically, but if the installation [fails due to network reasons](https://github.com/rustwasm/wasm-pack-template/issues/44#issuecomment-521657516), please try to install it manually.\r\n\r\n ```bash\r\n cargo install wasm-bindgen-cli\r\n ```\r\n\r\n On computers with Apple M chips, the following error message may appear:\r\n\r\n > Error: failed to download from https://github.com/WebAssembly/binaryen/releases/download/version_90/binaryen-version_90-x86_64-apple-darwin.tar.gz\r\n\r\n Please install binaryen manually:\r\n\r\n ```bash\r\n brew install binaryen\r\n ```\r\n\r\n### Build Node.js bindings\r\n\r\n```bash\r\nnpm i\r\nnpm run build\r\nnpm test\r\n```\r\n\r\n### Build WebAssembly bindings\r\n\r\n```bash\r\nnpm i\r\nnpm run build:wasm\r\nnpm run test:wasm\r\n```\r\n\r\n## Roadmap\r\n\r\nI will consider implementing the following features, if you happen to be interested,\r\nplease feel free to discuss with me or submit a PR.\r\n\r\n- [x] Support async API\r\n- [x] Upgrade to napi-rs v2\r\n- [x] Support WebAssembly\r\n- [x] Output usvg-simplified SVG string\r\n- [x] Support for getting SVG Bounding box\r\n- [ ] Support for generating more lossless bitmap formats, e.g. avif, webp, JPEG XL\r\n\r\n## Release package\r\n\r\nWe use GitHub actions to automatically publish npm packages.\r\n\r\n```bash\r\n# 1.0.0 => 1.0.1\r\nnpm version patch\r\n\r\n# or 1.0.0 => 1.1.0\r\nnpm version minor\r\n```\r\n\r\n## License\r\n\r\nPlease use all lowercase `resvg-js` when referencing project names.\r\n\r\n[MPLv2.0](https://www.mozilla.org/en-US/MPL/)\r\n\r\nCopyright (c) 2021-present, yisibl(一丝)\r\n" }, { "package": "resvg-js@1.0.0", "sourcePath": "wasm/README.md", "sha256": "f976bda8ccb79e0f7e1e64bde1403f3b130af6ca105ac273e6dac1290f12c268", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# resvg-js\r\n\r\n## @resvg/resvg-wasm\r\n\r\n\"GitHub\r\n\"npm\r\n\"downloads\"\r\n\r\n\r\n> resvg-js is a high-performance SVG renderer and toolkit, powered by Rust based [resvg](https://github.com/RazrFalcon/resvg/) and [napi-rs](https://github.com/napi-rs/napi-rs).\r\n\r\nThis is the **WebAssembly** binary for [`resvg-js`](https://github.com/yisibl/resvg-js)\r\n## Playground\r\n\r\nhttps://resvg-js.vercel.app\r\n\r\n## License\r\n\r\n[MPLv2.0](https://www.mozilla.org/en-US/MPL/)\r\n\r\nCopyright (c) 2021-present, yisibl(一丝)" }, { "package": "roxmltree@0.18.1", "sourcePath": "README.md", "sha256": "4ed952fe6175615766f1a6559d7f2868c2ac190b5cb79567e25fe6dce2f41352", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# roxmltree\n![Build Status](https://github.com/RazrFalcon/roxmltree/workflows/Rust/badge.svg)\n[![Crates.io](https://img.shields.io/crates/v/roxmltree.svg)](https://crates.io/crates/roxmltree)\n[![Documentation](https://docs.rs/roxmltree/badge.svg)](https://docs.rs/roxmltree)\n[![Rust 1.36+](https://img.shields.io/badge/rust-1.36+-orange.svg)](https://www.rust-lang.org)\n\nRepresents an [XML 1.0](https://www.w3.org/TR/xml/) document as a read-only tree.\n\n```rust\n// Find element by id.\nlet doc = roxmltree::Document::parse(\"\")?;\nlet elem = doc.descendants().find(|n| n.attribute(\"id\") == Some(\"rect1\"))?;\nassert!(elem.has_tag_name(\"rect\"));\n```\n\n## Why read-only?\n\nBecause in some cases all you need is to retrieve some data from an XML document.\nAnd for such cases, we can make a lot of optimizations.\n\nAs for *roxmltree*, it's fast not only because it's read-only, but also because\nit uses [xmlparser], which is many times faster than [xml-rs].\nSee the [Performance](#performance) section for details.\n\n## Parsing behavior\n\nSadly, XML can be parsed in many different ways. *roxmltree* tries to mimic the\nbehavior of Python's [lxml](https://lxml.de/).\nBut unlike *lxml*, *roxmltree* does support comments outside the root element.\n\nFor more details see [docs/parsing.md](https://github.com/RazrFalcon/roxmltree/blob/master/docs/parsing.md).\n\n## Alternatives\n\n| Feature/Crate | roxmltree | [libxml2] | [xmltree] | [sxd-document] |\n| ------------------------------- | :--------------: | :-----------------: | :--------------: | :--------------: |\n| Element namespace resolving | ✓ | ✓ | ✓ | ~1 |\n| Attribute namespace resolving | ✓ | ✓ | | ✓ |\n| [Entity references] | ✓ | ✓ | × | × |\n| [Character references] | ✓ | ✓ | ✓ | ✓ |\n| [Attribute-Value normalization] | ✓ | ✓ | | |\n| Comments | ✓ | ✓ | | ✓ |\n| Processing instructions | ✓ | ✓ | ✓ | ✓ |\n| UTF-8 BOM | ✓ | ✓ | × | × |\n| Non UTF-8 input | | ✓ | | |\n| Complete DTD support | | ✓ | | |\n| Position preserving2 | ✓ | ✓ | | |\n| HTML support | | ✓ | | |\n| Tree modification | | ✓ | ✓ | ✓ |\n| Writing | | ✓ | ✓ | ✓ |\n| No **unsafe** | ✓ | | ✓ | |\n| Language | Rust | C | Rust | Rust |\n| Size overhead4 | **~55KiB** | ~1.4MiB5 | ~78KiB | ~102KiB |\n| Dependencies | **1** | ?5 | 2 | 2 |\n| Tested version | 0.18.0 | Apple-provided | 0.10.3 | 0.3.2 |\n| License | MIT / Apache-2.0 | MIT | MIT | MIT |\n\nLegend:\n\n- ✓ - supported\n- × - parsing error\n- ~ - partial\n- *nothing* - not supported\n\nNotes:\n\n1. No default namespace propagation.\n2. *roxmltree* keeps all node and attribute positions in the original document,\n so you can easily retrieve it if you need it.\n See [examples/print_pos.rs](examples/print_pos.rs) for details.\n3. In the `memchr` crate.\n4. Binary size overhead according to [cargo-bloat](https://github.com/RazrFalcon/cargo-bloat).\n5. Depends on build flags.\n\nThere is also `elementtree` and `treexml` crates, but they are abandoned for a long time.\n\n[Entity references]: https://www.w3.org/TR/REC-xml/#dt-entref\n[Character references]: https://www.w3.org/TR/REC-xml/#NT-CharRef\n[Attribute-Value Normalization]: https://www.w3.org/TR/REC-xml/#AVNormalize\n\n[libxml2]: http://xmlsoft.org/\n[xmltree]: https://crates.io/crates/xmltree\n[sxd-document]: https://crates.io/crates/sxd-document\n\n## Performance\n\n### Parsing\n\n```text\ntest huge_roxmltree ... bench: 3,147,424 ns/iter (+/- 49,153)\ntest huge_libxml ... bench: 6,850,666 ns/iter (+/- 306,180)\ntest huge_sdx_document ... bench: 9,440,412 ns/iter (+/- 117,106)\ntest huge_xmltree ... bench: 41,662,316 ns/iter (+/- 850,360)\n\ntest large_roxmltree ... bench: 1,594,201 ns/iter (+/- 27,425)\ntest large_libxml ... bench: 3,250,606 ns/iter (+/- 140,201)\ntest large_sdx_document ... bench: 4,242,162 ns/iter (+/- 99,740)\ntest large_xmltree ... bench: 13,980,228 ns/iter (+/- 229,363)\n\ntest medium_roxmltree ... bench: 418,929 ns/iter (+/- 4,843)\ntest medium_libxml ... bench: 950,984 ns/iter (+/- 34,099)\ntest medium_sdx_document ... bench: 1,618,270 ns/iter (+/- 23,466)\ntest medium_xmltree ... bench: 4,315,974 ns/iter (+/- 31,849)\n\ntest tiny_roxmltree ... bench: 2,654 ns/iter (+/- 103)\ntest tiny_libxml ... bench: 8,931 ns/iter (+/- 235)\ntest tiny_sdx_document ... bench: 11,658 ns/iter (+/- 82)\ntest tiny_xmltree ... bench: 20,215 ns/iter (+/- 303)\n```\n\n*roxmltree* uses [xmlparser] internally,\nwhile *sdx-document* uses its own implementation,\n*xmltree* uses the [xml-rs].\nHere is a comparison between *xmlparser*, *xml-rs* and *quick-xml*:\n\n```text\ntest huge_xmlparser ... bench: 1,672,879 ns/iter (+/- 20,140)\ntest huge_quick_xml ... bench: 2,396,037 ns/iter (+/- 39,752)\ntest huge_xmlrs ... bench: 36,258,312 ns/iter (+/- 180,438)\n\ntest large_xmlparser ... bench: 730,787 ns/iter (+/- 22,924)\ntest large_quick_xml ... bench: 1,250,053 ns/iter (+/- 21,943)\ntest large_xmlrs ... bench: 11,239,516 ns/iter (+/- 76,937)\n\ntest medium_quick_xml ... bench: 206,232 ns/iter (+/- 2,157)\ntest medium_xmlparser ... bench: 240,737 ns/iter (+/- 4,531)\ntest medium_xmlrs ... bench: 3,975,916 ns/iter (+/- 44,967)\n\ntest tiny_xmlparser ... bench: 1,078 ns/iter (+/- 17)\ntest tiny_quick_xml ... bench: 2,233 ns/iter (+/- 70)\ntest tiny_xmlrs ... bench: 17,155 ns/iter (+/- 429)\n```\n\n### Iteration\n\n```text\ntest roxmltree_iter_descendants_expensive ... bench: 255,882 ns/iter (+/- 1,427)\ntest xmltree_iter_descendants_expensive ... bench: 372,464 ns/iter (+/- 52,938)\n\ntest roxmltree_iter_descendants_inexpensive ... bench: 20,800 ns/iter (+/- 642)\ntest xmltree_iter_descendants_inexpensive ... bench: 133,030 ns/iter (+/- 10,546)\n\ntest roxmltree_iter_children ... bench: 2,168 ns/iter (+/- 12)\n```\n\nWhere expensive refers to the matching done on each element. In these\nbenchmarks, *expensive* means searching for any node in the document which\ncontains a string. And *inexpensive* means searching for any element with a\nparticular name.\n\n### Notes\n\nThe benchmarks were taken on a Apple M1 Pro.\nYou can try running the benchmarks yourself by running `cargo bench` in the `benches` dir.\n\n- Since all libraries have a different XML support, benchmarking is a bit pointless.\n- Tree crates may use different *xml-rs* crate versions.\n- We bench *libxml2* using the *[rust-libxml]* wrapper crate\n- *quick-xml* is faster than *xmlparser* because it's more forgiving for the input,\n while *xmlparser* is very strict and does a lot of checks, which are expensive.\n So performance difference is mainly due to validation.\n\n[xml-rs]: https://crates.io/crates/xml-rs\n[quick-xml]: https://crates.io/crates/quick-xml\n[xmlparser]: https://crates.io/crates/xmlparser\n[rust-libxml]: https://github.com/KWARC/rust-libxml\n\n## Memory Overhead\n\n`roxmltree` tries to use as little memory as possible to allow parsing\nvery large (multi-GB) XML files.\n\nThe peak memory usage doesn't directly correlates with the file size\nbut rather with the amount of nodes and attributes a file has.\nHow many attributes had to be normalized (i.e. allocated).\nAnd how many text nodes had to be preprocessed (i.e. allocated).\n\n`roxmltree` never allocates element and attribute names, processing instructions\nand comments.\n\nBy disabling the `positions` feature, you can shave by 8 bytes from each node and attribute.\n\nOn average, the overhead is around 6-8x the file size.\nFor example, our 1.1GB sample XML will peak at 7.6GB RAM with default features enabled\nand at 6.8GB RAM when `positions` is disabled.\n\n## Safety\n\n- This library must not panic. Any panic should be considered a critical bug and reported.\n- This library forbids `unsafe` code.\n\n## Non-goals\n\n- Complete XML support.\n- Tree modification and writing.\n- XPath/XQuery.\n\n## API\n\nThis library uses Rust's idiomatic API based on iterators.\nIn case you are more familiar with browser/JS DOM APIs - you can check out\n[tests/dom-api.rs](tests/dom-api.rs) to see how it can be converted into a Rust one.\n\n## License\n\nLicensed under either of\n\n- [Apache License v2.0](LICENSE-APACHE)\n- [MIT license](LICENSE-MIT)\n\nat your option.\n\n## Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "package": "roxmltree@0.20.0", "sourcePath": "README.md", "sha256": "4fe693e39a50b30070b276e1f8bf6a5e9017b62423d766d986ae3f269f89b19d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# roxmltree\n![Build Status](https://github.com/RazrFalcon/roxmltree/workflows/Rust/badge.svg)\n[![Crates.io](https://img.shields.io/crates/v/roxmltree.svg)](https://crates.io/crates/roxmltree)\n[![Documentation](https://docs.rs/roxmltree/badge.svg)](https://docs.rs/roxmltree)\n[![Rust 1.60+](https://img.shields.io/badge/rust-1.60+-orange.svg)](https://www.rust-lang.org)\n\nRepresents an [XML](https://www.w3.org/TR/xml/) document as a read-only tree.\n\n```rust\n// Find element by id.\nlet doc = roxmltree::Document::parse(\"\")?;\nlet elem = doc.descendants().find(|n| n.attribute(\"id\") == Some(\"rect1\"))?;\nassert!(elem.has_tag_name(\"rect\"));\n```\n\n## Why read-only?\n\nBecause in some cases all you need is to retrieve some data from an XML document.\nAnd for such cases, we can make a lot of optimizations.\n\n## Parsing behavior\n\nSadly, XML can be parsed in many different ways. *roxmltree* tries to mimic the\nbehavior of Python's [lxml](https://lxml.de/).\nFor more details see [docs/parsing.md](https://github.com/RazrFalcon/roxmltree/blob/master/docs/parsing.md).\n\n## Alternatives\n\n| Feature/Crate | roxmltree | [libxml2] | [xmltree] | [sxd-document] |\n| ------------------------------- | :--------------: | :-----------------: | :--------------: | :--------------: |\n| Element namespace resolving | ✓ | ✓ | ✓ | ~1 |\n| Attribute namespace resolving | ✓ | ✓ | | ✓ |\n| [Entity references] | ✓ | ✓ | × | × |\n| [Character references] | ✓ | ✓ | ✓ | ✓ |\n| [Attribute-Value normalization] | ✓ | ✓ | | |\n| Comments | ✓ | ✓ | | ✓ |\n| Processing instructions | ✓ | ✓ | ✓ | ✓ |\n| UTF-8 BOM | ✓ | ✓ | × | × |\n| Non UTF-8 input | | ✓ | | |\n| Complete DTD support | | ✓ | | |\n| Position preserving2 | ✓ | ✓ | | |\n| HTML support | | ✓ | | |\n| Tree modification | | ✓ | ✓ | ✓ |\n| Writing | | ✓ | ✓ | ✓ |\n| No **unsafe** | ✓ | | ✓ | |\n| Language | Rust | C | Rust | Rust |\n| Dependencies | **0** | - | 2 | 2 |\n| Tested version | 0.20.0 | Apple-provided | 0.10.3 | 0.3.2 |\n| License | MIT / Apache-2.0 | MIT | MIT | MIT |\n\nLegend:\n\n- ✓ - supported\n- × - parsing error\n- ~ - partial\n- *nothing* - not supported\n\nNotes:\n\n1. No default namespace propagation.\n2. *roxmltree* keeps all node and attribute positions in the original document,\n so you can easily retrieve it if you need it.\n See [examples/print_pos.rs](examples/print_pos.rs) for details.\n\nThere is also `elementtree` and `treexml` crates, but they are abandoned for a long time.\n\n[Entity references]: https://www.w3.org/TR/REC-xml/#dt-entref\n[Character references]: https://www.w3.org/TR/REC-xml/#NT-CharRef\n[Attribute-Value Normalization]: https://www.w3.org/TR/REC-xml/#AVNormalize\n\n[libxml2]: http://xmlsoft.org/\n[xmltree]: https://crates.io/crates/xmltree\n[sxd-document]: https://crates.io/crates/sxd-document\n\n## Performance\n\nHere are some benchmarks comparing `roxmltree` to other XML tree libraries.\n\n```text\ntest huge_roxmltree ... bench: 2,997,887 ns/iter (+/- 48,976)\ntest huge_libxml2 ... bench: 6,850,666 ns/iter (+/- 306,180)\ntest huge_sdx_document ... bench: 9,440,412 ns/iter (+/- 117,106)\ntest huge_xmltree ... bench: 41,662,316 ns/iter (+/- 850,360)\n\ntest large_roxmltree ... bench: 1,494,886 ns/iter (+/- 30,384)\ntest large_libxml2 ... bench: 3,250,606 ns/iter (+/- 140,201)\ntest large_sdx_document ... bench: 4,242,162 ns/iter (+/- 99,740)\ntest large_xmltree ... bench: 13,980,228 ns/iter (+/- 229,363)\n\ntest medium_roxmltree ... bench: 421,137 ns/iter (+/- 13,855)\ntest medium_libxml2 ... bench: 950,984 ns/iter (+/- 34,099)\ntest medium_sdx_document ... bench: 1,618,270 ns/iter (+/- 23,466)\ntest medium_xmltree ... bench: 4,315,974 ns/iter (+/- 31,849)\n\ntest tiny_roxmltree ... bench: 2,522 ns/iter (+/- 31)\ntest tiny_libxml2 ... bench: 8,931 ns/iter (+/- 235)\ntest tiny_sdx_document ... bench: 11,658 ns/iter (+/- 82)\ntest tiny_xmltree ... bench: 20,215 ns/iter (+/- 303)\n```\n\nWhen comparing to streaming XML parsers `roxmltree` is slightly slower than `quick-xml`,\nbut still way faster than `xmlrs`.\nNote that streaming parsers usually do not provide a proper string unescaping,\nDTD resolving and namespaces support.\n\n```text\ntest huge_quick_xml ... bench: 2,997,887 ns/iter (+/- 48,976)\ntest huge_roxmltree ... bench: 3,147,424 ns/iter (+/- 49,153)\ntest huge_xmlrs ... bench: 36,258,312 ns/iter (+/- 180,438)\n\ntest large_quick_xml ... bench: 1,250,053 ns/iter (+/- 21,943)\ntest large_roxmltree ... bench: 1,494,886 ns/iter (+/- 30,384)\ntest large_xmlrs ... bench: 11,239,516 ns/iter (+/- 76,937)\n\ntest medium_quick_xml ... bench: 206,232 ns/iter (+/- 2,157)\ntest medium_roxmltree ... bench: 421,137 ns/iter (+/- 13,855)\ntest medium_xmlrs ... bench: 3,975,916 ns/iter (+/- 44,967)\n\ntest tiny_quick_xml ... bench: 2,233 ns/iter (+/- 70)\ntest tiny_roxmltree ... bench: 2,522 ns/iter (+/- 31)\ntest tiny_xmlrs ... bench: 17,155 ns/iter (+/- 429)\n```\n\n### Notes\n\nThe benchmarks were taken on a Apple M1 Pro.\nYou can try running the benchmarks yourself by running `cargo bench` in the `benches` dir.\n\n- Since all libraries have a different XML support, benchmarking is a bit pointless.\n- We bench *libxml2* using the *[rust-libxml]* wrapper crate\n\n[xml-rs]: https://crates.io/crates/xml-rs\n[quick-xml]: https://crates.io/crates/quick-xml\n[rust-libxml]: https://github.com/KWARC/rust-libxml\n\n## Memory overhead\n\n`roxmltree` tries to use as little memory as possible to allow parsing\nvery large (multi-GB) XML files.\n\nThe peak memory usage doesn't directly correlate with the file size\nbut rather with the amount of nodes and attributes a file has.\nHow many attributes had to be normalized (i.e. allocated).\nAnd how many text nodes had to be preprocessed (i.e. allocated).\n\n`roxmltree` never allocates element and attribute names, processing instructions\nand comments.\n\nBy disabling the `positions` feature, you can shave 8 bytes from each node and attribute.\n\nOn average, the overhead is around 6-8x the file size.\nFor example, our 1.1GB sample XML will peak at 7.6GB RAM with default features enabled\nand at 6.8GB RAM when `positions` is disabled.\n\n## Safety\n\n- This library must not panic. Any panic should be considered a critical bug and reported.\n- This library forbids `unsafe` code.\n\n## API\n\nThis library uses Rust's idiomatic API based on iterators.\nIn case you are more familiar with browser/JS DOM APIs - you can check out\n[tests/dom-api.rs](tests/dom-api.rs) to see how it can be mapped onto the Rust one.\n\n## License\n\nLicensed under either of\n\n- [Apache License v2.0](LICENSE-APACHE)\n- [MIT license](LICENSE-MIT)\n\nat your option.\n\n## Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "package": "rustc_version@0.4.1", "sourcePath": "README.md", "sha256": "36d55c32ba7174f7e705faf42eab21fd6e76e9b69093715f403ba9c48e4f9f2e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "rustc-version-rs\n================\n\n[![Documentation](https://docs.rs/rustc_version/badge.svg)](https://docs.rs/rustc_version/)\n[![Crates.io](https://img.shields.io/crates/v/rustc_version.svg)](https://crates.io/crates/rustc_version)\n[![Build status](https://github.com/djc/rustc-version-rs/workflows/CI/badge.svg)](https://github.com/djc/rustc-version-rs/actions?query=workflow%3ACI)\n\nA library for querying the version of a `rustc` compiler.\n\nThis can be used by build scripts or other tools dealing with Rust sources\nto make decisions based on the version of the compiler. Current MSRV is 1.32.0.\n\nIf this is of interest, also consider looking at these other crates:\n\n* [autocfg](https://crates.io/crates/autocfg/), which helps with feature detection instead of depending on compiler versions\n* [rustversion](https://github.com/dtolnay/rustversion) provides a procedural macro with no other dependencies\n\n# Getting Started\n\n[rustc-version-rs is available on crates.io](https://crates.io/crates/rustc_version).\nIt is recommended to look there for the newest released version, as well as links to the newest builds of the docs.\n\nAt the point of the last update of this README, the latest published version could be used like this:\n\nAdd the following dependency to your Cargo manifest...\n\n```toml\n[build-dependencies]\nrustc_version = \"0.2\"\n```\n\n... and see the [docs](https://docs.rs/rustc_version) for how to use it.\n\n# Example\n\n```rust\n// This could be a cargo build script\n\nuse rustc_version::{version, version_meta, Channel, Version};\n\nfn main() {\n // Assert we haven't travelled back in time\n assert!(version().unwrap().major >= 1);\n\n // Set cfg flags depending on release channel\n match version_meta().unwrap().channel {\n Channel::Stable => {\n println!(\"cargo:rustc-cfg=RUSTC_IS_STABLE\");\n }\n Channel::Beta => {\n println!(\"cargo:rustc-cfg=RUSTC_IS_BETA\");\n }\n Channel::Nightly => {\n println!(\"cargo:rustc-cfg=RUSTC_IS_NIGHTLY\");\n }\n Channel::Dev => {\n println!(\"cargo:rustc-cfg=RUSTC_IS_DEV\");\n }\n }\n\n // Check for a minimum version\n if version().unwrap() >= Version::parse(\"1.4.0\").unwrap() {\n println!(\"cargo:rustc-cfg=compiler_has_important_bugfix\");\n }\n}\n```\n\n## License\n\nLicensed under either of\n\n * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any\nadditional terms or conditions.\n" }, { "package": "rustc_version@0.4.1", "sourcePath": "src/lib.rs", "sha256": "6d0dbf7bd12ba7121e6767841a0cee23b8282cc4d7c4b778a1e33129abdd7d72", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2016 rustc-version-rs developers\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n#![warn(missing_docs)]\n\n//! Simple library for getting the version information of a `rustc`\n//! compiler.\n//!\n//! This can be used by build scripts or other tools dealing with Rust sources\n//! to make decisions based on the version of the compiler.\n//!\n//! It calls `$RUSTC --version -v` and parses the output, falling\n//! back to `rustc` if `$RUSTC` is not set.\n//!\n//! # Example\n//!\n//! ```rust\n//! // This could be a cargo build script\n//!\n//! use rustc_version::{version, version_meta, Channel, Version};\n//!\n//! // Assert we haven't travelled back in time\n//! assert!(version().unwrap().major >= 1);\n//!\n//! // Set cfg flags depending on release channel\n//! match version_meta().unwrap().channel {\n//! Channel::Stable => {\n//! println!(\"cargo:rustc-cfg=RUSTC_IS_STABLE\");\n//! }\n//! Channel::Beta => {\n//! println!(\"cargo:rustc-cfg=RUSTC_IS_BETA\");\n//! }\n//! Channel::Nightly => {\n//! println!(\"cargo:rustc-cfg=RUSTC_IS_NIGHTLY\");\n//! }\n//! Channel::Dev => {\n//! println!(\"cargo:rustc-cfg=RUSTC_IS_DEV\");\n//! }\n//! }\n//!\n//! // Check for a minimum version\n//! if version().unwrap() >= Version::parse(\"1.4.0\").unwrap() {\n//! println!(\"cargo:rustc-cfg=compiler_has_important_bugfix\");\n//! }\n//! ```\n\n#[cfg(test)]\n#[macro_use]\nextern crate doc_comment;\n\n#[cfg(test)]\ndoctest!(\"../README.md\");\n\nuse std::collections::HashMap;\nuse std::process::Command;\nuse std::{env, error, fmt, io, num, str};\nuse std::{ffi::OsString, str::FromStr};\n\n// Convenience re-export to allow version comparison without needing to add\n// semver crate.\npub use semver::Version;\n\nuse Error::*;\n\n/// Release channel of the compiler.\n#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)]\npub enum Channel {\n /// Development release channel\n Dev,\n /// Nightly release channel\n Nightly,\n /// Beta release channel\n Beta,\n /// Stable release channel\n Stable,\n}\n\n/// LLVM version\n///\n/// LLVM's version numbering scheme is not semver compatible until version 4.0\n///\n/// rustc [just prints the major and minor versions], so other parts of the version are not included.\n///\n/// [just prints the major and minor versions]: https://github.com/rust-lang/rust/blob/b5c9e2448c9ace53ad5c11585803894651b18b0a/compiler/rustc_codegen_llvm/src/llvm_util.rs#L173-L178\n#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]\npub struct LlvmVersion {\n // fields must be ordered major, minor for comparison to be correct\n /// Major version\n pub major: u64,\n /// Minor version\n pub minor: u64,\n // TODO: expose micro version here\n}\n\nimpl fmt::Display for LlvmVersion {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(f, \"{}.{}\", self.major, self.minor)\n }\n}\n\nimpl FromStr for LlvmVersion {\n type Err = LlvmVersionParseError;\n\n fn from_str(s: &str) -> Result {\n let mut parts = s\n .split('.')\n .map(|part| -> Result {\n if part == \"0\" {\n Ok(0)\n } else if part.starts_with('0') {\n Err(LlvmVersionParseError::ComponentMustNotHaveLeadingZeros)\n } else if part.starts_with('-') || part.starts_with('+') {\n Err(LlvmVersionParseError::ComponentMustNotHaveSign)\n } else {\n Ok(part.parse()?)\n }\n });\n\n let major = parts.next().unwrap()?;\n let mut minor = 0;\n\n if let Some(part) = parts.next() {\n minor = part?;\n } else if major < 4 {\n // LLVM versions earlier than 4.0 have significant minor versions, so require the minor version in this case.\n return Err(LlvmVersionParseError::MinorVersionRequiredBefore4);\n }\n\n if let Some(Err(e)) = parts.next() {\n return Err(e);\n }\n\n if parts.next().is_some() {\n return Err(LlvmVersionParseError::TooManyComponents);\n }\n\n Ok(Self { major, minor })\n }\n}\n\n/// Rustc version plus metadata like git short hash and build date.\n#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]\npub struct VersionMeta {\n /// Version of the compiler\n pub semver: Version,\n\n /// Git short hash of the build of the compiler\n pub commit_hash: Option,\n\n /// Commit date of the compiler\n pub commit_date: Option,\n\n /// Build date of the compiler; this was removed between Rust 1.0.0 and 1.1.0.\n pub build_date: Option,\n\n /// Release channel of the compiler\n pub channel: Channel,\n\n /// Host target triple of the compiler\n pub host: String,\n\n /// Short version string of the compiler\n pub short_version_string: String,\n\n /// Version of LLVM used by the compiler\n pub llvm_version: Option,\n}\n\nimpl VersionMeta {\n /// Returns the version metadata for `cmd`, which should be a `rustc` command.\n pub fn for_command(mut cmd: Command) -> Result {\n let out = cmd\n .arg(\"-vV\")\n .output()\n .map_err(Error::CouldNotExecuteCommand)?;\n\n if !out.status.success() {\n return Err(Error::CommandError {\n stdout: String::from_utf8_lossy(&out.stdout).into(),\n stderr: String::from_utf8_lossy(&out.stderr).into(),\n });\n }\n\n version_meta_for(str::from_utf8(&out.stdout)?)\n }\n}\n\n/// Returns the `rustc` SemVer version.\npub fn version() -> Result {\n Ok(version_meta()?.semver)\n}\n\n/// Returns the `rustc` SemVer version and additional metadata\n/// like the git short hash and build date.\npub fn version_meta() -> Result {\n let rustc = env::var_os(\"RUSTC\").unwrap_or_else(|| OsString::from(\"rustc\"));\n let cmd = if let Some(wrapper) = env::var_os(\"RUSTC_WRAPPER\").filter(|w| !w.is_empty()) {\n let mut cmd = Command::new(wrapper);\n cmd.arg(rustc);\n cmd\n } else {\n Command::new(rustc)\n };\n\n VersionMeta::for_command(cmd)\n}\n\n/// Parses a \"rustc -vV\" output string and returns\n/// the SemVer version and additional metadata\n/// like the git short hash and build date.\npub fn version_meta_for(verbose_version_string: &str) -> Result {\n let mut map = HashMap::new();\n for (i, line) in verbose_version_string.lines().enumerate() {\n if i == 0 {\n map.insert(\"short\", line);\n continue;\n }\n\n let mut parts = line.splitn(2, \": \");\n let key = match parts.next() {\n Some(key) => key,\n None => continue,\n };\n\n if let Some(value) = parts.next() {\n map.insert(key, value);\n }\n }\n\n let short_version_string = expect_key(\"short\", &map)?;\n let host = expect_key(\"host\", &map)?;\n let release = expect_key(\"release\", &map)?;\n let semver: Version = release.parse()?;\n\n let channel = match semver.pre.split('.').next().unwrap() {\n \"\" => Channel::Stable,\n \"dev\" => Channel::Dev,\n \"beta\" => Channel::Beta,\n \"nightly\" => Channel::Nightly,\n x => return Err(Error::UnknownPreReleaseTag(x.to_owned())),\n };\n\n let commit_hash = expect_key_or_unknown(\"commit-hash\", &map)?;\n let commit_date = expect_key_or_unknown(\"commit-date\", &map)?;\n let build_date = map\n .get(\"build-date\")\n .filter(|&v| *v != \"unknown\")\n .map(|&v| String::from(v));\n let llvm_version = match map.get(\"LLVM version\") {\n Some(&v) => Some(v.parse()?),\n None => None,\n };\n\n Ok(VersionMeta {\n semver,\n commit_hash,\n commit_date,\n build_date,\n channel,\n host,\n short_version_string,\n llvm_version,\n })\n}\n\nfn expect_key_or_unknown(key: &str, map: &HashMap<&str, &str>) -> Result, Error> {\n match map.get(key) {\n Some(&\"unknown\") => Ok(None),\n Some(&v) => Ok(Some(String::from(v))),\n None => Err(Error::UnexpectedVersionFormat),\n }\n}\n\nfn expect_key(key: &str, map: &HashMap<&str, &str>) -> Result {\n map.get(key)\n .map(|&v| String::from(v))\n .ok_or(Error::UnexpectedVersionFormat)\n}\n\n/// LLVM Version Parse Error\n#[derive(Debug)]\npub enum LlvmVersionParseError {\n /// An error occurred in parsing a version component as an integer\n ParseIntError(num::ParseIntError),\n /// A version component must not have leading zeros\n ComponentMustNotHaveLeadingZeros,\n /// A version component has a sign\n ComponentMustNotHaveSign,\n /// Minor version component must be zero on LLVM versions later than 4.0\n MinorVersionMustBeZeroAfter4,\n /// Minor version component is required on LLVM versions earlier than 4.0\n MinorVersionRequiredBefore4,\n /// Too many components\n TooManyComponents,\n}\n\nimpl From for LlvmVersionParseError {\n fn from(e: num::ParseIntError) -> Self {\n LlvmVersionParseError::ParseIntError(e)\n }\n}\n\nimpl fmt::Display for LlvmVersionParseError {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match self {\n LlvmVersionParseError::ParseIntError(e) => {\n write!(f, \"error parsing LLVM version component: {}\", e)\n }\n LlvmVersionParseError::ComponentMustNotHaveLeadingZeros => {\n write!(f, \"a version component must not have leading zeros\")\n }\n LlvmVersionParseError::ComponentMustNotHaveSign => {\n write!(f, \"a version component must not have a sign\")\n }\n LlvmVersionParseError::MinorVersionMustBeZeroAfter4 => write!(\n f,\n \"LLVM's minor version component must be 0 for versions greater than 4.0\"\n ),\n LlvmVersionParseError::MinorVersionRequiredBefore4 => write!(\n f,\n \"LLVM's minor version component is required for versions less than 4.0\"\n ),\n LlvmVersionParseError::TooManyComponents => write!(f, \"too many version components\"),\n }\n }\n}\n\nimpl error::Error for LlvmVersionParseError {\n fn source(&self) -> Option<&(dyn error::Error + 'static)> {\n match self {\n LlvmVersionParseError::ParseIntError(e) => Some(e),\n LlvmVersionParseError::ComponentMustNotHaveLeadingZeros\n | LlvmVersionParseError::ComponentMustNotHaveSign\n | LlvmVersionParseError::MinorVersionMustBeZeroAfter4\n | LlvmVersionParseError::MinorVersionRequiredBefore4\n | LlvmVersionParseError::TooManyComponents => None,\n }\n }\n}\n\n/// The error type for this crate.\n#[derive(Debug)]\npub enum Error {\n /// An error occurred while trying to find the `rustc` to run.\n CouldNotExecuteCommand(io::Error),\n /// Error output from the command that was run.\n CommandError {\n /// stdout output from the command\n stdout: String,\n /// stderr output from the command\n stderr: String,\n },\n /// The output of `rustc -vV` was not valid utf-8.\n Utf8Error(str::Utf8Error),\n /// The output of `rustc -vV` was not in the expected format.\n UnexpectedVersionFormat,\n /// An error occurred in parsing the semver.\n SemVerError(semver::Error),\n /// The pre-release tag is unknown.\n UnknownPreReleaseTag(String),\n /// An error occurred in parsing a `LlvmVersion`.\n LlvmVersionError(LlvmVersionParseError),\n}\n\nimpl fmt::Display for Error {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match *self {\n CouldNotExecuteCommand(ref e) => write!(f, \"could not execute command: {}\", e),\n CommandError {\n ref stdout,\n ref stderr,\n } => write!(\n f,\n \"error from command -- stderr:\\n\\n{}\\n\\nstderr:\\n\\n{}\",\n stderr, stdout,\n ),\n Utf8Error(_) => write!(f, \"invalid UTF-8 output from `rustc -vV`\"),\n UnexpectedVersionFormat => write!(f, \"unexpected `rustc -vV` format\"),\n SemVerError(ref e) => write!(f, \"error parsing version: {}\", e),\n UnknownPreReleaseTag(ref i) => write!(f, \"unknown pre-release tag: {}\", i),\n LlvmVersionError(ref e) => write!(f, \"error parsing LLVM's version: {}\", e),\n }\n }\n}\n\nimpl error::Error for Error {\n fn source(&self) -> Option<&(dyn error::Error + 'static)> {\n match *self {\n CouldNotExecuteCommand(ref e) => Some(e),\n CommandError { .. } => None,\n Utf8Error(ref e) => Some(e),\n UnexpectedVersionFormat => None,\n SemVerError(ref e) => Some(e),\n UnknownPreReleaseTag(_) => None,\n LlvmVersionError(ref e) => Some(e),\n }\n }\n}\n\nmacro_rules! impl_from {\n ($($err_ty:ty => $variant:ident),* $(,)*) => {\n $(\n impl From<$err_ty> for Error {\n fn from(e: $err_ty) -> Error {\n Error::$variant(e)\n }\n }\n )*\n }\n}\n\nimpl_from! {\n str::Utf8Error => Utf8Error,\n semver::Error => SemVerError,\n LlvmVersionParseError => LlvmVersionError,\n}\n\n/// The result type for this crate.\npub type Result = std::result::Result;\n" }, { "package": "rustybuzz@0.7.0", "sourcePath": "README.md", "sha256": "906a15703d1dd77945b73c1d9b4ebc2ecd6707a2ed591a1fbe86205b8d34631b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# rustybuzz\n![Build Status](https://github.com/RazrFalcon/rustybuzz/workflows/Rust/badge.svg)\n[![Crates.io](https://img.shields.io/crates/v/rustybuzz.svg)](https://crates.io/crates/rustybuzz)\n[![Documentation](https://docs.rs/rustybuzz/badge.svg)](https://docs.rs/rustybuzz)\n\n`rustybuzz` is a complete [harfbuzz](https://github.com/harfbuzz/harfbuzz)'s\nshaping algorithm port to Rust.\n\nMatches `harfbuzz` v2.7.1\n\n## Why?\n\nBecause you can add `rustybuzz = \"*\"` to your project and it just works.\nNo need for a C++ compiler. No need to configure anything. No need to link to system libraries.\n\n## Conformance\n\nrustybuzz passes 98% of harfbuzz tests (1764 to be more precise).\nSo it's mostly identical, but there are still some tiny edge-cases which\nare not implemented yet or cannot be implemented at all.\n\nAlso, Apple layout is largely untested, because we cannot include Apple fonts for legal reasons.\nharfbuzz uses macOS CI instances to test it, which is extremely painful\nand we do not do this for now.\n\n## Major changes\n\n- Subsetting removed.\n- TrueType parsing is completely handled by the\n [ttf-parser](https://github.com/RazrFalcon/ttf-parser).\n And while the parsing algorithm is very different, it's not better or worse, just different.\n- Malformed fonts will cause an error. HarfBuzz uses fallback/dummy shaper in this case.\n- No font size property. Shaping is always using UnitsPerEm. You should scale the result manually.\n- Most of the TrueType and Unicode handling code was moved into separate crates.\n- rustybuzz doesn't interact with any system libraries and must produce exactly the same\n results on all OS'es and targets.\n- `mort` table is not supported, since it's deprecated by Apple.\n- No Arabic fallback shaper, since it requires subsetting.\n- No `graphite` library support.\n- No automated Apple layout testing for legal reasons. We just cannot include Apple fonts.\n harfbuzz avoids this by running such tests only on CI, which is far from ideal.\n\n## Performance\n\nAt the moment, performance isn't that great. We're 1.5-2x slower than harfbuzz.\nAlso, rustybuzz doesn't support shaping plan caching at the moment.\n\nSee [benches/README.md](./benches/README.md) for details.\n\n## Notes about the port\n\nrustybuzz is not a faithful port.\n\nharfbuzz can roughly be split into 6 parts: shaping, subsetting, TrueType parsing,\nUnicode routines, custom containers and utilities (harfbuzz doesn't use C++ std)\nand glue for system/3rd party libraries. In the mean time, rustybuzz contains only shaping.\nAll of the TrueType parsing was moved to the [ttf-parser](https://github.com/RazrFalcon/ttf-parser).\nSubsetting was removed. Unicode code was mostly moved to external crates.\nWe don't need custom containers because Rust's std is good enough.\nAnd we do not use any non Rust libraries, so no glue code either.\n\nIn the end, we still have around 20 KLOC. While harfbuzz is around 80 KLOC.\n\n## Lines of code\n\nAs mentioned above, rustybuzz has around 20 KLOC. But this is not strictly true,\nbecause there are a lot of auto-generated data tables.\n\nYou can find the \"real\" code size using:\n\n```sh\ntokei --exclude unicode_norm.rs --exclude complex/vowel_constraints.rs \\\n --exclude '*_machine.rs' --exclude '*_table.rs' src\n```\n\nWhich gives us around 13 KLOC, which is still a lot.\n\n## Future work\n\nSince the port is finished, there is not much to do other than syncing it with\na new harfbuzz releases.\nBut there are still a lot of room for performance optimizations and refactoring.\n\nAlso, despite the fact that harfbuzz has a vast test suite, there are still a lot of\nthings left to test.\n\n## Safety\n\nThe library is completely safe.\n\nWe do have one `unsafe` to cast between two POD structures, which is perfectly safe.\nBut except that, there are no `unsafe` in this library and in most of its dependencies\n(excluding `bytemuck`).\n\n## Alternatives\n\n- [harfbuzz_rs](https://crates.io/crates/harfbuzz_rs) - bindings to the actual harfbuzz library.\n As of v2 doesn't expose subsetting and glyph outlining, which harfbuzz supports.\n- [allsorts](https://github.com/yeslogic/allsorts) - shaper and subsetter.\n As of v0.6 doesn't support variable fonts and\n [Apple Advanced Typography](https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6AATIntro.html).\n Relies on some unsafe code.\n- [swash](https://github.com/dfrg/swash) - Supports variable fonts, text layout and rendering.\n No subsetting. Relies on some unsafe code. As of v0.1.4 has zero tests.\n\n## License\n\n`rustybuzz` is licensed under the **MIT**.\n\n`harfbuzz` is [licensed](https://github.com/harfbuzz/harfbuzz/blob/master/COPYING) under the **Old MIT**\n" }, { "package": "semver@1.0.28", "sourcePath": "README.md", "sha256": "f1f736262db1f11353bca2fbdde906bfbb1d54e5eb7497dd110984f5eebb801e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "semver\n======\n\n[\"github\"](https://github.com/dtolnay/semver)\n[\"crates.io\"](https://crates.io/crates/semver)\n[\"docs.rs\"](https://docs.rs/semver)\n[\"build](https://github.com/dtolnay/semver/actions?query=branch%3Amaster)\n\nA parser and evaluator for Cargo's flavor of Semantic Versioning.\n\nSemantic Versioning (see ) is a guideline for how version\nnumbers are assigned and incremented. It is widely followed within the\nCargo/crates.io ecosystem for Rust.\n\n```toml\n[dependencies]\nsemver = \"1.0\"\n```\n\n
\n\n## Example\n\n```rust\nuse semver::{BuildMetadata, Prerelease, Version, VersionReq};\n\nfn main() {\n let req = VersionReq::parse(\">=1.2.3, <1.8.0\").unwrap();\n\n // Check whether this requirement matches version 1.2.3-alpha.1 (no)\n let version = Version {\n major: 1,\n minor: 2,\n patch: 3,\n pre: Prerelease::new(\"alpha.1\").unwrap(),\n build: BuildMetadata::EMPTY,\n };\n assert!(!req.matches(&version));\n\n // Check whether it matches 1.3.0 (yes it does)\n let version = Version::parse(\"1.3.0\").unwrap();\n assert!(req.matches(&version));\n}\n```\n\n
\n\n## Scope of this crate\n\nBesides Cargo, several other package ecosystems and package managers for other\nlanguages also use SemVer: RubyGems/Bundler for Ruby, npm for JavaScript,\nComposer for PHP, CocoaPods for Objective-C...\n\nThe `semver` crate is specifically intended to implement Cargo's interpretation\nof Semantic Versioning.\n\nWhere the various tools differ in their interpretation or implementation of the\nspec, this crate follows the implementation choices made by Cargo. If you are\noperating on version numbers from some other package ecosystem, you will want to\nuse a different semver library which is appropriate to that ecosystem.\n\nThe extent of Cargo's SemVer support is documented in the *[Specifying\nDependencies]* chapter of the Cargo reference.\n\n[Specifying Dependencies]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this crate by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "serde@1.0.229", "sourcePath": "README.md", "sha256": "0a33359b87747c0ae2437374c1dd4fc4d151ae3122b3931e2757b45d0ed519b8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# Serde   [![Build Status]][actions] [![Latest Version]][crates.io] [![serde msrv]][Rust 1.56] [![serde_derive msrv]][Rust 1.71]\n\n[Build Status]: https://img.shields.io/github/actions/workflow/status/serde-rs/serde/ci.yml?branch=master\n[actions]: https://github.com/serde-rs/serde/actions?query=branch%3Amaster\n[Latest Version]: https://img.shields.io/crates/v/serde.svg\n[crates.io]: https://crates.io/crates/serde\n[serde msrv]: https://img.shields.io/crates/msrv/serde.svg?label=serde%20msrv&color=lightgray\n[serde_derive msrv]: https://img.shields.io/crates/msrv/serde_derive.svg?label=serde_derive%20msrv&color=lightgray\n[Rust 1.56]: https://blog.rust-lang.org/2021/10/21/Rust-1.56.0/\n[Rust 1.71]: https://blog.rust-lang.org/2023/07/13/Rust-1.71.0/\n\n**Serde is a framework for *ser*ializing and *de*serializing Rust data structures efficiently and generically.**\n\n---\n\nYou may be looking for:\n\n- [An overview of Serde](https://serde.rs)\n- [Data formats supported by Serde](https://serde.rs/#data-formats)\n- [Setting up `#[derive(Serialize, Deserialize)]`](https://serde.rs/derive.html)\n- [Examples](https://serde.rs/examples.html)\n- [API documentation](https://docs.rs/serde)\n- [Release notes](https://github.com/serde-rs/serde/releases)\n\n## Serde in action\n\n
\n\nClick to show Cargo.toml.\nRun this code in the playground.\n\n\n```toml\n[dependencies]\n\n# The core APIs, including the Serialize and Deserialize traits. Always\n# required when using Serde. The \"derive\" feature is only required when\n# using #[derive(Serialize, Deserialize)] to make Serde work with structs\n# and enums defined in your crate.\nserde = { version = \"1.0\", features = [\"derive\"] }\n\n# Each data format lives in its own crate; the sample code below uses JSON\n# but you may be using a different one.\nserde_json = \"1.0\"\n```\n\n
\n

\n\n```rust\nuse serde::{Deserialize, Serialize};\n\n#[derive(Serialize, Deserialize, Debug)]\nstruct Point {\n x: i32,\n y: i32,\n}\n\nfn main() {\n let point = Point { x: 1, y: 2 };\n\n // Convert the Point to a JSON string.\n let serialized = serde_json::to_string(&point).unwrap();\n\n // Prints serialized = {\"x\":1,\"y\":2}\n println!(\"serialized = {}\", serialized);\n\n // Convert the JSON string back to a Point.\n let deserialized: Point = serde_json::from_str(&serialized).unwrap();\n\n // Prints deserialized = Point { x: 1, y: 2 }\n println!(\"deserialized = {:?}\", deserialized);\n}\n```\n\n## Getting help\n\nSerde is one of the most widely used Rust libraries so any place that Rustaceans\ncongregate will be able to help you out. For chat, consider trying the\n[#rust-questions] or [#rust-beginners] channels of the unofficial community\nDiscord (invite: ), the [#rust-usage] or\n[#beginners] channels of the official Rust Project Discord (invite:\n), or the [#general][zulip] stream in Zulip. For\nasynchronous, consider the [\\[rust\\] tag on StackOverflow][stackoverflow], the\n[/r/rust] subreddit which has a pinned weekly easy questions post, or the Rust\n[Discourse forum][discourse]. It's acceptable to file a support issue in this\nrepo but they tend not to get as many eyes as any of the above and may get\nclosed without a response after some time.\n\n[#rust-questions]: https://discord.com/channels/273534239310479360/274215136414400513\n[#rust-beginners]: https://discord.com/channels/273534239310479360/273541522815713281\n[#rust-usage]: https://discord.com/channels/442252698964721669/443150878111694848\n[#beginners]: https://discord.com/channels/442252698964721669/448238009733742612\n[zulip]: https://rust-lang.zulipchat.com/#narrow/stream/122651-general\n[stackoverflow]: https://stackoverflow.com/questions/tagged/rust\n[/r/rust]: https://www.reddit.com/r/rust\n[discourse]: https://users.rust-lang.org\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in Serde by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "serde_derive@1.0.229", "sourcePath": "README.md", "sha256": "0a33359b87747c0ae2437374c1dd4fc4d151ae3122b3931e2757b45d0ed519b8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# Serde   [![Build Status]][actions] [![Latest Version]][crates.io] [![serde msrv]][Rust 1.56] [![serde_derive msrv]][Rust 1.71]\n\n[Build Status]: https://img.shields.io/github/actions/workflow/status/serde-rs/serde/ci.yml?branch=master\n[actions]: https://github.com/serde-rs/serde/actions?query=branch%3Amaster\n[Latest Version]: https://img.shields.io/crates/v/serde.svg\n[crates.io]: https://crates.io/crates/serde\n[serde msrv]: https://img.shields.io/crates/msrv/serde.svg?label=serde%20msrv&color=lightgray\n[serde_derive msrv]: https://img.shields.io/crates/msrv/serde_derive.svg?label=serde_derive%20msrv&color=lightgray\n[Rust 1.56]: https://blog.rust-lang.org/2021/10/21/Rust-1.56.0/\n[Rust 1.71]: https://blog.rust-lang.org/2023/07/13/Rust-1.71.0/\n\n**Serde is a framework for *ser*ializing and *de*serializing Rust data structures efficiently and generically.**\n\n---\n\nYou may be looking for:\n\n- [An overview of Serde](https://serde.rs)\n- [Data formats supported by Serde](https://serde.rs/#data-formats)\n- [Setting up `#[derive(Serialize, Deserialize)]`](https://serde.rs/derive.html)\n- [Examples](https://serde.rs/examples.html)\n- [API documentation](https://docs.rs/serde)\n- [Release notes](https://github.com/serde-rs/serde/releases)\n\n## Serde in action\n\n
\n\nClick to show Cargo.toml.\nRun this code in the playground.\n\n\n```toml\n[dependencies]\n\n# The core APIs, including the Serialize and Deserialize traits. Always\n# required when using Serde. The \"derive\" feature is only required when\n# using #[derive(Serialize, Deserialize)] to make Serde work with structs\n# and enums defined in your crate.\nserde = { version = \"1.0\", features = [\"derive\"] }\n\n# Each data format lives in its own crate; the sample code below uses JSON\n# but you may be using a different one.\nserde_json = \"1.0\"\n```\n\n
\n

\n\n```rust\nuse serde::{Deserialize, Serialize};\n\n#[derive(Serialize, Deserialize, Debug)]\nstruct Point {\n x: i32,\n y: i32,\n}\n\nfn main() {\n let point = Point { x: 1, y: 2 };\n\n // Convert the Point to a JSON string.\n let serialized = serde_json::to_string(&point).unwrap();\n\n // Prints serialized = {\"x\":1,\"y\":2}\n println!(\"serialized = {}\", serialized);\n\n // Convert the JSON string back to a Point.\n let deserialized: Point = serde_json::from_str(&serialized).unwrap();\n\n // Prints deserialized = Point { x: 1, y: 2 }\n println!(\"deserialized = {:?}\", deserialized);\n}\n```\n\n## Getting help\n\nSerde is one of the most widely used Rust libraries so any place that Rustaceans\ncongregate will be able to help you out. For chat, consider trying the\n[#rust-questions] or [#rust-beginners] channels of the unofficial community\nDiscord (invite: ), the [#rust-usage] or\n[#beginners] channels of the official Rust Project Discord (invite:\n), or the [#general][zulip] stream in Zulip. For\nasynchronous, consider the [\\[rust\\] tag on StackOverflow][stackoverflow], the\n[/r/rust] subreddit which has a pinned weekly easy questions post, or the Rust\n[Discourse forum][discourse]. It's acceptable to file a support issue in this\nrepo but they tend not to get as many eyes as any of the above and may get\nclosed without a response after some time.\n\n[#rust-questions]: https://discord.com/channels/273534239310479360/274215136414400513\n[#rust-beginners]: https://discord.com/channels/273534239310479360/273541522815713281\n[#rust-usage]: https://discord.com/channels/442252698964721669/443150878111694848\n[#beginners]: https://discord.com/channels/442252698964721669/448238009733742612\n[zulip]: https://rust-lang.zulipchat.com/#narrow/stream/122651-general\n[stackoverflow]: https://stackoverflow.com/questions/tagged/rust\n[/r/rust]: https://www.reddit.com/r/rust\n[discourse]: https://users.rust-lang.org\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in Serde by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "serde_json@1.0.151", "sourcePath": "README.md", "sha256": "10cb8c14d9d29f29a031329747b352fc149a689b449ee5a8abb43712695aed72", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# Serde JSON   [![Build Status]][actions] [![Latest Version]][crates.io]\n\n[Build Status]: https://img.shields.io/github/actions/workflow/status/serde-rs/json/ci.yml?branch=master\n[actions]: https://github.com/serde-rs/json/actions?query=branch%3Amaster\n[Latest Version]: https://img.shields.io/crates/v/serde_json.svg\n[crates.io]: https://crates.io/crates/serde\\_json\n\n**Serde is a framework for *ser*ializing and *de*serializing Rust data structures efficiently and generically.**\n\n---\n\n```toml\n[dependencies]\nserde_json = \"1.0\"\n```\n\nYou may be looking for:\n\n- [JSON API documentation](https://docs.rs/serde_json)\n- [Serde API documentation](https://docs.rs/serde)\n- [Detailed documentation about Serde](https://serde.rs/)\n- [Setting up `#[derive(Serialize, Deserialize)]`](https://serde.rs/derive.html)\n- [Release notes](https://github.com/serde-rs/json/releases)\n\nJSON is a ubiquitous open-standard format that uses human-readable text to\ntransmit data objects consisting of key-value pairs.\n\n```json\n{\n \"name\": \"John Doe\",\n \"age\": 43,\n \"address\": {\n \"street\": \"10 Downing Street\",\n \"city\": \"London\"\n },\n \"phones\": [\n \"+44 1234567\",\n \"+44 2345678\"\n ]\n}\n```\n\nThere are three common ways that you might find yourself needing to work with\nJSON data in Rust.\n\n - **As text data.** An unprocessed string of JSON data that you receive on an\n HTTP endpoint, read from a file, or prepare to send to a remote server.\n - **As an untyped or loosely typed representation.** Maybe you want to check\n that some JSON data is valid before passing it on, but without knowing the\n structure of what it contains. Or you want to do very basic manipulations\n like insert a key in a particular spot.\n - **As a strongly typed Rust data structure.** When you expect all or most of\n your data to conform to a particular structure and want to get real work done\n without JSON's loosey-goosey nature tripping you up.\n\nSerde JSON provides efficient, flexible, safe ways of converting data between\neach of these representations.\n\n## Operating on untyped JSON values\n\nAny valid JSON data can be manipulated in the following recursive enum\nrepresentation. This data structure is [`serde_json::Value`][value].\n\n```rust\nenum Value {\n Null,\n Bool(bool),\n Number(Number),\n String(String),\n Array(Vec),\n Object(Map),\n}\n```\n\nA string of JSON data can be parsed into a `serde_json::Value` by the\n[`serde_json::from_str`][from_str] function. There is also\n[`from_slice`][from_slice] for parsing from a byte slice `&[u8]` and\n[`from_reader`][from_reader] for parsing from any `io::Read` like a File or a\nTCP stream.\n\n
\n\n\n\n
\n\n```rust\nuse serde_json::{Result, Value};\n\nfn untyped_example() -> Result<()> {\n // Some JSON input data as a &str. Maybe this comes from the user.\n let data = r#\"\n {\n \"name\": \"John Doe\",\n \"age\": 43,\n \"phones\": [\n \"+44 1234567\",\n \"+44 2345678\"\n ]\n }\"#;\n\n // Parse the string of data into serde_json::Value.\n let v: Value = serde_json::from_str(data)?;\n\n // Access parts of the data by indexing with square brackets.\n println!(\"Please call {} at the number {}\", v[\"name\"], v[\"phones\"][0]);\n\n Ok(())\n}\n```\n\nThe result of square bracket indexing like `v[\"name\"]` is a borrow of the data\nat that index, so the type is `&Value`. A JSON map can be indexed with string\nkeys, while a JSON array can be indexed with integer keys. If the type of the\ndata is not right for the type with which it is being indexed, or if a map does\nnot contain the key being indexed, or if the index into a vector is out of\nbounds, the returned element is `Value::Null`.\n\nWhen a `Value` is printed, it is printed as a JSON string. So in the code above,\nthe output looks like `Please call \"John Doe\" at the number \"+44 1234567\"`. The\nquotation marks appear because `v[\"name\"]` is a `&Value` containing a JSON\nstring and its JSON representation is `\"John Doe\"`. Printing as a plain string\nwithout quotation marks involves converting from a JSON string to a Rust string\nwith [`as_str()`] or avoiding the use of `Value` as described in the following\nsection.\n\n[`as_str()`]: https://docs.rs/serde_json/1/serde_json/enum.Value.html#method.as_str\n\nThe `Value` representation is sufficient for very basic tasks but can be tedious\nto work with for anything more significant. Error handling is verbose to\nimplement correctly, for example imagine trying to detect the presence of\nunrecognized fields in the input data. The compiler is powerless to help you\nwhen you make a mistake, for example imagine typoing `v[\"name\"]` as `v[\"nmae\"]`\nin one of the dozens of places it is used in your code.\n\n## Parsing JSON as strongly typed data structures\n\nSerde provides a powerful way of mapping JSON data into Rust data structures\nlargely automatically.\n\n
\n\n\n\n
\n\n```rust\nuse serde::{Deserialize, Serialize};\nuse serde_json::Result;\n\n#[derive(Serialize, Deserialize)]\nstruct Person {\n name: String,\n age: u8,\n phones: Vec,\n}\n\nfn typed_example() -> Result<()> {\n // Some JSON input data as a &str. Maybe this comes from the user.\n let data = r#\"\n {\n \"name\": \"John Doe\",\n \"age\": 43,\n \"phones\": [\n \"+44 1234567\",\n \"+44 2345678\"\n ]\n }\"#;\n\n // Parse the string of data into a Person object. This is exactly the\n // same function as the one that produced serde_json::Value above, but\n // now we are asking it for a Person as output.\n let p: Person = serde_json::from_str(data)?;\n\n // Do things just like with any other Rust data structure.\n println!(\"Please call {} at the number {}\", p.name, p.phones[0]);\n\n Ok(())\n}\n```\n\nThis is the same `serde_json::from_str` function as before, but this time we\nassign the return value to a variable of type `Person` so Serde will\nautomatically interpret the input data as a `Person` and produce informative\nerror messages if the layout does not conform to what a `Person` is expected to\nlook like.\n\nAny type that implements Serde's `Deserialize` trait can be deserialized this\nway. This includes built-in Rust standard library types like `Vec` and\n`HashMap`, as well as any structs or enums annotated with\n`#[derive(Deserialize)]`.\n\nOnce we have `p` of type `Person`, our IDE and the Rust compiler can help us use\nit correctly like they do for any other Rust code. The IDE can autocomplete\nfield names to prevent typos, which was impossible in the `serde_json::Value`\nrepresentation. And the Rust compiler can check that when we write\n`p.phones[0]`, then `p.phones` is guaranteed to be a `Vec` so indexing\ninto it makes sense and produces a `String`.\n\nThe necessary setup for using Serde's derive macros is explained on the *[Using\nderive]* page of the Serde site.\n\n[Using derive]: https://serde.rs/derive.html\n\n## Constructing JSON values\n\nSerde JSON provides a [`json!` macro][macro] to build `serde_json::Value`\nobjects with very natural JSON syntax.\n\n
\n\n\n\n
\n\n```rust\nuse serde_json::json;\n\nfn main() {\n // The type of `john` is `serde_json::Value`\n let john = json!({\n \"name\": \"John Doe\",\n \"age\": 43,\n \"phones\": [\n \"+44 1234567\",\n \"+44 2345678\"\n ]\n });\n\n println!(\"first phone number: {}\", john[\"phones\"][0]);\n\n // Convert to a string of JSON and print it out\n println!(\"{}\", john.to_string());\n}\n```\n\nThe `Value::to_string()` function converts a `serde_json::Value` into a `String`\nof JSON text.\n\nOne neat thing about the `json!` macro is that variables and expressions can be\ninterpolated directly into the JSON value as you are building it. Serde will\ncheck at compile time that the value you are interpolating is able to be\nrepresented as JSON.\n\n
\n\n\n\n
\n\n```rust\nlet full_name = \"John Doe\";\nlet age_last_year = 42;\n\n// The type of `john` is `serde_json::Value`\nlet john = json!({\n \"name\": full_name,\n \"age\": age_last_year + 1,\n \"phones\": [\n format!(\"+44 {}\", random_phone())\n ]\n});\n```\n\nThis is amazingly convenient, but we have the problem we had before with\n`Value`: the IDE and Rust compiler cannot help us if we get it wrong. Serde JSON\nprovides a better way of serializing strongly-typed data structures into JSON\ntext.\n\n## Creating JSON by serializing data structures\n\nA data structure can be converted to a JSON string by\n[`serde_json::to_string`][to_string]. There is also\n[`serde_json::to_vec`][to_vec] which serializes to a `Vec` and\n[`serde_json::to_writer`][to_writer] which serializes to any `io::Write`\nsuch as a File or a TCP stream.\n\n
\n\n\n\n
\n\n```rust\nuse serde::{Deserialize, Serialize};\nuse serde_json::Result;\n\n#[derive(Serialize, Deserialize)]\nstruct Address {\n street: String,\n city: String,\n}\n\nfn print_an_address() -> Result<()> {\n // Some data structure.\n let address = Address {\n street: \"10 Downing Street\".to_owned(),\n city: \"London\".to_owned(),\n };\n\n // Serialize it to a JSON string.\n let j = serde_json::to_string(&address)?;\n\n // Print, write to a file, or send to an HTTP server.\n println!(\"{}\", j);\n\n Ok(())\n}\n```\n\nAny type that implements Serde's `Serialize` trait can be serialized this way.\nThis includes built-in Rust standard library types like `Vec` and `HashMap`, as well as any structs or enums annotated with `#[derive(Serialize)]`.\n\n## Performance\n\nIt is fast. You should expect in the ballpark of 500 to 1000 megabytes per\nsecond deserialization and 600 to 900 megabytes per second serialization,\ndepending on the characteristics of your data. This is competitive with the\nfastest C and C++ JSON libraries or even 30% faster for many use cases.\nBenchmarks live in the [serde-rs/json-benchmark] repo.\n\n[serde-rs/json-benchmark]: https://github.com/serde-rs/json-benchmark\n\n## Getting help\n\nSerde is one of the most widely used Rust libraries, so any place that\nRustaceans congregate will be able to help you out. For chat, consider trying\nthe [#rust-questions] or [#rust-beginners] channels of the unofficial community\nDiscord (invite: ), the [#rust-usage] or\n[#beginners] channels of the official Rust Project Discord (invite:\n), or the [#general][zulip] stream in Zulip. For\nasynchronous, consider the [\\[rust\\] tag on StackOverflow][stackoverflow], the\n[/r/rust] subreddit which has a pinned weekly easy questions post, or the Rust\n[Discourse forum][discourse]. It's acceptable to file a support issue in this\nrepo, but they tend not to get as many eyes as any of the above and may get\nclosed without a response after some time.\n\n[#rust-questions]: https://discord.com/channels/273534239310479360/274215136414400513\n[#rust-beginners]: https://discord.com/channels/273534239310479360/273541522815713281\n[#rust-usage]: https://discord.com/channels/442252698964721669/443150878111694848\n[#beginners]: https://discord.com/channels/442252698964721669/448238009733742612\n[zulip]: https://rust-lang.zulipchat.com/#narrow/stream/122651-general\n[stackoverflow]: https://stackoverflow.com/questions/tagged/rust\n[/r/rust]: https://www.reddit.com/r/rust\n[discourse]: https://users.rust-lang.org\n\n## No-std support\n\nAs long as there is a memory allocator, it is possible to use serde_json without\nthe rest of the Rust standard library. Disable the default \"std\" feature and\nenable the \"alloc\" feature:\n\n```toml\n[dependencies]\nserde_json = { version = \"1.0\", default-features = false, features = [\"alloc\"] }\n```\n\nFor JSON support in Serde without a memory allocator, please see the\n[`serde-json-core`] crate.\n\n[`serde-json-core`]: https://github.com/rust-embedded-community/serde-json-core\n\n[value]: https://docs.rs/serde_json/1/serde_json/value/enum.Value.html\n[from_str]: https://docs.rs/serde_json/1/serde_json/de/fn.from_str.html\n[from_slice]: https://docs.rs/serde_json/1/serde_json/de/fn.from_slice.html\n[from_reader]: https://docs.rs/serde_json/1/serde_json/de/fn.from_reader.html\n[to_string]: https://docs.rs/serde_json/1/serde_json/ser/fn.to_string.html\n[to_vec]: https://docs.rs/serde_json/1/serde_json/ser/fn.to_vec.html\n[to_writer]: https://docs.rs/serde_json/1/serde_json/ser/fn.to_writer.html\n[macro]: https://docs.rs/serde_json/1/serde_json/macro.json.html\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this crate by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "serde_json@1.0.151", "sourcePath": "src/lexical/mod.rs", "sha256": "ace2ccea2cd541f019a84b9eeb3b9ecd88daf6bccbe9edf0eac00b62704f47c4", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// The code in this module is derived from the `lexical` crate by @Alexhuszagh\n// which the author condensed into this minimal subset for use in serde_json.\n// For the serde_json use case we care more about reliably round tripping all\n// possible floating point values than about parsing any arbitrarily long string\n// of digits with perfect accuracy, as the latter would take a high cost in\n// compile time and performance.\n//\n// Dual licensed as MIT and Apache 2.0 just like the rest of serde_json, but\n// copyright Alexander Huszagh.\n\n//! Fast, minimal float-parsing algorithm.\n\n// MODULES\npub(crate) mod algorithm;\nmod bhcomp;\nmod bignum;\nmod cached;\nmod cached_float80;\nmod digit;\nmod errors;\npub(crate) mod exponent;\npub(crate) mod float;\nmod large_powers;\npub(crate) mod math;\npub(crate) mod num;\npub(crate) mod parse;\npub(crate) mod rounding;\nmod shift;\nmod small_powers;\n\n#[cfg(fast_arithmetic = \"32\")]\nmod large_powers32;\n\n#[cfg(fast_arithmetic = \"64\")]\nmod large_powers64;\n\n// API\npub use self::parse::{parse_concise_float, parse_truncated_float};\n" }, { "package": "shlex@2.0.1", "sourcePath": "README.md", "sha256": "082e505bba5dffc5904af5602b45d01129173e617db62c81e6c11d71c964ea71", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "[![ci badge]][ci link] [![crates.io badge]][crates.io link] [![docs.rs badge]][docs.rs link]\n\n[crates.io badge]: https://img.shields.io/crates/v/shlex.svg?style=flat-square\n[crates.io link]: https://crates.io/crates/shlex\n[docs.rs badge]: https://img.shields.io/badge/docs-online-dddddd.svg?style=flat-square\n[docs.rs link]: https://docs.rs/shlex\n[ci badge]: https://img.shields.io/github/actions/workflow/status/comex/rust-shlex/test.yml?branch=master&style=flat-square\n[ci link]: https://github.com/comex/rust-shlex/actions\n\nSame idea as (but implementation not directly based on) the Python shlex\nmodule. However, this implementation does not support any of the Python\nmodule's customization because it makes parsing slower and is fairly useless.\nYou only get the default settings of shlex.split, which mimic the POSIX shell:\n\n\nThis implementation also deviates from the Python version in not treating \\r\nspecially, which I believe is more compliant.\n\nThis crate can be used on either normal Rust strings, or on byte strings with\nthe `bytes` module. The algorithms used are oblivious to UTF-8 high bytes, so\ninternally they all work on bytes directly as a micro-optimization.\n\nDisabling the `std` feature (which is enabled by default) will allow the crate\nto work in `no_std` environments, where the `alloc` crate, and a global\nallocator, are available.\n\n# LICENSE\n\nThe source code in this repository is Licensed under either of\n- Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or\n https://www.apache.org/licenses/LICENSE-2.0)\n- MIT license ([LICENSE-MIT](LICENSE-MIT) or\n https://opensource.org/licenses/MIT)\n\nat your option.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n" }, { "package": "shlex@2.0.1", "sourcePath": "src/bytes.rs", "sha256": "14ac6d79be6cd93543de6cbba8a4c07181aacf7d02d01a1e5f51b63a89bf11c0", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 Nicholas Allegra (comex).\n// Licensed under the Apache License, Version 2.0 or\n// the MIT license , at your option. This file may not be\n// copied, modified, or distributed except according to those terms.\n\n//! [`Shlex`] and friends for byte strings.\n//!\n//! This is used internally by the [outer module](crate), and may be more\n//! convenient if you are working with byte slices (`[u8]`) or types that are\n//! wrappers around bytes, such as [`OsStr`](std::ffi::OsStr):\n//!\n//! ```rust\n//! #[cfg(unix)] {\n//! use shlex::bytes::try_quote;\n//! use std::ffi::OsStr;\n//! use std::os::unix::ffi::OsStrExt;\n//!\n//! // `\\x80` is invalid in UTF-8.\n//! let os_str = OsStr::from_bytes(b\"a\\x80b c\");\n//! assert_eq!(try_quote(os_str.as_bytes()).unwrap(), &b\"'a\\x80b c'\"[..]);\n//! }\n//! ```\n//!\n//! (On Windows, `OsStr` uses 16 bit wide characters so this will not work.)\n\nextern crate alloc;\nuse alloc::vec::Vec;\nuse alloc::borrow::Cow;\n#[cfg(test)]\nuse alloc::borrow::ToOwned;\n#[cfg(all(doc, not(doctest)))]\nuse crate::{self as shlex, quoting_warning};\n\nuse super::QuoteError;\n\n/// An iterator that takes an input byte string and splits it into the words using the same syntax as\n/// the POSIX shell.\npub struct Shlex<'a> {\n in_iter: core::slice::Iter<'a, u8>,\n /// The number of newlines read so far, plus one.\n pub line_no: usize,\n /// An input string is erroneous if it ends while inside a quotation or right after an\n /// unescaped backslash. Since Iterator does not have a mechanism to return an error, if that\n /// happens, Shlex just throws out the last token, ends the iteration, and sets 'had_error' to\n /// true; best to check it after you're done iterating.\n pub had_error: bool,\n}\n\nimpl<'a> Shlex<'a> {\n pub fn new(in_bytes: &'a [u8]) -> Self {\n Shlex {\n in_iter: in_bytes.iter(),\n line_no: 1,\n had_error: false,\n }\n }\n\n fn parse_word(&mut self, mut ch: u8) -> Option> {\n let mut result: Vec = Vec::new();\n loop {\n match ch as char {\n '\"' => if let Err(()) = self.parse_double(&mut result) {\n self.had_error = true;\n return None;\n },\n '\\'' => if let Err(()) = self.parse_single(&mut result) {\n self.had_error = true;\n return None;\n },\n '\\\\' => if let Some(ch2) = self.next_char() {\n if ch2 != b'\\n' { result.push(ch2); }\n } else {\n self.had_error = true;\n return None;\n },\n ' ' | '\\t' | '\\n' => { break; },\n _ => { result.push(ch); },\n }\n if let Some(ch2) = self.next_char() { ch = ch2; } else { break; }\n }\n Some(result)\n }\n\n fn parse_double(&mut self, result: &mut Vec) -> Result<(), ()> {\n loop {\n if let Some(ch2) = self.next_char() {\n match ch2 as char {\n '\\\\' => {\n if let Some(ch3) = self.next_char() {\n match ch3 as char {\n // \\$ => $\n '$' | '`' | '\"' | '\\\\' => { result.push(ch3); },\n // \\ => nothing\n '\\n' => {},\n // \\x => =x\n _ => { result.push(b'\\\\'); result.push(ch3); }\n }\n } else {\n return Err(());\n }\n },\n '\"' => { return Ok(()); },\n _ => { result.push(ch2); },\n }\n } else {\n return Err(());\n }\n }\n }\n\n fn parse_single(&mut self, result: &mut Vec) -> Result<(), ()> {\n loop {\n if let Some(ch2) = self.next_char() {\n match ch2 as char {\n '\\'' => { return Ok(()); },\n _ => { result.push(ch2); },\n }\n } else {\n return Err(());\n }\n }\n }\n\n fn next_char(&mut self) -> Option {\n let res = self.in_iter.next().copied();\n if res == Some(b'\\n') { self.line_no += 1; }\n res\n }\n}\n\nimpl Iterator for Shlex<'_> {\n type Item = Vec;\n fn next(&mut self) -> Option {\n if let Some(mut ch) = self.next_char() {\n // skip initial whitespace\n loop {\n match ch as char {\n ' ' | '\\t' | '\\n' => {},\n '#' => {\n while let Some(ch2) = self.next_char() {\n if ch2 as char == '\\n' { break; }\n }\n },\n _ => { break; }\n }\n if let Some(ch2) = self.next_char() { ch = ch2; } else { return None; }\n }\n self.parse_word(ch)\n } else { // no initial character\n None\n }\n }\n\n}\n\n/// Convenience function that consumes the whole byte string at once. Returns None if the input was\n/// erroneous.\npub fn split(in_bytes: &[u8]) -> Option>> {\n let mut shl = Shlex::new(in_bytes);\n let res = shl.by_ref().collect();\n if shl.had_error { None } else { Some(res) }\n}\n\n/// A more configurable interface to quote strings. If you only want the default settings you can\n/// use the convenience functions [`try_quote`] and [`try_join`].\n///\n/// The string equivalent is [`shlex::Quoter`].\n#[derive(Default, Debug, Clone)]\npub struct Quoter {\n allow_nul: bool,\n // TODO: more options\n}\n\nimpl Quoter {\n /// Create a new [`Quoter`] with default settings.\n #[inline]\n pub fn new() -> Self {\n Self::default()\n }\n\n /// Set whether to allow [nul bytes](quoting_warning#nul-bytes). By default they are not\n /// allowed and will result in an error of [`QuoteError::Nul`].\n #[inline]\n pub fn allow_nul(mut self, allow: bool) -> Self {\n self.allow_nul = allow;\n self\n }\n\n /// Convenience function that consumes an iterable of words and turns it into a single byte string,\n /// quoting words when necessary. Consecutive words will be separated by a single space.\n pub fn join<'a, I: IntoIterator>(&self, words: I) -> Result, QuoteError> {\n Ok(words.into_iter()\n .map(|word| self.quote(word))\n .collect::>, QuoteError>>()?\n .join(&b' '))\n }\n\n /// Given a single word, return a byte string suitable to encode it as a shell argument.\n ///\n /// If given valid UTF-8, this will never produce invalid UTF-8. This is because it only\n /// ever inserts valid ASCII characters before or after existing ASCII characters (or\n /// returns two single quotes if the input was an empty string). It will never modify a\n /// multibyte UTF-8 character.\n pub fn quote<'a>(&self, mut in_bytes: &'a [u8]) -> Result, QuoteError> {\n if in_bytes.is_empty() {\n // Empty string. Special case that isn't meaningful as only part of a word.\n return Ok(b\"''\"[..].into());\n }\n if !self.allow_nul && in_bytes.contains(&b'\\0') {\n return Err(QuoteError::Nul);\n }\n let mut out: Vec = Vec::new();\n while !in_bytes.is_empty() {\n // Pick a quoting strategy for some prefix of the input. Normally this will cover the\n // entire input, but in some case we might need to divide the input into multiple chunks\n // that are quoted differently.\n let (cur_len, strategy) = quoting_strategy(in_bytes);\n if cur_len == in_bytes.len() && strategy == QuotingStrategy::Unquoted && out.is_empty() {\n // Entire string can be represented unquoted. Reuse the allocation.\n return Ok(in_bytes.into());\n }\n let (cur_chunk, rest) = in_bytes.split_at(cur_len);\n assert!(rest.len() < in_bytes.len()); // no infinite loop\n in_bytes = rest;\n append_quoted_chunk(&mut out, cur_chunk, strategy);\n }\n Ok(out.into())\n }\n\n}\n\n#[derive(PartialEq)]\nenum QuotingStrategy {\n /// No quotes and no backslash escapes. (If backslash escapes would be necessary, we use a\n /// different strategy instead.)\n Unquoted,\n /// Single quoted.\n SingleQuoted,\n /// Double quotes, potentially with backslash escapes.\n DoubleQuoted,\n // TODO: add $'xxx' and \"$(printf 'xxx')\" styles\n}\n\n/// Is this ASCII byte okay to emit unquoted?\nconst fn unquoted_ok(c: u8) -> bool {\n match c as char {\n // Allowed characters:\n '+' | '-' | '.' | '/' | ':' | '@' | ']' | '_' |\n '0'..='9' | 'A'..='Z' | 'a'..='z'\n => true,\n\n // Non-allowed characters:\n // From POSIX https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html\n // \"The application shall quote the following characters if they are to represent themselves:\"\n '|' | '&' | ';' | '<' | '>' | '(' | ')' | '$' | '`' | '\\\\' | '\"' | '\\'' | ' ' | '\\t' | '\\n' |\n // \"and the following may need to be quoted under certain circumstances[..]:\"\n '*' | '?' | '[' | '#' | '~' | '=' | '%' |\n // Brace expansion. These ought to be in the POSIX list but aren't yet;\n // see: https://www.austingroupbugs.net/view.php?id=1193\n '{' | '}' |\n // Also quote comma, just to be safe in the extremely odd case that the user of this crate\n // is intentionally placing a quoted string inside a brace expansion, e.g.:\n // format!(\"echo foo{{a,b,{}}}\" | shlex::quote(some_str))\n ',' |\n // '\\r' is allowed in a word by all real shells I tested, but is treated as a word\n // separator by Python `shlex` | and might be translated to '\\n' in interactive mode.\n '\\r' |\n // '!' and '^' are treated specially in interactive mode; see quoting_warning.\n '!' | '^' |\n // Nul bytes and control characters.\n '\\x00' ..= '\\x1f' | '\\x7f'\n => false,\n '\\u{80}' ..= '\\u{10ffff}' => {\n // This is unreachable since `unquoted_ok` is only called for 0..128.\n // Non-ASCII bytes are handled separately in `quoting_strategy`.\n // Can't call unreachable!() from `const fn` on old Rust, so...\n unquoted_ok(c)\n },\n }\n // Note: The logic cited above for quoting comma might suggest that `..` should also be quoted,\n // it as a special case of brace expansion). But it's not necessary. There are three cases:\n //\n // 1. The user wants comma-based brace expansion, but the untrusted string being `quote`d\n // contains `..`, so they get something like `{foo,bar,3..5}`.\n // => That's safe; both Bash and Zsh expand this to `foo bar 3..5` rather than\n // `foo bar 3 4 5`. The presence of commas disables sequence expression expansion.\n //\n // 2. The user wants comma-based brace expansion where the contents of the braces are a\n // variable number of `quote`d strings and nothing else. There happens to be exactly\n // one string and it contains `..`, so they get something like `{3..5}`.\n // => Then this will expand as a sequence expression, which is unintended. But I don't mind,\n // because any such code is already buggy. Suppose the untrusted string *didn't* contain\n // `,` or `..`, resulting in shell input like `{foo}`. Then the shell would interpret it\n // as the literal string `{foo}` rather than brace-expanding it into `foo`.\n //\n // 3. The user wants a sequence expression and wants to supply an untrusted string as one of\n // the endpoints or the increment.\n // => Well, that's just silly, since the endpoints can only be numbers or single letters.\n}\n\n/// Optimized version of `unquoted_ok`.\nfn unquoted_ok_fast(c: u8) -> bool {\n const UNQUOTED_OK_MASK: u128 = {\n // Make a mask of all bytes in 0..<0x80 that pass.\n let mut c = 0u8;\n let mut mask = 0u128;\n while c < 0x80 {\n if unquoted_ok(c) {\n mask |= 1u128 << c;\n }\n c += 1;\n }\n mask\n };\n ((UNQUOTED_OK_MASK >> c) & 1) != 0\n}\n\n/// Is this ASCII byte okay to emit in single quotes?\nfn single_quoted_ok(c: u8) -> bool {\n match c {\n // No single quotes in single quotes.\n b'\\'' => false,\n // To work around a Bash bug, ^ is only allowed right after an opening single quote; see\n // quoting_warning.\n b'^' => false,\n // Backslashes in single quotes are literal according to POSIX, but Fish treats them as an\n // escape character. Ban them. Fish doesn't aim to be POSIX-compatible, but we *can*\n // achieve Fish compatibility using double quotes, so we might as well.\n b'\\\\' => false,\n _ => true\n }\n}\n\n/// Is this ASCII byte okay to emit in double quotes?\nfn double_quoted_ok(c: u8) -> bool {\n match c {\n // Work around Python `shlex` bug where parsing \"\\`\" and \"\\$\" doesn't strip the\n // backslash, even though POSIX requires it.\n b'`' | b'$' => false,\n // '!' and '^' are treated specially in interactive mode; see quoting_warning.\n b'!' | b'^' => false,\n _ => true\n }\n}\n\n/// Given an input, return a quoting strategy that can cover some prefix of the string, along with\n/// the size of that prefix.\n///\n/// Precondition: input size is nonzero. (Empty strings are handled by the caller.)\n/// Postcondition: returned size is nonzero.\n#[cfg_attr(manual_codegen_check, inline(never))]\nfn quoting_strategy(in_bytes: &[u8]) -> (usize, QuotingStrategy) {\n const UNQUOTED_OK: u8 = 1;\n const SINGLE_QUOTED_OK: u8 = 2;\n const DOUBLE_QUOTED_OK: u8 = 4;\n\n let mut prev_ok = SINGLE_QUOTED_OK | DOUBLE_QUOTED_OK | UNQUOTED_OK;\n let mut i = 0;\n\n if in_bytes[0] == b'^' {\n // To work around a Bash bug, ^ is only allowed right after an opening single quote; see\n // quoting_warning.\n prev_ok = SINGLE_QUOTED_OK;\n i = 1;\n }\n\n while i < in_bytes.len() {\n let c = in_bytes[i];\n let mut cur_ok = prev_ok;\n\n if c >= 0x80 {\n // Normally, non-ASCII characters shouldn't require quoting, but see quoting_warning.md\n // about \\xa0. For now, just treat all non-ASCII characters as requiring quotes. This\n // also ensures things are safe in the off-chance that you're in a legacy 8-bit locale that\n // has additional characters satisfying `isblank`.\n cur_ok &= !UNQUOTED_OK;\n } else {\n if !unquoted_ok_fast(c) {\n cur_ok &= !UNQUOTED_OK;\n }\n if !single_quoted_ok(c){\n cur_ok &= !SINGLE_QUOTED_OK;\n }\n if !double_quoted_ok(c) {\n cur_ok &= !DOUBLE_QUOTED_OK;\n }\n }\n\n if cur_ok == 0 {\n // There are no quoting strategies that would work for both the previous characters and\n // this one. So we have to end the chunk before this character. The caller will call\n // `quoting_strategy` again to handle the rest of the string.\n break;\n }\n\n prev_ok = cur_ok;\n i += 1;\n }\n\n // Pick the best allowed strategy.\n let strategy = if prev_ok & UNQUOTED_OK != 0 {\n QuotingStrategy::Unquoted\n } else if prev_ok & SINGLE_QUOTED_OK != 0 {\n QuotingStrategy::SingleQuoted\n } else if prev_ok & DOUBLE_QUOTED_OK != 0 {\n QuotingStrategy::DoubleQuoted\n } else {\n unreachable!()\n };\n debug_assert!(i > 0);\n (i, strategy)\n}\n\nfn append_quoted_chunk(out: &mut Vec, cur_chunk: &[u8], strategy: QuotingStrategy) {\n match strategy {\n QuotingStrategy::Unquoted => {\n out.extend_from_slice(cur_chunk);\n },\n QuotingStrategy::SingleQuoted => {\n out.reserve(cur_chunk.len() + 2);\n out.push(b'\\'');\n out.extend_from_slice(cur_chunk);\n out.push(b'\\'');\n },\n QuotingStrategy::DoubleQuoted => {\n out.reserve(cur_chunk.len() + 2);\n out.push(b'\"');\n for &c in cur_chunk.iter() {\n if let b'$' | b'`' | b'\"' | b'\\\\' = c {\n // Add a preceding backslash.\n // Note: We shouldn't actually get here for $ and ` because they don't pass\n // `double_quoted_ok`.\n out.push(b'\\\\');\n }\n // Add the character itself.\n out.push(c);\n }\n out.push(b'\"');\n },\n }\n}\n\n/// Convenience function that consumes an iterable of words and turns it into a single byte string,\n/// quoting words when necessary. Consecutive words will be separated by a single space.\n///\n/// Uses default settings. The only error that can be returned is [`QuoteError::Nul`].\n///\n/// Equivalent to [`Quoter::new().join(words)`](Quoter).\n///\n/// The string equivalent is [shlex::try_join].\npub fn try_join<'a, I: IntoIterator>(words: I) -> Result, QuoteError> {\n Quoter::new().join(words)\n}\n\n/// Given a single word, return a string suitable to encode it as a shell argument.\n///\n/// Uses default settings. The only error that can be returned is [`QuoteError::Nul`].\n///\n/// Equivalent to [`Quoter::new().quote(in_bytes)`](Quoter).\n///\n/// The string equivalent is [shlex::try_quote].\npub fn try_quote(in_bytes: &[u8]) -> Result, QuoteError> {\n Quoter::new().quote(in_bytes)\n}\n\n#[cfg(test)]\nconst INVALID_UTF8: &[u8] = b\"\\xa1\";\n\n#[test]\n#[allow(invalid_from_utf8)]\nfn test_invalid_utf8() {\n // Check that our test string is actually invalid UTF-8.\n assert!(core::str::from_utf8(INVALID_UTF8).is_err());\n}\n\n#[cfg(test)]\nstatic SPLIT_TEST_ITEMS: &[(&[u8], Option<&[&[u8]]>)] = &[\n (b\"foo$baz\", Some(&[b\"foo$baz\"])),\n (b\"foo baz\", Some(&[b\"foo\", b\"baz\"])),\n (b\"foo\\\"bar\\\"baz\", Some(&[b\"foobarbaz\"])),\n (b\"foo \\\"bar\\\"baz\", Some(&[b\"foo\", b\"barbaz\"])),\n (b\" foo \\nbar\", Some(&[b\"foo\", b\"bar\"])),\n (b\"foo\\\\\\nbar\", Some(&[b\"foobar\"])),\n (b\"\\\"foo\\\\\\nbar\\\"\", Some(&[b\"foobar\"])),\n (b\"'baz\\\\$b'\", Some(&[b\"baz\\\\$b\"])),\n (b\"'baz\\\\\\''\", None),\n (b\"\\\\\", None),\n (b\"\\\"\\\\\", None),\n (b\"'\\\\\", None),\n (b\"\\\"\", None),\n (b\"'\", None),\n (b\"foo #bar\\nbaz\", Some(&[b\"foo\", b\"baz\"])),\n (b\"foo #bar\", Some(&[b\"foo\"])),\n (b\"foo#bar\", Some(&[b\"foo#bar\"])),\n (b\"foo\\\"#bar\", None),\n (b\"'\\\\n'\", Some(&[b\"\\\\n\"])),\n (b\"'\\\\\\\\n'\", Some(&[b\"\\\\\\\\n\"])),\n (INVALID_UTF8, Some(&[INVALID_UTF8])),\n];\n\n#[test]\nfn test_split() {\n for &(input, output) in SPLIT_TEST_ITEMS {\n assert_eq!(split(input), output.map(|o| o.iter().map(|&x| x.to_owned()).collect()));\n }\n}\n\n#[test]\nfn test_lineno() {\n let mut sh = Shlex::new(b\"\\nfoo\\nbar\");\n while let Some(word) = sh.next() {\n if word == b\"bar\" {\n assert_eq!(sh.line_no, 3);\n }\n }\n}\n" }, { "package": "shlex@2.0.1", "sourcePath": "src/lib.rs", "sha256": "d707a605a9dbe52b48c53f5d0209ab457d1db535f10acad060883b6c3d8d7a91", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 Nicholas Allegra (comex).\n// Licensed under the Apache License, Version 2.0 or\n// the MIT license , at your option. This file may not be\n// copied, modified, or distributed except according to those terms.\n\n//! Parse strings like, and escape strings for, POSIX shells.\n//!\n//! Same idea as (but implementation not directly based on) the Python shlex module.\n//!\n//! Disabling the `std` feature (which is enabled by default) will allow the crate to work in\n//! `no_std` environments, where the `alloc` crate, and a global allocator, are available.\n//!\n//! ## Warning\n//!\n//! The [`try_quote`]/[`try_join`] family of APIs does not quote control characters (because they\n//! cannot be quoted portably).\n//!\n//! This is fully safe in noninteractive contexts, like shell scripts and `sh -c` arguments (or\n//! even scripts `source`d from interactive shells).\n//!\n//! But if you are quoting for human consumption, you should keep in mind that ugly inputs produce\n//! ugly outputs (which may not be copy-pastable).\n//!\n//! And if by chance you are piping the output of [`try_quote`]/[`try_join`] directly to the stdin\n//! of an interactive shell, you should stop, because control characters can lead to arbitrary\n//! command injection.\n//!\n//! For more information, and for information about more minor issues, please see [quoting_warning].\n//!\n//! ## Compatibility\n//!\n//! This crate's quoting functionality tries to be compatible with **any POSIX-compatible shell**;\n//! it's tested against `bash`, `zsh`, `dash`, Busybox `ash`, and `mksh`, plus `fish` (which is not\n//! POSIX-compatible but close enough).\n//!\n//! It also aims to be compatible with Python `shlex` and C `wordexp`.\n\n#![cfg_attr(not(feature = \"std\"), no_std)]\n\nextern crate alloc;\nuse alloc::vec::Vec;\nuse alloc::borrow::Cow;\nuse alloc::string::String;\n#[cfg(test)]\nuse alloc::vec;\n#[cfg(test)]\nuse alloc::borrow::ToOwned;\n\npub mod bytes;\n#[cfg(all(doc, not(doctest)))]\n#[path = \"quoting_warning.md\"]\npub mod quoting_warning;\n\n/// An iterator that takes an input string and splits it into the words using the same syntax as\n/// the POSIX shell.\n///\n/// See [`bytes::Shlex`].\npub struct Shlex<'a>(bytes::Shlex<'a>);\n\nimpl<'a> Shlex<'a> {\n pub fn new(in_str: &'a str) -> Self {\n Self(bytes::Shlex::new(in_str.as_bytes()))\n }\n\n /// # Safety\n ///\n /// The parameter must have been constructed from valid UTF-8.\n pub unsafe fn from_bytes(bytes: bytes::Shlex<'a>) -> Self {\n Self(bytes)\n }\n\n /// # Safety\n ///\n /// If the returned reference is reassigned, the new [`bytes::Shlex`] must have been constructed from valid UTF-8.\n pub unsafe fn as_bytes_mut(&mut self) -> &mut bytes::Shlex<'a> {\n &mut self.0\n }\n}\n\nimpl Iterator for Shlex<'_> {\n type Item = String;\n fn next(&mut self) -> Option {\n self.0.next().map(|byte_word| {\n // Safety: given valid UTF-8, bytes::Shlex will always return valid UTF-8.\n unsafe { String::from_utf8_unchecked(byte_word) }\n })\n }\n}\n\nimpl<'a> core::ops::Deref for Shlex<'a> {\n type Target = bytes::Shlex<'a>;\n\n fn deref(&self) -> &Self::Target {\n &self.0\n }\n}\n\n/// Convenience function that consumes the whole string at once. Returns None if the input was\n/// erroneous.\npub fn split(in_str: &str) -> Option> {\n let mut shl = Shlex::new(in_str);\n let res = shl.by_ref().collect();\n if shl.had_error { None } else { Some(res) }\n}\n\n/// Errors from [`Quoter::quote`], [`Quoter::join`], etc. (and their [`bytes`] counterparts).\n///\n/// By default, the only error that can be returned is [`QuoteError::Nul`]. If you call\n/// `allow_nul(true)`, then no errors can be returned at all. Any error variants added in the\n/// future will not be enabled by default; they will be enabled through corresponding non-default\n/// [`Quoter`] options.\n///\n/// ...In theory. In the unlikely event that additional classes of inputs are discovered that,\n/// like nul bytes, are fundamentally unsafe to quote even for non-interactive shells, the risk\n/// will be mitigated by adding corresponding [`QuoteError`] variants that *are* enabled by\n/// default.\n#[non_exhaustive]\n#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]\npub enum QuoteError {\n /// The input contained a nul byte. In most cases, shells fundamentally [cannot handle strings\n /// containing nul bytes](quoting_warning#nul-bytes), no matter how they are quoted. But if\n /// you're sure you can handle nul bytes, you can call `allow_nul(true)` on the `Quoter` to let\n /// them pass through.\n Nul,\n}\n\nimpl core::fmt::Display for QuoteError {\n fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n match self {\n QuoteError::Nul => f.write_str(\"cannot shell-quote string containing nul byte\"),\n }\n }\n}\n\n#[cfg(feature = \"std\")]\nimpl std::error::Error for QuoteError {}\n\n/// A more configurable interface to quote strings. If you only want the default settings you can\n/// use the convenience functions [`try_quote`] and [`try_join`].\n///\n/// The bytes equivalent is [`bytes::Quoter`].\n#[derive(Default, Debug, Clone)]\npub struct Quoter {\n inner: bytes::Quoter,\n}\n\nimpl Quoter {\n /// Create a new [`Quoter`] with default settings.\n #[inline]\n pub fn new() -> Self {\n Self::default()\n }\n\n /// Set whether to allow [nul bytes](quoting_warning#nul-bytes). By default they are not\n /// allowed and will result in an error of [`QuoteError::Nul`].\n #[inline]\n pub fn allow_nul(mut self, allow: bool) -> Self {\n self.inner = self.inner.allow_nul(allow);\n self\n }\n\n /// Convenience function that consumes an iterable of words and turns it into a single string,\n /// quoting words when necessary. Consecutive words will be separated by a single space.\n pub fn join<'a, I: IntoIterator>(&self, words: I) -> Result {\n // Safety: given valid UTF-8, bytes::join() will always return valid UTF-8.\n self.inner.join(words.into_iter().map(|s| s.as_bytes()))\n .map(|bytes| unsafe { String::from_utf8_unchecked(bytes) })\n }\n\n /// Given a single word, return a string suitable to encode it as a shell argument.\n pub fn quote<'a>(&self, in_str: &'a str) -> Result, QuoteError> {\n Ok(match self.inner.quote(in_str.as_bytes())? {\n Cow::Borrowed(out) => {\n // Safety: given valid UTF-8, bytes::quote() will always return valid UTF-8.\n unsafe { core::str::from_utf8_unchecked(out) }.into()\n }\n Cow::Owned(out) => {\n // Safety: given valid UTF-8, bytes::quote() will always return valid UTF-8.\n unsafe { String::from_utf8_unchecked(out) }.into()\n }\n })\n }\n}\n\nimpl From for Quoter {\n fn from(inner: bytes::Quoter) -> Quoter {\n Quoter { inner }\n }\n}\n\nimpl From for bytes::Quoter {\n fn from(quoter: Quoter) -> bytes::Quoter {\n quoter.inner\n }\n}\n\n/// Convenience function that consumes an iterable of words and turns it into a single string,\n/// quoting words when necessary. Consecutive words will be separated by a single space.\n///\n/// Uses default settings. The only error that can be returned is [`QuoteError::Nul`].\n///\n/// Equivalent to [`Quoter::new().join(words)`](Quoter).\n///\n/// The bytes equivalent is [bytes::try_join].\npub fn try_join<'a, I: IntoIterator>(words: I) -> Result {\n Quoter::new().join(words)\n}\n\n/// Given a single word, return a string suitable to encode it as a shell argument.\n///\n/// Uses default settings. The only error that can be returned is [`QuoteError::Nul`].\n///\n/// Equivalent to [`Quoter::new().quote(in_str)`](Quoter).\n///\n/// The bytes equivalent is [bytes::try_quote].\npub fn try_quote(in_str: &str) -> Result, QuoteError> {\n Quoter::new().quote(in_str)\n}\n\n#[cfg(test)]\nstatic SPLIT_TEST_ITEMS: &[(&str, Option<&[&str]>)] = &[\n (\"foo$baz\", Some(&[\"foo$baz\"])),\n (\"foo baz\", Some(&[\"foo\", \"baz\"])),\n (\"foo\\\"bar\\\"baz\", Some(&[\"foobarbaz\"])),\n (\"foo \\\"bar\\\"baz\", Some(&[\"foo\", \"barbaz\"])),\n (\" foo \\nbar\", Some(&[\"foo\", \"bar\"])),\n (\"foo\\\\\\nbar\", Some(&[\"foobar\"])),\n (\"\\\"foo\\\\\\nbar\\\"\", Some(&[\"foobar\"])),\n (\"'baz\\\\$b'\", Some(&[\"baz\\\\$b\"])),\n (\"'baz\\\\\\''\", None),\n (\"\\\\\", None),\n (\"\\\"\\\\\", None),\n (\"'\\\\\", None),\n (\"\\\"\", None),\n (\"'\", None),\n (\"foo #bar\\nbaz\", Some(&[\"foo\", \"baz\"])),\n (\"foo #bar\", Some(&[\"foo\"])),\n (\"foo#bar\", Some(&[\"foo#bar\"])),\n (\"foo\\\"#bar\", None),\n (\"'\\\\n'\", Some(&[\"\\\\n\"])),\n (\"'\\\\\\\\n'\", Some(&[\"\\\\\\\\n\"])),\n];\n\n#[test]\nfn test_split() {\n for &(input, output) in SPLIT_TEST_ITEMS {\n assert_eq!(split(input), output.map(|o| o.iter().map(|&x| x.to_owned()).collect()));\n }\n}\n\n#[test]\nfn test_lineno() {\n let mut sh = Shlex::new(\"\\nfoo\\nbar\");\n while let Some(word) = sh.next() {\n if word == \"bar\" {\n assert_eq!(sh.line_no, 3);\n }\n }\n}\n\n#[test]\n#[cfg_attr(not(feature = \"std\"), allow(unreachable_code, unused_mut))]\nfn test_quote() {\n // This is a list of (unquoted, quoted) pairs.\n // But it's using a single long (raw) string literal with an ad-hoc format, just because it's\n // hard to read if we have to put the test strings through Rust escaping on top of the escaping\n // being tested. (Even raw string literals are noisy for short strings).\n // Ad-hoc: \"NL\" is replaced with a literal newline; no other escape sequences.\n let tests = r#\"\n <> => <''>\n => \n => <'foo bar'>\n <\"foo bar'\"> => <\"\\\"foo bar'\\\"\">\n <'foo bar'> => <\"'foo bar'\">\n <\"> => <'\"'>\n <\"'> => <\"\\\"'\">\n => <'hello!world'>\n <'hello!world> => <\"'hello\"'!world'>\n <'hello!> => <\"'hello\"'!'>\n => <'hello ''^ world'>\n => \n => <'!world'\"'\">\n <{a, b}> => <'{a, b}'>\n => <'NL'>\n <^> => <'^'>\n => \n => <'NLx''^'>\n => <'NL''^x'>\n => <'NL ''^x'>\n <{a,b}> => <'{a,b}'>\n => <'a,b'>\n \n <'$> => <\"'\"'$'>\n <\"^> => <'\"''^'>\n \"#;\n let mut ok = true;\n for test in tests.trim().split('\\n') {\n let parts: Vec = test\n .replace(\"NL\", \"\\n\")\n .split(\"=>\")\n .map(|part| part.trim().trim_start_matches('<').trim_end_matches('>').to_owned())\n .collect();\n assert!(parts.len() == 2);\n let unquoted = &*parts[0];\n let quoted_expected = &*parts[1];\n let quoted_actual = try_quote(&parts[0]).unwrap();\n if quoted_expected != quoted_actual {\n #[cfg(not(feature = \"std\"))]\n panic!(\"FAIL: for input <{}>, expected <{}>, got <{}>\",\n unquoted, quoted_expected, quoted_actual);\n #[cfg(feature = \"std\")]\n println!(\"FAIL: for input <{}>, expected <{}>, got <{}>\",\n unquoted, quoted_expected, quoted_actual);\n ok = false;\n }\n }\n assert!(ok);\n}\n\n#[test]\nfn test_fallible() {\n assert_eq!(try_join(vec![\"\\0\"]), Err(QuoteError::Nul));\n assert_eq!(try_quote(\"\\0\"), Err(QuoteError::Nul));\n}\n" }, { "package": "simd-adler32@0.3.10", "sourcePath": "LICENSE.md", "sha256": "42a35170233e83e18856792e748de4c1ce4a63b2afce9a370c89ef3fe23f9f2d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "MIT License\n\nCopyright (c) [2021] [Marvin Countryman]\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "package": "simplecss@0.2.2", "sourcePath": "README.md", "sha256": "a6e895d2e8e499be9240a1fdcd9b9addf712dfe0026e320aa51529643088abd4", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "
\n\n# SimpleCSS\n\n**A simple [CSS 2.1](https://www.w3.org/TR/CSS21/) parser and selector.**\n\n[![Linebender Zulip, #resvg channel](https://img.shields.io/badge/Linebender-%23resvg-blue?logo=Zulip)](https://xi.zulipchat.com/#narrow/channel/465085-resvg)\n[![dependency status](https://deps.rs/repo/github/linebender/simplecss/status.svg)](https://deps.rs/repo/github/linebender/simplecss)\n[![Apache 2.0 or MIT license.](https://img.shields.io/badge/license-Apache--2.0_OR_MIT-blue.svg)](#license)\n[![Build status](https://github.com/linebender/simplecss/workflows/CI/badge.svg)](https://github.com/linebender/simplecss/actions)\n[![Crates.io](https://img.shields.io/crates/v/simplecss.svg)](https://crates.io/crates/simplecss)\n[![Docs](https://docs.rs/simplecss/badge.svg)](https://docs.rs/simplecss)\n![](https://img.shields.io/badge/unsafe-forbidden-brightgreen.svg)\n\n
\n\nThis is not a browser-grade CSS parser.\nIf you need one, use [cssparser](https://crates.io/crates/cssparser) + [selectors](https://crates.io/crates/selectors).\n\nSince it's very simple we will start with limitations:\n\n## Limitations\n\n- [At-rules](https://www.w3.org/TR/CSS21/syndata.html#at-rules) are not supported.\n They will be skipped during parsing.\n- Property values are not parsed.\n In CSS like `* { width: 5px }` you will get a `width` property with a `5px` value as a string.\n- CDO/CDC comments are not supported.\n- Parser is case sensitive.\n All keywords must be lowercase.\n- Unicode escape, like `\\26`, is not supported.\n\n## Features\n\n- Selector matching support.\n- The rules are sorted by specificity.\n- `!important` parsing support.\n- Has a high-level parsers and low-level, zero-allocation tokenizers.\n- No unsafe.\n\n## Minimum supported Rust Version (MSRV)\n\nThis version of SimpleCSS has been verified to compile with **Rust 1.65** and later.\n\nFuture versions of SimpleCSS might increase the Rust version requirement.\nIt will not be treated as a breaking change and as such can even happen with small patch releases.\n\n
\nClick here if compiling fails.\n\nAs time has passed, some of SimpleCSS's dependencies could have released versions with a higher Rust requirement.\nIf you encounter a compilation issue due to a dependency and don't want to upgrade your Rust toolchain, then you could downgrade the dependency.\n\n```sh\n# Use the problematic dependency's name and version\ncargo update -p package_name --precise 0.1.1\n```\n
\n\n## Community\n\n[![Linebender Zulip, #resvg channel](https://img.shields.io/badge/Linebender-%23resvg-blue?logo=Zulip)](https://xi.zulipchat.com/#narrow/channel/465085-resvg)\n\nDiscussion of SimpleCSS development happens in the Linebender Zulip at , specifically the [#resvg channel](https://xi.zulipchat.com/#narrow/channel/465085-resvg).\nAll public content can be read without logging in.\n\n## License\n\nLicensed under either of\n\n- Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or )\n- MIT license ([LICENSE-MIT](LICENSE-MIT) or )\n\nat your option.\n\n## Contribution\n\nContributions are welcome by pull request. The [Rust code of conduct] applies.\nPlease feel free to add your name to the [AUTHORS] file in any substantive pull request.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be licensed as above, without any additional terms or conditions.\n\n[Rust Code of Conduct]: https://www.rust-lang.org/policies/code-of-conduct\n[AUTHORS]: ./AUTHORS\n" }, { "package": "simplecss@0.2.2", "sourcePath": "examples/parse.rs", "sha256": "6439b8d0257865d1351645856c54586a6e44ba119eb28ab37e6be3c2c23f0de5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the SimpleCSS Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Parse\n\nuse std::io::{Read, Write};\n\nfn main() {\n let args: Vec<_> = std::env::args().collect();\n if args.len() != 2 {\n println!(\"Usage:\\n\\tparse style.css\\n\\tparse - 'p {{ color:red }}'\");\n std::process::exit(1);\n }\n\n std::env::set_var(\"RUST_LOG\", \"simplecss=warn\");\n env_logger::builder()\n .format(|buf, record| writeln!(buf, \"{}: {}\", record.level(), record.args()))\n .init();\n\n let text = if args[1] == \"-\" {\n let mut buffer = String::new();\n let stdin = std::io::stdin();\n let mut handle = stdin.lock();\n handle.read_to_string(&mut buffer).unwrap();\n buffer\n } else {\n std::fs::read_to_string(&args[1]).unwrap()\n };\n\n let style = simplecss::StyleSheet::parse(&text);\n println!(\"{:#?}\", style);\n}\n" }, { "package": "simplecss@0.2.2", "sourcePath": "examples/select.rs", "sha256": "4a18448eeac79fd59540f1f86af514524a0fb474f87c755508efa7319f0f05e5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the SimpleCSS Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Select\n\nstruct XmlNode<'a, 'input: 'a>(roxmltree::Node<'a, 'input>);\n\nimpl<'a, 'input: 'a> XmlNode<'a, 'input> {\n fn select(&self, text: &str) -> Option> {\n let selectors = simplecss::Selector::parse(text)?;\n self.0\n .descendants()\n .filter(|n| n.is_element())\n .find(|&node| selectors.matches(&XmlNode(node)))\n }\n}\n\nimpl simplecss::Element for XmlNode<'_, '_> {\n fn parent_element(&self) -> Option {\n self.0.parent_element().map(XmlNode)\n }\n\n fn prev_sibling_element(&self) -> Option {\n self.0\n .prev_siblings()\n .filter(|n| n.is_element())\n .nth(0)\n .map(XmlNode)\n }\n\n fn has_local_name(&self, local_name: &str) -> bool {\n self.0.tag_name().name() == local_name\n }\n\n fn attribute_matches(\n &self,\n local_name: &str,\n operator: simplecss::AttributeOperator<'_>,\n ) -> bool {\n match self.0.attribute(local_name) {\n Some(value) => operator.matches(value),\n None => false,\n }\n }\n\n fn pseudo_class_matches(&self, class: simplecss::PseudoClass<'_>) -> bool {\n match class {\n simplecss::PseudoClass::FirstChild => self.prev_sibling_element().is_none(),\n _ => false, // Since we are querying a static XML we can ignore other pseudo-classes.\n }\n }\n}\n\nfn main() {\n let doc = roxmltree::Document::parse(\n \"\n \n \n \n \n \",\n )\n .unwrap();\n let root = XmlNode(doc.root_element());\n\n assert_eq!(\n root.select(\"rect:first-child\")\n .unwrap()\n .attribute(\"id\")\n .unwrap(),\n \"rect1\",\n \"selected wrong element\"\n );\n\n assert_eq!(\n root.select(\"[color=red]\").unwrap().attribute(\"id\").unwrap(),\n \"rect2\",\n \"selected wrong element\"\n );\n\n assert_eq!(\n root.select(\"svg rect\").unwrap().attribute(\"id\").unwrap(),\n \"rect1\",\n \"selected wrong element\"\n );\n\n assert_eq!(\n root.select(\"svg > g > rect\")\n .unwrap()\n .attribute(\"id\")\n .unwrap(),\n \"rect1\",\n \"selected wrong element\"\n );\n\n assert_eq!(\n root.select(\".blue\").unwrap().attribute(\"id\").unwrap(),\n \"rect1\",\n \"selected wrong element\"\n );\n}\n" }, { "package": "simplecss@0.2.2", "sourcePath": "src/lib.rs", "sha256": "36389e726cf7c266026a22202ce79d1b3131f22a65443b1a070999694bdb0852", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2016 the SimpleCSS Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n/*!\nA simple [CSS 2.1](https://www.w3.org/TR/CSS21/) parser and selector.\n\nThis is not a browser-grade CSS parser. If you need one,\nuse [cssparser](https://crates.io/crates/cssparser) +\n[selectors](https://crates.io/crates/selectors).\n\nSince it's very simple we will start with limitations:\n\n## Limitations\n\n- [At-rules](https://www.w3.org/TR/CSS21/syndata.html#at-rules) are not supported.\n They will be skipped during parsing.\n- Property values are not parsed.\n In CSS like `* { width: 5px }` you will get a `width` property with a `5px` value as a string.\n- CDO/CDC comments are not supported.\n- Parser is case sensitive. All keywords must be lowercase.\n- Unicode escape, like `\\26`, is not supported.\n\n## Features\n\n- Selector matching support.\n- The rules are sorted by specificity.\n- `!important` parsing support.\n- Has a high-level parsers and low-level, zero-allocation tokenizers.\n- No unsafe.\n*/\n\n// LINEBENDER LINT SET - lib.rs - v2\n// See https://linebender.org/wiki/canonical-lints/\n// These lints aren't included in Cargo.toml because they\n// shouldn't apply to examples and tests\n#![warn(unused_crate_dependencies)]\n#![warn(clippy::print_stdout, clippy::print_stderr)]\n// Targeting e.g. 32-bit means structs containing usize can give false positives for 64-bit.\n#![cfg_attr(target_pointer_width = \"64\", warn(clippy::trivially_copy_pass_by_ref))]\n// END LINEBENDER LINT SET\n#![cfg_attr(docsrs, feature(doc_auto_cfg))]\n#![cfg_attr(all(not(feature = \"std\"), not(test)), no_std)]\n// The following lints are part of the Linebender standard set,\n// but resolving them has been deferred for now.\n// Feel free to send a PR that solves one or more of these.\n#![allow(\n missing_debug_implementations,\n unreachable_pub,\n clippy::use_self,\n clippy::missing_assert_message,\n clippy::missing_panics_doc,\n clippy::exhaustive_enums,\n clippy::unseparated_literal_suffix\n)]\n#![cfg_attr(test, allow(unused_crate_dependencies))] // Some dev dependencies are only used in tests\n\nextern crate alloc;\n\nuse alloc::vec::Vec;\nuse core::fmt;\n\nuse log::warn;\n\nmod selector;\nmod stream;\n\npub use selector::*;\nuse stream::Stream;\n\n/// A list of possible errors.\n#[derive(Clone, Copy, PartialEq, Debug)]\npub enum Error {\n /// The steam ended earlier than we expected.\n ///\n /// Should only appear on invalid input data.\n UnexpectedEndOfStream,\n\n /// An invalid ident.\n InvalidIdent(TextPos),\n\n /// An unclosed comment.\n InvalidComment(TextPos),\n\n /// An invalid declaration value.\n InvalidValue(TextPos),\n\n /// An invalid byte.\n #[allow(missing_docs)]\n InvalidByte {\n expected: u8,\n actual: u8,\n pos: TextPos,\n },\n\n /// A missing selector.\n SelectorMissing,\n\n /// An unexpected selector.\n UnexpectedSelector,\n\n /// An unexpected combinator.\n UnexpectedCombinator,\n\n /// An invalid or unsupported attribute selector.\n InvalidAttributeSelector,\n\n /// An invalid language pseudo-class.\n InvalidLanguagePseudoClass,\n}\n\nimpl fmt::Display for Error {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match *self {\n Error::UnexpectedEndOfStream => {\n write!(f, \"unexpected end of stream\")\n }\n Error::InvalidIdent(pos) => {\n write!(f, \"invalid ident at {}\", pos)\n }\n Error::InvalidComment(pos) => {\n write!(f, \"invalid comment at {}\", pos)\n }\n Error::InvalidValue(pos) => {\n write!(f, \"invalid value at {}\", pos)\n }\n Error::InvalidByte {\n expected,\n actual,\n pos,\n } => {\n write!(\n f,\n \"expected '{}' not '{}' at {}\",\n expected as char, actual as char, pos\n )\n }\n Error::SelectorMissing => {\n write!(f, \"selector missing\")\n }\n Error::UnexpectedSelector => {\n write!(f, \"unexpected selector\")\n }\n Error::UnexpectedCombinator => {\n write!(f, \"unexpected combinator\")\n }\n Error::InvalidAttributeSelector => {\n write!(f, \"invalid or unsupported attribute selector\")\n }\n Error::InvalidLanguagePseudoClass => {\n write!(f, \"invalid language pseudo-class\")\n }\n }\n }\n}\n\n#[cfg(feature = \"std\")]\nimpl std::error::Error for Error {}\n\n/// A position in text.\n///\n/// Position indicates a row/line and a column in the original text. Starting from 1:1.\n#[derive(Clone, Copy, PartialEq, Debug)]\n#[allow(missing_docs)]\npub struct TextPos {\n pub row: u32,\n pub col: u32,\n}\n\nimpl TextPos {\n /// Constructs a new `TextPos`.\n ///\n /// Should not be invoked manually, but rather via `Stream::gen_text_pos`.\n pub fn new(row: u32, col: u32) -> TextPos {\n TextPos { row, col }\n }\n}\n\nimpl fmt::Display for TextPos {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(f, \"{}:{}\", self.row, self.col)\n }\n}\n\n/// A declaration.\n#[derive(Clone, Copy, PartialEq, Debug)]\n#[allow(missing_docs)]\npub struct Declaration<'a> {\n pub name: &'a str,\n pub value: &'a str,\n pub important: bool,\n}\n\n/// A rule.\n#[derive(Clone, Debug)]\npub struct Rule<'a> {\n /// A rule selector.\n pub selector: Selector<'a>,\n /// A rule declarations.\n pub declarations: Vec>,\n}\n\n/// A style sheet.\n#[derive(Clone, Debug)]\npub struct StyleSheet<'a> {\n /// A list of rules.\n pub rules: Vec>,\n}\n\nimpl<'a> StyleSheet<'a> {\n /// Creates an empty style sheet.\n pub fn new() -> Self {\n StyleSheet { rules: Vec::new() }\n }\n\n /// Parses a style sheet from text.\n ///\n /// At-rules are not supported and will be skipped.\n ///\n /// # Errors\n ///\n /// Doesn't produce any errors. In worst case scenario will return an empty stylesheet.\n ///\n /// All warnings will be logged.\n pub fn parse(text: &'a str) -> Self {\n let mut sheet = StyleSheet::new();\n sheet.parse_more(text);\n sheet\n }\n\n /// Parses a style sheet from a text to the current style sheet.\n pub fn parse_more(&mut self, text: &'a str) {\n let mut s = Stream::from(text);\n\n if s.skip_spaces_and_comments().is_err() {\n return;\n }\n\n while !s.at_end() {\n if s.skip_spaces_and_comments().is_err() {\n break;\n }\n\n let _ = consume_statement(&mut s, &mut self.rules);\n }\n\n if !s.at_end() {\n warn!(\"{} bytes were left.\", s.slice_tail().len());\n }\n\n // Remove empty rules.\n self.rules.retain(|rule| !rule.declarations.is_empty());\n\n // Sort the rules by specificity.\n self.rules\n .sort_by_cached_key(|rule| rule.selector.specificity());\n }\n}\n\nimpl fmt::Display for StyleSheet<'_> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n for (i, rule) in self.rules.iter().enumerate() {\n write!(f, \"{} {{ \", rule.selector)?;\n for dec in &rule.declarations {\n write!(f, \"{}:{}\", dec.name, dec.value)?;\n if dec.important {\n write!(f, \" !important\")?;\n }\n write!(f, \";\")?;\n }\n write!(f, \" }}\")?;\n\n if i != self.rules.len() - 1 {\n writeln!(f)?;\n }\n }\n\n Ok(())\n }\n}\n\nimpl Default for StyleSheet<'_> {\n fn default() -> Self {\n Self::new()\n }\n}\n\nfn consume_statement<'a>(s: &mut Stream<'a>, rules: &mut Vec>) -> Result<(), Error> {\n if s.curr_byte() == Ok(b'@') {\n s.advance(1);\n consume_at_rule(s)\n } else {\n consume_rule_set(s, rules)\n }\n}\n\nfn consume_at_rule(s: &mut Stream<'_>) -> Result<(), Error> {\n let ident = s.consume_ident()?;\n warn!(\"The @{} rule is not supported. Skipped.\", ident);\n\n s.skip_bytes(|c| c != b';' && c != b'{');\n\n match s.curr_byte()? {\n b';' => s.advance(1),\n b'{' => consume_block(s),\n _ => {}\n }\n\n Ok(())\n}\n\nfn consume_rule_set<'a>(s: &mut Stream<'a>, rules: &mut Vec>) -> Result<(), Error> {\n let start_rule_idx = rules.len();\n\n while s.curr_byte()? == b',' || start_rule_idx == rules.len() {\n if s.curr_byte()? == b',' {\n s.advance(1);\n }\n\n let (selector, offset) = parse(s.slice_tail());\n s.advance(offset);\n s.skip_spaces();\n\n if let Some(selector) = selector {\n rules.push(Rule {\n selector,\n declarations: Vec::new(),\n });\n }\n\n match s.curr_byte()? {\n b'{' => break,\n b',' => {}\n _ => {\n s.skip_bytes(|c| c != b'{');\n break;\n }\n }\n }\n\n s.try_consume_byte(b'{');\n\n let declarations = consume_declarations(s)?;\n for rule in rules.iter_mut().skip(start_rule_idx) {\n rule.declarations = declarations.clone();\n }\n\n s.try_consume_byte(b'}');\n\n Ok(())\n}\n\nfn consume_block(s: &mut Stream<'_>) {\n s.try_consume_byte(b'{');\n consume_until_block_end(s);\n}\n\nfn consume_until_block_end(s: &mut Stream<'_>) {\n // Block can have nested blocks, so we have to check for matching braces.\n // We simply counting the number of opening braces, which is incorrect,\n // since `{` can be inside a string, but it's fine for majority of the cases.\n\n let mut braces = 0;\n while !s.at_end() {\n match s.curr_byte_unchecked() {\n b'{' => {\n braces += 1;\n }\n b'}' => {\n if braces == 0 {\n break;\n } else {\n braces -= 1;\n }\n }\n _ => {}\n }\n\n s.advance(1);\n }\n\n s.try_consume_byte(b'}');\n}\n\nfn consume_declarations<'a>(s: &mut Stream<'a>) -> Result>, Error> {\n let mut declarations = Vec::new();\n\n while !s.at_end() && s.curr_byte() != Ok(b'}') {\n match consume_declaration(s) {\n Ok(declaration) => declarations.push(declaration),\n Err(_) => {\n consume_until_block_end(s);\n break;\n }\n }\n }\n\n Ok(declarations)\n}\n\n/// A declaration tokenizer.\n///\n/// Tokenizer will stop at the first invalid token.\n///\n/// # Example\n///\n/// ```\n/// use simplecss::{DeclarationTokenizer, Declaration};\n///\n/// let mut t = DeclarationTokenizer::from(\"background: url(\\\"img.png\\\"); color:red !important\");\n/// assert_eq!(t.next().unwrap(), Declaration { name: \"background\", value: \"url(\\\"img.png\\\")\", important: false });\n/// assert_eq!(t.next().unwrap(), Declaration { name: \"color\", value: \"red\", important: true });\n/// ```\npub struct DeclarationTokenizer<'a> {\n stream: Stream<'a>,\n}\n\nimpl<'a> From<&'a str> for DeclarationTokenizer<'a> {\n fn from(text: &'a str) -> Self {\n DeclarationTokenizer {\n stream: Stream::from(text),\n }\n }\n}\n\nimpl<'a> Iterator for DeclarationTokenizer<'a> {\n type Item = Declaration<'a>;\n\n fn next(&mut self) -> Option {\n let _ = self.stream.skip_spaces_and_comments();\n\n if self.stream.at_end() {\n return None;\n }\n\n match consume_declaration(&mut self.stream) {\n Ok(v) => Some(v),\n Err(_) => {\n self.stream.jump_to_end();\n None\n }\n }\n }\n}\n\nfn consume_declaration<'a>(s: &mut Stream<'a>) -> Result, Error> {\n s.skip_spaces_and_comments()?;\n\n // Parse name.\n\n // https://snook.ca/archives/html_and_css/targetting_ie7\n if s.curr_byte() == Ok(b'*') {\n s.advance(1);\n }\n\n let name = s.consume_ident()?;\n\n s.skip_spaces_and_comments()?;\n s.consume_byte(b':')?;\n s.skip_spaces_and_comments()?;\n\n // Parse value.\n let start = s.pos();\n let mut end = s.pos();\n while consume_term(s).is_ok() {\n end = s.pos();\n s.skip_spaces_and_comments()?;\n }\n let value = s.slice_range(start, end).trim();\n\n s.skip_spaces_and_comments()?;\n\n // Check for `important`.\n let mut important = false;\n if s.curr_byte() == Ok(b'!') {\n s.advance(1);\n s.skip_spaces_and_comments()?;\n if s.slice_tail().starts_with(\"important\") {\n s.advance(9);\n important = true;\n }\n }\n\n s.skip_spaces_and_comments()?;\n\n while s.curr_byte() == Ok(b';') {\n s.advance(1);\n s.skip_spaces_and_comments()?;\n }\n\n s.skip_spaces_and_comments()?;\n\n if value.is_empty() {\n return Err(Error::InvalidValue(s.gen_text_pos_from(start)));\n }\n\n Ok(Declaration {\n name,\n value,\n important,\n })\n}\n\nfn consume_term(s: &mut Stream<'_>) -> Result<(), Error> {\n fn consume_digits(s: &mut Stream<'_>) {\n while let Ok(b'0'..=b'9') = s.curr_byte() {\n s.advance(1);\n }\n }\n\n match s.curr_byte()? {\n b'#' => {\n s.advance(1);\n match s.consume_ident() {\n Ok(_) => {}\n Err(_) => {\n // Try consume as a hex color.\n while let Ok(c) = s.curr_byte() {\n match c {\n b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' => s.advance(1),\n _ => break,\n }\n }\n }\n }\n }\n b'+' | b'-' | b'0'..=b'9' | b'.' => {\n // Consume number.\n\n s.advance(1);\n consume_digits(s);\n if s.curr_byte() == Ok(b'.') {\n s.advance(1);\n consume_digits(s);\n }\n\n if s.curr_byte() == Ok(b'%') {\n s.advance(1);\n } else {\n // Consume suffix if any.\n let _ = s.consume_ident();\n }\n }\n b'\\'' | b'\"' => {\n s.consume_string()?;\n }\n b',' => {\n s.advance(1);\n }\n _ => {\n let _ = s.consume_ident()?;\n\n // Consume function.\n if s.curr_byte() == Ok(b'(') {\n s.skip_bytes(|c| c != b')');\n s.consume_byte(b')')?;\n }\n }\n }\n\n Ok(())\n}\n" }, { "package": "simplecss@0.2.2", "sourcePath": "src/selector.rs", "sha256": "f1283a29b37232ee9fb8f9715da9d564d96c309e59e770e5fdddff6f9cae006e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the SimpleCSS Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse alloc::{vec, vec::Vec};\nuse core::fmt;\n\nuse log::warn;\n\nuse crate::stream::Stream;\nuse crate::Error;\n\n/// An attribute selector operator.\n#[derive(Clone, Copy, PartialEq, Debug)]\npub enum AttributeOperator<'a> {\n /// `[attr]`\n Exists,\n /// `[attr=value]`\n Matches(&'a str),\n /// `[attr~=value]`\n Contains(&'a str),\n /// `[attr|=value]`\n StartsWith(&'a str),\n}\n\nimpl AttributeOperator<'_> {\n /// Checks that value is matching the operator.\n pub fn matches(&self, value: &str) -> bool {\n match *self {\n AttributeOperator::Exists => true,\n AttributeOperator::Matches(v) => value == v,\n AttributeOperator::Contains(v) => value.split(' ').any(|s| s == v),\n AttributeOperator::StartsWith(v) => {\n // exactly `v` or beginning with `v` immediately followed by `-`\n if value == v {\n true\n } else if value.starts_with(v) {\n value.get(v.len()..v.len() + 1) == Some(\"-\")\n } else {\n false\n }\n }\n }\n }\n}\n\n/// A pseudo-class.\n#[derive(Clone, Copy, PartialEq, Debug)]\n#[allow(missing_docs)]\npub enum PseudoClass<'a> {\n FirstChild,\n Link,\n Visited,\n Hover,\n Active,\n Focus,\n Lang(&'a str),\n}\n\nimpl fmt::Display for PseudoClass<'_> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n match self {\n PseudoClass::FirstChild => write!(f, \"first-child\"),\n PseudoClass::Link => write!(f, \"link\"),\n PseudoClass::Visited => write!(f, \"visited\"),\n PseudoClass::Hover => write!(f, \"hover\"),\n PseudoClass::Active => write!(f, \"active\"),\n PseudoClass::Focus => write!(f, \"focus\"),\n PseudoClass::Lang(lang) => write!(f, \"lang({})\", lang),\n }\n }\n}\n\n/// A trait to query an element node metadata.\npub trait Element: Sized {\n /// Returns a parent element.\n fn parent_element(&self) -> Option;\n\n /// Returns a previous sibling element.\n fn prev_sibling_element(&self) -> Option;\n\n /// Checks that the element has a specified local name.\n fn has_local_name(&self, name: &str) -> bool;\n\n /// Checks that the element has a specified attribute.\n fn attribute_matches(&self, local_name: &str, operator: AttributeOperator<'_>) -> bool;\n\n /// Checks that the element matches a specified pseudo-class.\n fn pseudo_class_matches(&self, class: PseudoClass<'_>) -> bool;\n}\n\n#[derive(Clone, Copy, PartialEq, Debug)]\nenum SimpleSelectorType<'a> {\n Type(&'a str),\n Universal,\n}\n\n#[derive(Clone, Copy, PartialEq, Debug)]\nenum SubSelector<'a> {\n Attribute(&'a str, AttributeOperator<'a>),\n PseudoClass(PseudoClass<'a>),\n}\n\n#[derive(Clone, Debug)]\nstruct SimpleSelector<'a> {\n kind: SimpleSelectorType<'a>,\n subselectors: Vec>,\n}\n\n#[derive(Clone, Copy, PartialEq, Debug)]\nenum Combinator {\n None,\n Descendant,\n Child,\n AdjacentSibling,\n}\n\n#[derive(Clone, Debug)]\nstruct Component<'a> {\n /// A combinator that precede the selector.\n combinator: Combinator,\n selector: SimpleSelector<'a>,\n}\n\n/// A selector.\n#[derive(Clone, Debug)]\npub struct Selector<'a> {\n components: Vec>,\n}\n\nimpl<'a> Selector<'a> {\n /// Parses a selector from a string.\n ///\n /// Will log any errors as a warnings.\n ///\n /// Parsing will be stopped at EOF, `,` or `{`.\n pub fn parse(text: &'a str) -> Option {\n parse(text).0\n }\n\n /// Compute the selector's specificity.\n ///\n /// Cf. .\n pub fn specificity(&self) -> [u8; 3] {\n let mut spec = [0u8; 3];\n\n for selector in self.components.iter().map(|c| &c.selector) {\n if matches!(selector.kind, SimpleSelectorType::Type(_)) {\n spec[2] = spec[2].saturating_add(1);\n }\n\n for sub in &selector.subselectors {\n match sub {\n SubSelector::Attribute(\"id\", _) => spec[0] = spec[0].saturating_add(1),\n _ => spec[1] = spec[1].saturating_add(1),\n }\n }\n }\n\n spec\n }\n\n /// Checks that the provided element matches the current selector.\n pub fn matches(&self, element: &E) -> bool {\n assert!(!self.components.is_empty(), \"selector must not be empty\");\n assert_eq!(\n self.components[0].combinator,\n Combinator::None,\n \"the first component must not have a combinator\"\n );\n\n self.matches_impl(self.components.len() - 1, element)\n }\n\n fn matches_impl(&self, idx: usize, element: &E) -> bool {\n let component = &self.components[idx];\n\n if !match_selector(&component.selector, element) {\n return false;\n }\n\n match component.combinator {\n Combinator::Descendant => {\n let mut parent = element.parent_element();\n while let Some(e) = parent {\n if self.matches_impl(idx - 1, &e) {\n return true;\n }\n\n parent = e.parent_element();\n }\n\n false\n }\n Combinator::Child => {\n if let Some(parent) = element.parent_element() {\n if self.matches_impl(idx - 1, &parent) {\n return true;\n }\n }\n\n false\n }\n Combinator::AdjacentSibling => {\n if let Some(prev) = element.prev_sibling_element() {\n if self.matches_impl(idx - 1, &prev) {\n return true;\n }\n }\n\n false\n }\n Combinator::None => true,\n }\n }\n}\n\nfn match_selector(selector: &SimpleSelector<'_>, element: &E) -> bool {\n if let SimpleSelectorType::Type(ident) = selector.kind {\n if !element.has_local_name(ident) {\n return false;\n }\n }\n\n for sub in &selector.subselectors {\n match sub {\n SubSelector::Attribute(name, operator) => {\n if !element.attribute_matches(name, *operator) {\n return false;\n }\n }\n SubSelector::PseudoClass(class) => {\n if !element.pseudo_class_matches(*class) {\n return false;\n }\n }\n }\n }\n\n true\n}\n\npub(crate) fn parse(text: &str) -> (Option>, usize) {\n let mut components: Vec> = Vec::new();\n let mut combinator = Combinator::None;\n\n let mut tokenizer = SelectorTokenizer::from(text);\n for token in &mut tokenizer {\n let mut add_sub = |sub| {\n if combinator == Combinator::None && !components.is_empty() {\n if let Some(ref mut component) = components.last_mut() {\n component.selector.subselectors.push(sub);\n }\n } else {\n components.push(Component {\n selector: SimpleSelector {\n kind: SimpleSelectorType::Universal,\n subselectors: vec![sub],\n },\n combinator,\n });\n\n combinator = Combinator::None;\n }\n };\n\n let token = match token {\n Ok(t) => t,\n Err(e) => {\n warn!(\"Selector parsing failed cause {}.\", e);\n return (None, tokenizer.stream.pos());\n }\n };\n\n match token {\n SelectorToken::UniversalSelector => {\n components.push(Component {\n selector: SimpleSelector {\n kind: SimpleSelectorType::Universal,\n subselectors: Vec::new(),\n },\n combinator,\n });\n\n combinator = Combinator::None;\n }\n SelectorToken::TypeSelector(ident) => {\n components.push(Component {\n selector: SimpleSelector {\n kind: SimpleSelectorType::Type(ident),\n subselectors: Vec::new(),\n },\n combinator,\n });\n\n combinator = Combinator::None;\n }\n SelectorToken::ClassSelector(ident) => {\n add_sub(SubSelector::Attribute(\n \"class\",\n AttributeOperator::Contains(ident),\n ));\n }\n SelectorToken::IdSelector(id) => {\n add_sub(SubSelector::Attribute(\"id\", AttributeOperator::Matches(id)));\n }\n SelectorToken::AttributeSelector(name, op) => {\n add_sub(SubSelector::Attribute(name, op));\n }\n SelectorToken::PseudoClass(ident) => {\n let class = match ident {\n \"first-child\" => PseudoClass::FirstChild,\n \"link\" => PseudoClass::Link,\n \"visited\" => PseudoClass::Visited,\n \"hover\" => PseudoClass::Hover,\n \"active\" => PseudoClass::Active,\n \"focus\" => PseudoClass::Focus,\n _ => {\n warn!(\"':{}' is not supported. Selector skipped.\", ident);\n return (None, tokenizer.stream.pos());\n }\n };\n\n // TODO: duplicates\n // TODO: order\n\n add_sub(SubSelector::PseudoClass(class));\n }\n SelectorToken::LangPseudoClass(lang) => {\n add_sub(SubSelector::PseudoClass(PseudoClass::Lang(lang)));\n }\n SelectorToken::DescendantCombinator => {\n combinator = Combinator::Descendant;\n }\n SelectorToken::ChildCombinator => {\n combinator = Combinator::Child;\n }\n SelectorToken::AdjacentCombinator => {\n combinator = Combinator::AdjacentSibling;\n }\n }\n }\n\n if components.is_empty() {\n (None, tokenizer.stream.pos())\n } else if components[0].combinator != Combinator::None {\n debug_assert_eq!(\n components[0].combinator,\n Combinator::None,\n \"the first component must not have a combinator\"\n );\n\n (None, tokenizer.stream.pos())\n } else {\n (Some(Selector { components }), tokenizer.stream.pos())\n }\n}\n\nimpl fmt::Display for Selector<'_> {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n for component in &self.components {\n match component.combinator {\n Combinator::Descendant => write!(f, \" \")?,\n Combinator::Child => write!(f, \" > \")?,\n Combinator::AdjacentSibling => write!(f, \" + \")?,\n Combinator::None => {}\n }\n\n match component.selector.kind {\n SimpleSelectorType::Universal => write!(f, \"*\")?,\n SimpleSelectorType::Type(ident) => write!(f, \"{}\", ident)?,\n };\n\n for sel in &component.selector.subselectors {\n match sel {\n SubSelector::Attribute(name, operator) => {\n match operator {\n AttributeOperator::Exists => {\n write!(f, \"[{}]\", name)?;\n }\n AttributeOperator::Matches(value) => {\n write!(f, \"[{}='{}']\", name, value)?;\n }\n AttributeOperator::Contains(value) => {\n write!(f, \"[{}~='{}']\", name, value)?;\n }\n AttributeOperator::StartsWith(value) => {\n write!(f, \"[{}|='{}']\", name, value)?;\n }\n };\n }\n SubSelector::PseudoClass(class) => write!(f, \":{}\", class)?,\n }\n }\n }\n\n Ok(())\n }\n}\n\n/// A selector token.\n#[derive(Clone, Copy, PartialEq, Debug)]\npub enum SelectorToken<'a> {\n /// `*`\n UniversalSelector,\n\n /// `div`\n TypeSelector(&'a str),\n\n /// `.class`\n ClassSelector(&'a str),\n\n /// `#id`\n IdSelector(&'a str),\n\n /// `[color=red]`\n AttributeSelector(&'a str, AttributeOperator<'a>),\n\n /// `:first-child`\n PseudoClass(&'a str),\n\n /// `:lang(en)`\n LangPseudoClass(&'a str),\n\n /// `a b`\n DescendantCombinator,\n\n /// `a > b`\n ChildCombinator,\n\n /// `a + b`\n AdjacentCombinator,\n}\n\n/// A selector tokenizer.\n///\n/// # Example\n///\n/// ```\n/// use simplecss::{SelectorTokenizer, SelectorToken};\n///\n/// let mut t = SelectorTokenizer::from(\"div > p:first-child\");\n/// assert_eq!(t.next().unwrap().unwrap(), SelectorToken::TypeSelector(\"div\"));\n/// assert_eq!(t.next().unwrap().unwrap(), SelectorToken::ChildCombinator);\n/// assert_eq!(t.next().unwrap().unwrap(), SelectorToken::TypeSelector(\"p\"));\n/// assert_eq!(t.next().unwrap().unwrap(), SelectorToken::PseudoClass(\"first-child\"));\n/// assert!(t.next().is_none());\n/// ```\npub struct SelectorTokenizer<'a> {\n stream: Stream<'a>,\n after_combinator: bool,\n finished: bool,\n}\n\nimpl<'a> From<&'a str> for SelectorTokenizer<'a> {\n fn from(text: &'a str) -> Self {\n SelectorTokenizer {\n stream: Stream::from(text),\n after_combinator: true,\n finished: false,\n }\n }\n}\n\nimpl<'a> Iterator for SelectorTokenizer<'a> {\n type Item = Result, Error>;\n\n fn next(&mut self) -> Option {\n if self.finished || self.stream.at_end() {\n if self.after_combinator {\n self.after_combinator = false;\n return Some(Err(Error::SelectorMissing));\n }\n\n return None;\n }\n\n macro_rules! try2 {\n ($e:expr) => {\n match $e {\n Ok(v) => v,\n Err(e) => {\n self.finished = true;\n return Some(Err(e));\n }\n }\n };\n }\n\n match self.stream.curr_byte_unchecked() {\n b'*' => {\n if !self.after_combinator {\n self.finished = true;\n return Some(Err(Error::UnexpectedSelector));\n }\n\n self.after_combinator = false;\n self.stream.advance(1);\n Some(Ok(SelectorToken::UniversalSelector))\n }\n b'#' => {\n self.after_combinator = false;\n self.stream.advance(1);\n let ident = try2!(self.stream.consume_ident());\n Some(Ok(SelectorToken::IdSelector(ident)))\n }\n b'.' => {\n self.after_combinator = false;\n self.stream.advance(1);\n let ident = try2!(self.stream.consume_ident());\n Some(Ok(SelectorToken::ClassSelector(ident)))\n }\n b'[' => {\n self.after_combinator = false;\n self.stream.advance(1);\n let ident = try2!(self.stream.consume_ident());\n\n let op = match try2!(self.stream.curr_byte()) {\n b']' => AttributeOperator::Exists,\n b'=' => {\n self.stream.advance(1);\n let value = try2!(self.stream.consume_string());\n AttributeOperator::Matches(value)\n }\n b'~' => {\n self.stream.advance(1);\n try2!(self.stream.consume_byte(b'='));\n let value = try2!(self.stream.consume_string());\n AttributeOperator::Contains(value)\n }\n b'|' => {\n self.stream.advance(1);\n try2!(self.stream.consume_byte(b'='));\n let value = try2!(self.stream.consume_string());\n AttributeOperator::StartsWith(value)\n }\n _ => {\n self.finished = true;\n return Some(Err(Error::InvalidAttributeSelector));\n }\n };\n\n try2!(self.stream.consume_byte(b']'));\n\n Some(Ok(SelectorToken::AttributeSelector(ident, op)))\n }\n b':' => {\n self.after_combinator = false;\n self.stream.advance(1);\n let ident = try2!(self.stream.consume_ident());\n\n if ident == \"lang\" {\n try2!(self.stream.consume_byte(b'('));\n let lang = self.stream.consume_bytes(|c| c != b')').trim();\n try2!(self.stream.consume_byte(b')'));\n\n if lang.is_empty() {\n self.finished = true;\n return Some(Err(Error::InvalidLanguagePseudoClass));\n }\n\n Some(Ok(SelectorToken::LangPseudoClass(lang)))\n } else {\n Some(Ok(SelectorToken::PseudoClass(ident)))\n }\n }\n b'>' => {\n if self.after_combinator {\n self.after_combinator = false;\n self.finished = true;\n return Some(Err(Error::UnexpectedCombinator));\n }\n\n self.stream.advance(1);\n self.after_combinator = true;\n Some(Ok(SelectorToken::ChildCombinator))\n }\n b'+' => {\n if self.after_combinator {\n self.after_combinator = false;\n self.finished = true;\n return Some(Err(Error::UnexpectedCombinator));\n }\n\n self.stream.advance(1);\n self.after_combinator = true;\n Some(Ok(SelectorToken::AdjacentCombinator))\n }\n b' ' | b'\\t' | b'\\n' | b'\\r' | b'\\x0C' => {\n self.stream.skip_spaces();\n\n if self.after_combinator {\n return self.next();\n }\n\n while self.stream.curr_byte() == Ok(b'/') {\n try2!(self.stream.skip_comment());\n self.stream.skip_spaces();\n }\n\n match self.stream.curr_byte() {\n Ok(b'>') | Ok(b'+') | Ok(b',') | Ok(b'{') | Err(_) => self.next(),\n _ => {\n if self.after_combinator {\n self.after_combinator = false;\n self.finished = true;\n return Some(Err(Error::UnexpectedSelector));\n }\n\n self.after_combinator = true;\n Some(Ok(SelectorToken::DescendantCombinator))\n }\n }\n }\n b'/' => {\n if self.stream.next_byte() == Ok(b'*') {\n try2!(self.stream.skip_comment());\n } else {\n self.finished = true;\n }\n\n self.next()\n }\n b',' | b'{' => {\n self.finished = true;\n self.next()\n }\n _ => {\n let ident = try2!(self.stream.consume_ident());\n\n if !self.after_combinator {\n self.finished = true;\n return Some(Err(Error::UnexpectedSelector));\n }\n\n self.after_combinator = false;\n Some(Ok(SelectorToken::TypeSelector(ident)))\n }\n }\n }\n}\n" }, { "package": "simplecss@0.2.2", "sourcePath": "src/stream.rs", "sha256": "b9750b32e474dd7d69430c9ed7e848df6ceb5fe52a0da20159d6d1274dfeee8e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2016 the SimpleCSS Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\nuse core::str;\n\nuse crate::{Error, TextPos};\n\ntrait CssCharExt {\n fn is_name_start(&self) -> bool;\n fn is_name_char(&self) -> bool;\n fn is_non_ascii(&self) -> bool;\n fn is_escape(&self) -> bool;\n}\n\nimpl CssCharExt for char {\n #[inline]\n fn is_name_start(&self) -> bool {\n match *self {\n '_' | 'a'..='z' | 'A'..='Z' => true,\n _ => self.is_non_ascii() || self.is_escape(),\n }\n }\n\n #[inline]\n fn is_name_char(&self) -> bool {\n match *self {\n '_' | 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' => true,\n _ => self.is_non_ascii() || self.is_escape(),\n }\n }\n\n #[inline]\n fn is_non_ascii(&self) -> bool {\n *self as u32 > 237\n }\n\n #[inline]\n fn is_escape(&self) -> bool {\n // TODO: this\n false\n }\n}\n\n#[derive(Clone, Copy, PartialEq, Debug)]\npub(crate) struct Stream<'a> {\n text: &'a str,\n pos: usize,\n end: usize,\n}\n\nimpl<'a> From<&'a str> for Stream<'a> {\n fn from(text: &'a str) -> Self {\n Stream::new(text)\n }\n}\n\nimpl<'a> Stream<'a> {\n pub fn new(text: &'a str) -> Self {\n Stream {\n text,\n pos: 0,\n end: text.len(),\n }\n }\n\n #[inline]\n pub fn pos(&self) -> usize {\n self.pos\n }\n\n #[inline]\n pub fn jump_to_end(&mut self) {\n self.pos = self.end;\n }\n\n #[inline]\n pub fn at_end(&self) -> bool {\n self.pos >= self.end\n }\n\n #[inline]\n pub fn curr_byte(&self) -> Result {\n if self.at_end() {\n return Err(Error::UnexpectedEndOfStream);\n }\n\n Ok(self.curr_byte_unchecked())\n }\n\n #[inline]\n pub fn curr_byte_unchecked(&self) -> u8 {\n self.text.as_bytes()[self.pos]\n }\n\n #[inline]\n pub fn next_byte(&self) -> Result {\n if self.pos + 1 >= self.end {\n return Err(Error::UnexpectedEndOfStream);\n }\n\n Ok(self.text.as_bytes()[self.pos + 1])\n }\n\n #[inline]\n pub fn advance(&mut self, n: usize) {\n debug_assert!(self.pos + n <= self.end);\n self.pos += n;\n }\n\n pub fn consume_byte(&mut self, c: u8) -> Result<(), Error> {\n if self.curr_byte()? != c {\n return Err(Error::InvalidByte {\n expected: c,\n actual: self.curr_byte()?,\n pos: self.gen_text_pos(),\n });\n }\n\n self.advance(1);\n Ok(())\n }\n\n pub fn try_consume_byte(&mut self, c: u8) {\n if self.curr_byte() == Ok(c) {\n self.advance(1);\n }\n }\n\n pub fn consume_bytes(&mut self, f: F) -> &'a str\n where\n F: Fn(u8) -> bool,\n {\n let start = self.pos;\n self.skip_bytes(f);\n self.slice_back(start)\n }\n\n pub fn skip_bytes(&mut self, f: F)\n where\n F: Fn(u8) -> bool,\n {\n while !self.at_end() && f(self.curr_byte_unchecked()) {\n self.advance(1);\n }\n }\n\n #[inline]\n fn chars(&self) -> str::Chars<'a> {\n self.text[self.pos..self.end].chars()\n }\n\n #[inline]\n pub fn slice_range(&self, start: usize, end: usize) -> &'a str {\n &self.text[start..end]\n }\n\n #[inline]\n pub fn slice_back(&self, pos: usize) -> &'a str {\n &self.text[pos..self.pos]\n }\n\n #[inline]\n pub fn slice_tail(&self) -> &'a str {\n &self.text[self.pos..]\n }\n\n #[inline]\n pub fn skip_spaces(&mut self) {\n while !self.at_end() {\n match self.curr_byte_unchecked() {\n b' ' | b'\\t' | b'\\n' | b'\\r' | b'\\x0C' => self.advance(1),\n _ => break,\n }\n }\n }\n\n #[inline]\n pub fn skip_spaces_and_comments(&mut self) -> Result<(), Error> {\n self.skip_spaces();\n while self.curr_byte() == Ok(b'/') && self.next_byte() == Ok(b'*') {\n self.skip_comment()?;\n self.skip_spaces();\n }\n\n Ok(())\n }\n\n pub fn consume_ident(&mut self) -> Result<&'a str, Error> {\n let start = self.pos();\n\n if self.curr_byte() == Ok(b'-') {\n self.advance(1);\n }\n\n let mut iter = self.chars();\n if let Some(c) = iter.next() {\n if c.is_name_start() {\n self.advance(c.len_utf8());\n } else {\n return Err(Error::InvalidIdent(self.gen_text_pos_from(start)));\n }\n }\n\n for c in iter {\n if c.is_name_char() {\n self.advance(c.len_utf8());\n } else {\n break;\n }\n }\n\n if start == self.pos() {\n return Err(Error::InvalidIdent(self.gen_text_pos_from(start)));\n }\n\n let name = self.slice_back(start);\n Ok(name)\n }\n\n pub fn consume_string(&mut self) -> Result<&'a str, Error> {\n // Check for opening quote.\n let quote = self.curr_byte()?;\n if quote == b'\\'' || quote == b'\"' {\n let mut prev = quote;\n self.advance(1);\n\n let start = self.pos();\n\n while !self.at_end() {\n let curr = self.curr_byte_unchecked();\n\n // Advance until the closing quote.\n if curr == quote {\n // Check for escaped quote.\n if prev != b'\\\\' {\n break;\n }\n }\n\n prev = curr;\n self.advance(1);\n }\n\n let value = self.slice_back(start);\n\n // Check for closing quote.\n self.consume_byte(quote)?;\n\n Ok(value)\n } else {\n self.consume_ident()\n }\n }\n\n pub fn skip_comment(&mut self) -> Result<(), Error> {\n let start = self.pos();\n self.skip_comment_impl()\n .map_err(|_| Error::InvalidComment(self.gen_text_pos_from(start)))?;\n Ok(())\n }\n\n fn skip_comment_impl(&mut self) -> Result<(), Error> {\n self.consume_byte(b'/')?;\n self.consume_byte(b'*')?;\n\n while !self.at_end() {\n let curr = self.curr_byte_unchecked();\n if curr == b'*' && self.next_byte() == Ok(b'/') {\n break;\n }\n\n self.advance(1);\n }\n\n self.consume_byte(b'*')?;\n self.consume_byte(b'/')?;\n Ok(())\n }\n\n #[inline(never)]\n pub fn gen_text_pos(&self) -> TextPos {\n let row = Self::calc_curr_row(self.text, self.pos);\n let col = Self::calc_curr_col(self.text, self.pos);\n TextPos::new(row, col)\n }\n\n #[inline(never)]\n pub fn gen_text_pos_from(&self, pos: usize) -> TextPos {\n let mut s = *self;\n s.pos = core::cmp::min(pos, self.text.len());\n s.gen_text_pos()\n }\n\n fn calc_curr_row(text: &str, end: usize) -> u32 {\n let mut row = 1;\n for c in &text.as_bytes()[..end] {\n if *c == b'\\n' {\n row += 1;\n }\n }\n\n row\n }\n\n fn calc_curr_col(text: &str, end: usize) -> u32 {\n let mut col = 1;\n for c in text[..end].chars().rev() {\n if c == '\\n' {\n break;\n } else {\n col += 1;\n }\n }\n\n col\n }\n}\n" }, { "package": "simplecss@0.2.2", "sourcePath": "tests/declaration_tokenizer.rs", "sha256": "b2ffc1d4a84ae740dfd6fba873b3b32604a7d445d9ab8ddb3927e7103ae0b04b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the SimpleCSS Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Declaration Tokenizer\n\nuse simplecss::*;\n\nmacro_rules! tokenize {\n ($name:ident, $text:expr, $( $token:expr ),*) => (\n #[test]\n fn $name() {\n let mut t = DeclarationTokenizer::from($text);\n $(\n assert_eq!(t.next().unwrap(), $token);\n )*\n\n assert!(t.next().is_none());\n }\n )\n}\n\nfn declare<'a>(name: &'a str, value: &'a str) -> Declaration<'a> {\n Declaration {\n name,\n value,\n important: false,\n }\n}\n\nfn declare_important<'a>(name: &'a str, value: &'a str) -> Declaration<'a> {\n Declaration {\n name,\n value,\n important: true,\n }\n}\n\ntokenize!(tokenize_01, \"\",);\n\ntokenize!(tokenize_02, \" \",);\n\ntokenize!(tokenize_03, \"/**/\",);\n\ntokenize!(tokenize_04, \"color:red\", declare(\"color\", \"red\"));\n\ntokenize!(tokenize_05, \"color:red;\", declare(\"color\", \"red\"));\n\ntokenize!(tokenize_06, \"color:red \", declare(\"color\", \"red\"));\n\ntokenize!(tokenize_07, \" color: red; \", declare(\"color\", \"red\"));\n\ntokenize!(tokenize_08, \" color : red ; \", declare(\"color\", \"red\"));\n\ntokenize!(\n tokenize_09,\n \" color:red;;;;color:red; \",\n declare(\"color\", \"red\"),\n declare(\"color\", \"red\")\n);\n\ntokenize!(\n tokenize_10,\n \"background: url(\\\"img.png\\\");\",\n declare(\"background\", \"url(\\\"img.png\\\")\")\n);\n\ntokenize!(\n tokenize_11,\n \"background: url(\\\"{}\\\");\",\n declare(\"background\", \"url(\\\"{}\\\")\")\n);\n\ntokenize!(\n tokenize_12,\n \"color: red ! important\",\n declare_important(\"color\", \"red\")\n);\n\ntokenize!(\n tokenize_13,\n \"color: red !important\",\n declare_important(\"color\", \"red\")\n);\n\ntokenize!(\n tokenize_14,\n \"color: red!important\",\n declare_important(\"color\", \"red\")\n);\n\ntokenize!(\n tokenize_15,\n \"color: red !/**/important\",\n declare_important(\"color\", \"red\")\n);\n\ntokenize!(\n tokenize_16,\n \"border: 1em solid blue\",\n declare(\"border\", \"1em solid blue\")\n);\n\ntokenize!(\n tokenize_17,\n \"background: navy url(support/diamond.png) -2em -2em no-repeat\",\n declare(\n \"background\",\n \"navy url(support/diamond.png) -2em -2em no-repeat\"\n )\n);\n\ntokenize!(tokenize_18, \"/**/color:red\", declare(\"color\", \"red\"));\n\ntokenize!(tokenize_19, \"/* *\\\\/*/color: red;\", declare(\"color\", \"red\"));\n\ntokenize!(\n tokenize_20,\n \"/**/color/**/:/**/red/**/;/**/\",\n declare(\"color\", \"red\")\n);\n\ntokenize!(tokenize_21, \"\\ncolor\\n:\\nred\\n;\\n\", declare(\"color\", \"red\"));\n\ntokenize!(tokenize_22, \"{color:red}\",);\n\ntokenize!(tokenize_23, \"(color:red)\",);\n\ntokenize!(tokenize_24, \"[color:red]\",);\n\ntokenize!(tokenize_25, \"color:\",);\n\ntokenize!(tokenize_26, \"value:\\\"text\\\"\", declare(\"value\", \"\\\"text\\\"\"));\n\ntokenize!(tokenize_27, \"value:'text'\", declare(\"value\", \"'text'\"));\n\ntokenize!(tokenize_28, \"color:#fff\", declare(\"color\", \"#fff\"));\ntokenize!(tokenize_29, \"color:0.5\", declare(\"color\", \"0.5\"));\n\ntokenize!(tokenize_30, \"color:.5\", declare(\"color\", \".5\"));\n\ntokenize!(tokenize_31, \"color:#FFF\", declare(\"color\", \"#FFF\"));\n\ntokenize!(\n tokenize_32,\n \"content: counter(chapno, upper-roman) \\\". \\\"\",\n declare(\"content\", \"counter(chapno, upper-roman) \\\". \\\"\")\n);\n\ntokenize!(\n tokenize_33,\n \"font-family:'Noto Serif','DejaVu Serif',serif\",\n declare(\"font-family\", \"'Noto Serif','DejaVu Serif',serif\")\n);\n\ntokenize!(tokenize_34, \"*zoom:1;\", declare(\"zoom\", \"1\"));\n\n//tokenize!(tokenize_, \"@unsupported { splines: reticulating } color: green\",\n// declare(\"color\", \"green\")\n//);\n\n//tokenize!(tokenize_, \"/*\\\\*/*/color: red;\", declare(\"color\", \"red\"));\n\n//tokenize!(tokenize_, \"\\\"this is a string]}\\\"\\\"[{\\\\\\\"'\\\"; /*should be parsed as a string but be ignored*/\n// {{}}[]''; /*should be parsed as nested blocks and a string but be ignored*/\n// color: red;\", declare(\"color\", \"red\"));\n" }, { "package": "simplecss@0.2.2", "sourcePath": "tests/select.rs", "sha256": "180301f547fe7dc44efdc26e27cee4df7ffb0a19372f79ef22b6d75c5e85b6cf", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the SimpleCSS Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Select\n\nuse simplecss::*;\n\nstruct XmlNode<'a, 'input: 'a>(roxmltree::Node<'a, 'input>);\n\nimpl<'a, 'input: 'a> XmlNode<'a, 'input> {\n fn select(&self, text: &str) -> Vec> {\n let selectors = Selector::parse(text).unwrap();\n let mut nodes = Vec::new();\n for node in self.0.descendants().filter(|n| n.is_element()) {\n if selectors.matches(&XmlNode(node)) {\n nodes.push(node);\n }\n }\n\n nodes\n }\n}\n\nimpl Element for XmlNode<'_, '_> {\n fn parent_element(&self) -> Option {\n self.0.parent_element().map(XmlNode)\n }\n\n fn prev_sibling_element(&self) -> Option {\n self.0.prev_sibling_element().map(XmlNode)\n }\n\n fn has_local_name(&self, local_name: &str) -> bool {\n self.0.tag_name().name() == local_name\n }\n\n fn attribute_matches(&self, local_name: &str, operator: AttributeOperator<'_>) -> bool {\n match self.0.attribute(local_name) {\n Some(value) => operator.matches(value),\n None => false,\n }\n }\n\n fn pseudo_class_matches(&self, class: PseudoClass<'_>) -> bool {\n match class {\n PseudoClass::FirstChild => self.prev_sibling_element().is_none(),\n _ => false,\n }\n }\n}\n\nmacro_rules! match_single {\n ($doc:expr, $selector:expr) => {{\n let nodes = XmlNode($doc.root_element()).select($selector);\n assert_eq!(nodes.len(), 1);\n nodes[0].attribute(\"id\").unwrap()\n }};\n}\n\nmacro_rules! match_none {\n ($doc:expr, $selector:expr) => {{\n assert_eq!(XmlNode($doc.root_element()).select($selector).len(), 0);\n }};\n}\n\n#[test]\nfn select_01() {\n let doc = roxmltree::Document::parse(\"
\").unwrap();\n assert_eq!(match_single!(doc, \"*\"), \"div1\");\n}\n\n#[test]\nfn select_02() {\n let doc = roxmltree::Document::parse(\"
\").unwrap();\n assert_eq!(match_single!(doc, \"div\"), \"div1\");\n match_none!(doc, \"p\");\n}\n\n#[test]\nfn select_03() {\n let doc = roxmltree::Document::parse(\"
\").unwrap();\n assert_eq!(match_single!(doc, \"#div1\"), \"div1\");\n match_none!(doc, \"#d1\");\n}\n\n#[test]\nfn select_04() {\n let doc = roxmltree::Document::parse(\"
\").unwrap();\n match_none!(doc, \"p#div1\");\n}\n\n#[test]\nfn select_05() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"div p\"), \"p1\");\n}\n\n#[test]\nfn select_06() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n \n

\n \n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"div p\"), \"p1\");\n}\n\n#[test]\nfn select_07() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n
\n \n

\n \n

\n
\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"div p\"), \"p1\");\n}\n\n#[test]\nfn select_08() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n \n

\n

\n

\n \n
\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"div p\"), \"p1\");\n}\n\n#[test]\nfn select_09() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n \n

\n \n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"div g p\"), \"p1\");\n}\n\n#[test]\nfn select_10() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n \n

\n \n

\n\",\n )\n .unwrap();\n\n match_none!(doc, \"div g p\");\n}\n\n#[test]\nfn select_11() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n \n

\n \n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"div * p\"), \"p1\");\n}\n\n#[test]\nfn select_12() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n \n \n

\n
\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"div p *[color]\"), \"rect2\");\n assert_eq!(match_single!(doc, \"div p [color]\"), \"rect2\");\n}\n\n#[test]\nfn select_13() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"div > p\"), \"p1\");\n}\n\n#[test]\nfn select_14() {\n let doc = roxmltree::Document::parse(\n \"\\\n

\n\",\n )\n .unwrap();\n\n match_none!(doc, \"div > p\");\n}\n\n#[test]\nfn select_15() {\n let doc = roxmltree::Document::parse(\n \"\\\n

\n \n

\n \n

\n\",\n )\n .unwrap();\n\n match_none!(doc, \"div > p\");\n}\n\n#[test]\nfn select_16() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

    \n
  1. \n \n

    \n \n

    2. \n
\n

\n
\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"div ol>li p\"), \"p1\");\n}\n\n#[test]\nfn select_17() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

    \n \n
  1. \n \n

    \n \n

    2. \n     \n
\n

\n
\n\",\n )\n .unwrap();\n\n match_none!(doc, \"div ol>li p\");\n}\n\n#[test]\nfn select_18() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n \n

\n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"g + p\"), \"p1\");\n}\n\n#[test]\nfn select_19() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n \n \n

\n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"g + p\"), \"p1\");\n}\n\n#[test]\nfn select_20() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n \n

\n\",\n )\n .unwrap();\n\n match_none!(doc, \"g + p\");\n}\n\n#[test]\nfn select_21() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

\n\",\n )\n .unwrap();\n\n match_none!(doc, \"div + p\");\n}\n\n#[test]\nfn select_22() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"[id=p1]\"), \"p1\");\n}\n\n#[test]\nfn select_23() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"[class~=warn]\"), \"p1\");\n}\n\n#[test]\nfn select_24() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

\n\",\n )\n .unwrap();\n\n match_none!(doc, \"[class~='test warn']\");\n}\n\n#[test]\nfn select_25() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"[lang=en]\"), \"p1\");\n assert_eq!(match_single!(doc, \"[lang|=en]\"), \"p1\");\n}\n\n#[test]\nfn select_26() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"[lang='en-US']\"), \"p1\");\n assert_eq!(match_single!(doc, \"[lang|=en]\"), \"p1\");\n}\n\n#[test]\nfn select_27() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \".marine.pastoral\"), \"p1\");\n}\n\n#[test]\nfn select_28() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

\n\",\n )\n .unwrap();\n\n assert_eq!(match_single!(doc, \"p:first-child\"), \"p1\");\n}\n\n#[test]\nfn select_29() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n \n

\n

\n\",\n )\n .unwrap();\n\n match_none!(doc, \"p:first-child\");\n}\n\n#[test]\nfn select_30() {\n let doc = roxmltree::Document::parse(\n \"\\\n
\n

\n

\n

\n\",\n )\n .unwrap();\n\n let nodes = XmlNode(doc.root_element()).select(\":first-child\");\n assert_eq!(nodes.len(), 2);\n assert_eq!(nodes[0].attribute(\"id\").unwrap(), \"div1\");\n assert_eq!(nodes[1].attribute(\"id\").unwrap(), \"p1\");\n}\n\n#[test]\nfn to_string() {\n let selectors = Selector::parse(\"a > b\").unwrap();\n assert_eq!(selectors.to_string(), \"a > b\");\n}\n" }, { "package": "simplecss@0.2.2", "sourcePath": "tests/selector_tokenizer.rs", "sha256": "45c32806e8857bd6cf560d4c787aa6fb872f40505159a2643a254c664ec85ec9", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the SimpleCSS Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Selector Tokenizer\n\nuse simplecss::*;\n\nmacro_rules! tokenize {\n ($name:ident, $text:expr, $( $token:expr ),*) => (\n #[test]\n fn $name() {\n let mut t = SelectorTokenizer::from($text);\n $(\n assert_eq!(t.next().unwrap().unwrap(), $token);\n )*\n\n assert!(t.next().is_none());\n }\n )\n}\n\ntokenize!(tokenize_01, \"*\", SelectorToken::UniversalSelector);\n\ntokenize!(tokenize_02, \"div\", SelectorToken::TypeSelector(\"div\"));\n\ntokenize!(tokenize_03, \"#div\", SelectorToken::IdSelector(\"div\"));\n\ntokenize!(tokenize_04, \".div\", SelectorToken::ClassSelector(\"div\"));\n\ntokenize!(\n tokenize_05,\n \"[id]\",\n SelectorToken::AttributeSelector(\"id\", AttributeOperator::Exists)\n);\n\ntokenize!(\n tokenize_06,\n \"[id=test]\",\n SelectorToken::AttributeSelector(\"id\", AttributeOperator::Matches(\"test\"))\n);\n\ntokenize!(\n tokenize_07,\n \"[id~=test]\",\n SelectorToken::AttributeSelector(\"id\", AttributeOperator::Contains(\"test\"))\n);\n\ntokenize!(\n tokenize_08,\n \"[id|=test]\",\n SelectorToken::AttributeSelector(\"id\", AttributeOperator::StartsWith(\"test\"))\n);\n\ntokenize!(\n tokenize_09,\n \"[id='test']\",\n SelectorToken::AttributeSelector(\"id\", AttributeOperator::Matches(\"test\"))\n);\n\ntokenize!(\n tokenize_10,\n \"[id=\\\"test\\\"]\",\n SelectorToken::AttributeSelector(\"id\", AttributeOperator::Matches(\"test\"))\n);\n\ntokenize!(\n tokenize_11,\n \"[id='te\\\\'st']\",\n SelectorToken::AttributeSelector(\"id\", AttributeOperator::Matches(\"te\\\\'st\"))\n);\n\ntokenize!(\n tokenize_12,\n \"[id=\\\"te\\\\\\\"st\\\"]\",\n SelectorToken::AttributeSelector(\"id\", AttributeOperator::Matches(\"te\\\\\\\"st\"))\n);\n\ntokenize!(\n tokenize_13,\n \"div:first-child\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::PseudoClass(\"first-child\")\n);\n\ntokenize!(\n tokenize_14,\n \":first-child\",\n SelectorToken::PseudoClass(\"first-child\")\n);\n\ntokenize!(\n tokenize_15,\n \"div p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::TypeSelector(\"p\")\n);\n\ntokenize!(\n tokenize_16,\n \"div p a\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::TypeSelector(\"p\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::TypeSelector(\"a\")\n);\n\ntokenize!(\n tokenize_17,\n \"div>p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::ChildCombinator,\n SelectorToken::TypeSelector(\"p\")\n);\n\ntokenize!(\n tokenize_18,\n \"div >p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::ChildCombinator,\n SelectorToken::TypeSelector(\"p\")\n);\n\ntokenize!(\n tokenize_19,\n \"div> p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::ChildCombinator,\n SelectorToken::TypeSelector(\"p\")\n);\n\ntokenize!(\n tokenize_20,\n \"div > p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::ChildCombinator,\n SelectorToken::TypeSelector(\"p\")\n);\n\ntokenize!(\n tokenize_21,\n \"div .p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::ClassSelector(\"p\")\n);\n\ntokenize!(\n tokenize_22,\n \"div *\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::UniversalSelector\n);\n\ntokenize!(\n tokenize_23,\n \"div #p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::IdSelector(\"p\")\n);\n\ntokenize!(\n tokenize_24,\n \"div [id]\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::AttributeSelector(\"id\", AttributeOperator::Exists)\n);\n\ntokenize!(\n tokenize_25,\n \"div :link\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::PseudoClass(\"link\")\n);\n\ntokenize!(\n tokenize_26,\n \"div+p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::AdjacentCombinator,\n SelectorToken::TypeSelector(\"p\")\n);\n\ntokenize!(\n tokenize_27,\n \"div +p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::AdjacentCombinator,\n SelectorToken::TypeSelector(\"p\")\n);\n\ntokenize!(\n tokenize_28,\n \"div+ p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::AdjacentCombinator,\n SelectorToken::TypeSelector(\"p\")\n);\n\ntokenize!(\n tokenize_29,\n \"div + p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::AdjacentCombinator,\n SelectorToken::TypeSelector(\"p\")\n);\n\ntokenize!(tokenize_30, \"div {\", SelectorToken::TypeSelector(\"div\"));\n\ntokenize!(tokenize_31, \"div,\", SelectorToken::TypeSelector(\"div\"));\n\ntokenize!(tokenize_32, \"div{\", SelectorToken::TypeSelector(\"div\"));\n\ntokenize!(tokenize_33, \"div ,\", SelectorToken::TypeSelector(\"div\"));\n\ntokenize!(\n tokenize_34,\n \"div.test\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::ClassSelector(\"test\")\n);\n\ntokenize!(\n tokenize_35,\n \"div.test.warn\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::ClassSelector(\"test\"),\n SelectorToken::ClassSelector(\"warn\")\n);\n\ntokenize!(\n tokenize_36,\n \"div#id\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::IdSelector(\"id\")\n);\n\ntokenize!(\n tokenize_37,\n \"*[id]\",\n SelectorToken::UniversalSelector,\n SelectorToken::AttributeSelector(\"id\", AttributeOperator::Exists)\n);\n\ntokenize!(\n tokenize_38,\n \"*.test\",\n SelectorToken::UniversalSelector,\n SelectorToken::ClassSelector(\"test\")\n);\n\ntokenize!(\n tokenize_39,\n \"*#id\",\n SelectorToken::UniversalSelector,\n SelectorToken::IdSelector(\"id\")\n);\n\ntokenize!(\n tokenize_40,\n \"div * p\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::UniversalSelector,\n SelectorToken::DescendantCombinator,\n SelectorToken::TypeSelector(\"p\")\n);\n\ntokenize!(\n tokenize_41,\n \"div[id=test][color=red]\",\n SelectorToken::TypeSelector(\"div\"),\n SelectorToken::AttributeSelector(\"id\", AttributeOperator::Matches(\"test\")),\n SelectorToken::AttributeSelector(\"color\", AttributeOperator::Matches(\"red\"))\n);\n\ntokenize!(\n tokenize_42,\n \"a.external:visited\",\n SelectorToken::TypeSelector(\"a\"),\n SelectorToken::ClassSelector(\"external\"),\n SelectorToken::PseudoClass(\"visited\")\n);\n\ntokenize!(\n tokenize_43,\n \":lang(en)\",\n SelectorToken::LangPseudoClass(\"en\")\n);\n\ntokenize!(\n tokenize_44,\n \"a\\nb\",\n SelectorToken::TypeSelector(\"a\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::TypeSelector(\"b\")\n);\n\ntokenize!(\n tokenize_45,\n \".warn :first-child\",\n SelectorToken::ClassSelector(\"warn\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::PseudoClass(\"first-child\")\n);\n\nmacro_rules! malformed {\n ($name:ident, $text:expr, $err_str:expr) => {\n #[test]\n fn $name() {\n for token in SelectorTokenizer::from($text) {\n match token {\n Ok(_) => {}\n Err(e) => {\n assert_eq!(e.to_string(), $err_str);\n return;\n }\n }\n }\n\n unreachable!()\n }\n };\n}\n\nmalformed!(malformed_01, \">\", \"unexpected combinator\");\n\nmalformed!(malformed_02, \"+\", \"unexpected combinator\");\n\nmalformed!(malformed_03, \"> a\", \"unexpected combinator\");\n\nmalformed!(malformed_04, \"a >\", \"selector missing\");\n\nmalformed!(malformed_05, \"*a\", \"unexpected selector\");\n\nmalformed!(malformed_06, \"a*\", \"unexpected selector\");\n\nmalformed!(malformed_07, \"a > ,\", \"selector missing\");\n\nmalformed!(malformed_08, \"a > >\", \"unexpected combinator\");\n\nmalformed!(malformed_09, \"a > {\", \"selector missing\");\n\nmalformed!(malformed_10, \"a/**/b\", \"unexpected selector\");\n\nmalformed!(malformed_11, \"a < b\", \"invalid ident at 1:3\");\n\nmalformed!(malformed_12, \":lang()\", \"invalid language pseudo-class\");\n\nmalformed!(malformed_13, \":lang( )\", \"invalid language pseudo-class\");\n\nmalformed!(malformed_14, \"::first-child\", \"invalid ident at 1:2\");\n\nmalformed!(\n malformed_15,\n \"[olor:red\",\n \"invalid or unsupported attribute selector\"\n);\n\nmalformed!(malformed_16, \"\", \"selector missing\");\n\nmalformed!(malformed_17, \" \", \"selector missing\");\n\nmalformed!(malformed_18, \"/**/\", \"selector missing\");\n\ntokenize!(comment_01, \"/**/a\", SelectorToken::TypeSelector(\"a\"));\n\ntokenize!(comment_02, \"/* */a\", SelectorToken::TypeSelector(\"a\"));\n\ntokenize!(\n comment_03,\n \"/* comment */a\",\n SelectorToken::TypeSelector(\"a\")\n);\n\ntokenize!(comment_04, \"/**/ /**/a\", SelectorToken::TypeSelector(\"a\"));\n\ntokenize!(comment_05, \"/**/ a /**/\", SelectorToken::TypeSelector(\"a\"));\n\ntokenize!(\n comment_06,\n \"a /**/ b\",\n SelectorToken::TypeSelector(\"a\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::TypeSelector(\"b\")\n);\n\ntokenize!(\n comment_08,\n \"a /**/b\",\n SelectorToken::TypeSelector(\"a\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::TypeSelector(\"b\")\n);\n\ntokenize!(\n comment_09,\n \"a/**/ b\",\n SelectorToken::TypeSelector(\"a\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::TypeSelector(\"b\")\n);\n\ntokenize!(\n comment_10,\n \"a/**/ /**/b\",\n SelectorToken::TypeSelector(\"a\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::TypeSelector(\"b\")\n);\n\ntokenize!(\n comment_11,\n \"a /**/ /**/ b\",\n SelectorToken::TypeSelector(\"a\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::TypeSelector(\"b\")\n);\n\ntokenize!(\n comment_12,\n \"a /**//**/ b\",\n SelectorToken::TypeSelector(\"a\"),\n SelectorToken::DescendantCombinator,\n SelectorToken::TypeSelector(\"b\")\n);\n" }, { "package": "simplecss@0.2.2", "sourcePath": "tests/specificity.rs", "sha256": "f3d2c6532238602d616073c321313013fc1965af8df79b8cf6d49a7ad1073378", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the SimpleCSS Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Specificity\n\nuse simplecss::*;\n\n#[test]\nfn spec_01() {\n let selectors = Selector::parse(\"*\").unwrap();\n assert_eq!(selectors.specificity(), [0, 0, 0]);\n}\n\n#[test]\nfn spec_02() {\n let selectors = Selector::parse(\"li\").unwrap();\n assert_eq!(selectors.specificity(), [0, 0, 1]);\n}\n\n#[test]\nfn spec_03() {\n let selectors = Selector::parse(\"ul li\").unwrap();\n assert_eq!(selectors.specificity(), [0, 0, 2]);\n}\n\n#[test]\nfn spec_04() {\n let selectors = Selector::parse(\"ul ol + li\").unwrap();\n assert_eq!(selectors.specificity(), [0, 0, 3]);\n}\n\n#[test]\nfn spec_05() {\n let selectors = Selector::parse(\"h1 + *[rel=up]\").unwrap();\n assert_eq!(selectors.specificity(), [0, 1, 1]);\n}\n\n#[test]\nfn spec_06() {\n let selectors = Selector::parse(\"ul ol li.red\").unwrap();\n assert_eq!(selectors.specificity(), [0, 1, 3]);\n}\n\n#[test]\nfn spec_07() {\n let selectors = Selector::parse(\"li.red.level\").unwrap();\n assert_eq!(selectors.specificity(), [0, 2, 1]);\n}\n\n#[test]\nfn spec_08() {\n let selectors = Selector::parse(\"#x34y\").unwrap();\n assert_eq!(selectors.specificity(), [1, 0, 0]);\n}\n" }, { "package": "simplecss@0.2.2", "sourcePath": "tests/stylesheet.rs", "sha256": "ebef9d5edef430fa44032e86a276b9f4d06a5eb3b25a47b7acb1e03a68218429", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the SimpleCSS Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Stylesheet\n\nuse simplecss::*;\n\n#[test]\nfn style_01() {\n let style = StyleSheet::parse(\"\");\n assert_eq!(style.to_string(), \"\");\n}\n\n#[test]\nfn style_02() {\n let style = StyleSheet::parse(\"a {}\");\n assert_eq!(style.to_string(), \"\");\n}\n\n#[test]\nfn style_03() {\n let style = StyleSheet::parse(\"a { color:red }\");\n assert_eq!(style.to_string(), \"a { color:red; }\");\n}\n\n#[test]\nfn style_04() {\n let style = StyleSheet::parse(\"/**/\");\n assert_eq!(style.to_string(), \"\");\n}\n\n#[test]\nfn style_05() {\n let style = StyleSheet::parse(\"a { color:red } /**/\");\n assert_eq!(style.to_string(), \"a { color:red; }\");\n}\n\n#[test]\nfn style_06() {\n let style = StyleSheet::parse(\"a, b { color:red }\");\n assert_eq!(style.to_string(), \"a { color:red; }\\nb { color:red; }\");\n}\n\n#[test]\nfn style_07() {\n let style = StyleSheet::parse(\"a, { color:red }\");\n assert_eq!(style.to_string(), \"a { color:red; }\");\n}\n\n#[test]\nfn style_08() {\n let style = StyleSheet::parse(\"a,, { color:red }\");\n assert_eq!(style.to_string(), \"a { color:red; }\");\n}\n\n#[test]\nfn style_09() {\n let style = StyleSheet::parse(\"a,,b { color:red }\");\n assert_eq!(style.to_string(), \"a { color:red; }\\nb { color:red; }\");\n}\n\n#[test]\nfn style_10() {\n let style = StyleSheet::parse(\",a { color:red }\");\n assert_eq!(style.to_string(), \"a { color:red; }\");\n}\n\n#[test]\nfn style_11() {\n let style = StyleSheet::parse(\"@import \\\"subs.css\\\";\\na { color:red }\");\n assert_eq!(style.to_string(), \"a { color:red; }\");\n}\n\n#[test]\nfn style_12() {\n let style = StyleSheet::parse(\n \"\\\n@media screen {\n p:before { content: 'Hello'; }\n}\na { color:red }\",\n );\n assert_eq!(style.to_string(), \"a { color:red; }\");\n}\n\n#[test]\nfn style_13() {\n let style = StyleSheet::parse(\"a > { color:red }\");\n assert_eq!(style.to_string(), \"\");\n}\n\n#[test]\nfn style_14() {\n let style = StyleSheet::parse(\"p { color:green; color }\");\n assert_eq!(style.to_string(), \"p { color:green; }\");\n}\n\n#[test]\nfn style_15() {\n let style = StyleSheet::parse(\"p { color; color:green }\");\n assert_eq!(style.to_string(), \"\"); // TODO: should be 'p { color:green; }'\n}\n\n#[test]\nfn style_16() {\n let style = StyleSheet::parse(\"p { color:green; color: }\");\n assert_eq!(style.to_string(), \"p { color:green; }\");\n}\n\n#[test]\nfn style_17() {\n let style = StyleSheet::parse(\"p { color:green; color:; color:red; }\");\n assert_eq!(style.to_string(), \"p { color:green; }\");\n}\n\n#[test]\nfn style_18() {\n let style = StyleSheet::parse(\"p { color:green; color{;color:maroon} }\");\n assert_eq!(style.to_string(), \"p { color:green; }\");\n}\n\n#[test]\nfn style_19() {\n let style = StyleSheet::parse(\"p { color{;color:maroon} color:green; }\");\n assert_eq!(style.to_string(), \"\"); // TODO: should be 'p { color:green; }'\n}\n\n#[test]\nfn style_20() {\n let style = StyleSheet::parse(\n \"\\\n h1 { color: green }\n h2 & h3 { color: red }\n h4 { color: black }\n \",\n );\n assert_eq!(\n style.to_string(),\n \"h1 { color:green; }\\nh4 { color:black; }\"\n );\n}\n\n#[test]\nfn style_21() {\n let style = StyleSheet::parse(\":le>*\");\n assert_eq!(style.to_string(), \"\");\n}\n" }, { "package": "simplecss@0.2.2", "sourcePath": "tests/warnings.rs", "sha256": "b93bb7e1ac75575aeb19302de8570738576e7c4aae8b4ef9a8b6d0c5b9330a6f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2019 the SimpleCSS Authors\n// SPDX-License-Identifier: Apache-2.0 OR MIT\n\n//! Warnings\n\nfn run_process(input: &str) -> String {\n use std::io::Write;\n use std::process::Stdio;\n\n let mut child = std::process::Command::new(\"target/debug/examples/parse\")\n .arg(\"-\")\n .stdin(Stdio::piped())\n .stdout(Stdio::null())\n .stderr(Stdio::piped())\n .spawn()\n .unwrap();\n\n child\n .stdin\n .as_mut()\n .unwrap()\n .write_all(input.as_bytes())\n .unwrap();\n\n let output = child.wait_with_output().expect(\"Failed to read stdout\");\n String::from_utf8(output.stderr).unwrap()\n}\n\n#[test]\nfn style_01() {\n assert_eq!(\n run_process(\"> {}\"),\n \"WARN: Selector parsing failed cause unexpected combinator.\\n\"\n );\n}\n\n#[test]\nfn style_02() {\n assert_eq!(\n run_process(\"@import 'subs.css';\"),\n \"WARN: The @import rule is not supported. Skipped.\\n\"\n );\n}\n" }, { "package": "siphasher@0.3.11", "sourcePath": "src/sip.rs", "sha256": "b2f0067642100232f78f046167299972ec4e64bf8b9a48d5283ab9e0101e12da", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! An implementation of SipHash.\n\nuse core::cmp;\nuse core::hash;\nuse core::hash::Hasher as _;\nuse core::marker::PhantomData;\nuse core::mem;\nuse core::ptr;\nuse core::u64;\n\n/// An implementation of SipHash 1-3.\n///\n/// See: \n#[derive(Debug, Clone, Copy, Default)]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct SipHasher13 {\n hasher: Hasher,\n}\n\n/// An implementation of SipHash 2-4.\n///\n/// See: \n#[derive(Debug, Clone, Copy, Default)]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct SipHasher24 {\n hasher: Hasher,\n}\n\n/// An implementation of SipHash 2-4.\n///\n/// See: \n///\n/// SipHash is a general-purpose hashing function: it runs at a good\n/// speed (competitive with Spooky and City) and permits strong _keyed_\n/// hashing. This lets you key your hashtables from a strong RNG, such as\n/// [`rand::os::OsRng`](https://doc.rust-lang.org/rand/rand/os/struct.OsRng.html).\n///\n/// Although the SipHash algorithm is considered to be generally strong,\n/// it is not intended for cryptographic purposes. As such, all\n/// cryptographic uses of this implementation are _strongly discouraged_.\n#[derive(Debug, Clone, Copy, Default)]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct SipHasher(SipHasher24);\n\n#[derive(Debug, Clone, Copy)]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\nstruct Hasher {\n k0: u64,\n k1: u64,\n length: usize, // how many bytes we've processed\n state: State, // hash State\n tail: u64, // unprocessed bytes le\n ntail: usize, // how many bytes in tail are valid\n _marker: PhantomData,\n}\n\n#[derive(Debug, Clone, Copy)]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\nstruct State {\n // v0, v2 and v1, v3 show up in pairs in the algorithm,\n // and simd implementations of SipHash will use vectors\n // of v02 and v13. By placing them in this order in the struct,\n // the compiler can pick up on just a few simd optimizations by itself.\n v0: u64,\n v2: u64,\n v1: u64,\n v3: u64,\n}\n\nmacro_rules! compress {\n ($state:expr) => {{\n compress!($state.v0, $state.v1, $state.v2, $state.v3)\n }};\n ($v0:expr, $v1:expr, $v2:expr, $v3:expr) => {{\n $v0 = $v0.wrapping_add($v1);\n $v1 = $v1.rotate_left(13);\n $v1 ^= $v0;\n $v0 = $v0.rotate_left(32);\n $v2 = $v2.wrapping_add($v3);\n $v3 = $v3.rotate_left(16);\n $v3 ^= $v2;\n $v0 = $v0.wrapping_add($v3);\n $v3 = $v3.rotate_left(21);\n $v3 ^= $v0;\n $v2 = $v2.wrapping_add($v1);\n $v1 = $v1.rotate_left(17);\n $v1 ^= $v2;\n $v2 = $v2.rotate_left(32);\n }};\n}\n\n/// Loads an integer of the desired type from a byte stream, in LE order. Uses\n/// `copy_nonoverlapping` to let the compiler generate the most efficient way\n/// to load it from a possibly unaligned address.\n///\n/// Unsafe because: unchecked indexing at `i..i+size_of(int_ty)`\nmacro_rules! load_int_le {\n ($buf:expr, $i:expr, $int_ty:ident) => {{\n debug_assert!($i + mem::size_of::<$int_ty>() <= $buf.len());\n let mut data = 0 as $int_ty;\n ptr::copy_nonoverlapping(\n $buf.as_ptr().add($i),\n &mut data as *mut _ as *mut u8,\n mem::size_of::<$int_ty>(),\n );\n data.to_le()\n }};\n}\n\n/// Loads a u64 using up to 7 bytes of a byte slice. It looks clumsy but the\n/// `copy_nonoverlapping` calls that occur (via `load_int_le!`) all have fixed\n/// sizes and avoid calling `memcpy`, which is good for speed.\n///\n/// Unsafe because: unchecked indexing at start..start+len\n#[inline]\nunsafe fn u8to64_le(buf: &[u8], start: usize, len: usize) -> u64 {\n debug_assert!(len < 8);\n let mut i = 0; // current byte index (from LSB) in the output u64\n let mut out = 0;\n if i + 3 < len {\n out = load_int_le!(buf, start + i, u32) as u64;\n i += 4;\n }\n if i + 1 < len {\n out |= (load_int_le!(buf, start + i, u16) as u64) << (i * 8);\n i += 2\n }\n if i < len {\n out |= (*buf.get_unchecked(start + i) as u64) << (i * 8);\n i += 1;\n }\n debug_assert_eq!(i, len);\n out\n}\n\nimpl SipHasher {\n /// Creates a new `SipHasher` with the two initial keys set to 0.\n #[inline]\n pub fn new() -> SipHasher {\n SipHasher::new_with_keys(0, 0)\n }\n\n /// Creates a `SipHasher` that is keyed off the provided keys.\n #[inline]\n pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher {\n SipHasher(SipHasher24::new_with_keys(key0, key1))\n }\n\n /// Creates a `SipHasher` from a 16 byte key.\n pub fn new_with_key(key: &[u8; 16]) -> SipHasher {\n let mut b0 = [0u8; 8];\n let mut b1 = [0u8; 8];\n b0.copy_from_slice(&key[0..8]);\n b1.copy_from_slice(&key[8..16]);\n let key0 = u64::from_le_bytes(b0);\n let key1 = u64::from_le_bytes(b1);\n Self::new_with_keys(key0, key1)\n }\n\n /// Get the keys used by this hasher\n pub fn keys(&self) -> (u64, u64) {\n (self.0.hasher.k0, self.0.hasher.k1)\n }\n\n /// Get the key used by this hasher as a 16 byte vector\n pub fn key(&self) -> [u8; 16] {\n let mut bytes = [0u8; 16];\n bytes[0..8].copy_from_slice(&self.0.hasher.k0.to_le_bytes());\n bytes[8..16].copy_from_slice(&self.0.hasher.k1.to_le_bytes());\n bytes\n }\n\n /// Hash a byte array - This is the easiest and safest way to use SipHash.\n #[inline]\n pub fn hash(&self, bytes: &[u8]) -> u64 {\n let mut hasher = self.0.hasher;\n hasher.write(bytes);\n hasher.finish()\n }\n}\n\nimpl SipHasher13 {\n /// Creates a new `SipHasher13` with the two initial keys set to 0.\n #[inline]\n pub fn new() -> SipHasher13 {\n SipHasher13::new_with_keys(0, 0)\n }\n\n /// Creates a `SipHasher13` that is keyed off the provided keys.\n #[inline]\n pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher13 {\n SipHasher13 {\n hasher: Hasher::new_with_keys(key0, key1),\n }\n }\n\n /// Creates a `SipHasher13` from a 16 byte key.\n pub fn new_with_key(key: &[u8; 16]) -> SipHasher13 {\n let mut b0 = [0u8; 8];\n let mut b1 = [0u8; 8];\n b0.copy_from_slice(&key[0..8]);\n b1.copy_from_slice(&key[8..16]);\n let key0 = u64::from_le_bytes(b0);\n let key1 = u64::from_le_bytes(b1);\n Self::new_with_keys(key0, key1)\n }\n\n /// Get the keys used by this hasher\n pub fn keys(&self) -> (u64, u64) {\n (self.hasher.k0, self.hasher.k1)\n }\n\n /// Get the key used by this hasher as a 16 byte vector\n pub fn key(&self) -> [u8; 16] {\n let mut bytes = [0u8; 16];\n bytes[0..8].copy_from_slice(&self.hasher.k0.to_le_bytes());\n bytes[8..16].copy_from_slice(&self.hasher.k1.to_le_bytes());\n bytes\n }\n\n /// Hash a byte array - This is the easiest and safest way to use SipHash.\n #[inline]\n pub fn hash(&self, bytes: &[u8]) -> u64 {\n let mut hasher = self.hasher;\n hasher.write(bytes);\n hasher.finish()\n }\n}\n\nimpl SipHasher24 {\n /// Creates a new `SipHasher24` with the two initial keys set to 0.\n #[inline]\n pub fn new() -> SipHasher24 {\n SipHasher24::new_with_keys(0, 0)\n }\n\n /// Creates a `SipHasher24` that is keyed off the provided keys.\n #[inline]\n pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher24 {\n SipHasher24 {\n hasher: Hasher::new_with_keys(key0, key1),\n }\n }\n\n /// Creates a `SipHasher24` from a 16 byte key.\n pub fn new_with_key(key: &[u8; 16]) -> SipHasher24 {\n let mut b0 = [0u8; 8];\n let mut b1 = [0u8; 8];\n b0.copy_from_slice(&key[0..8]);\n b1.copy_from_slice(&key[8..16]);\n let key0 = u64::from_le_bytes(b0);\n let key1 = u64::from_le_bytes(b1);\n Self::new_with_keys(key0, key1)\n }\n\n /// Get the keys used by this hasher\n pub fn keys(&self) -> (u64, u64) {\n (self.hasher.k0, self.hasher.k1)\n }\n\n /// Get the key used by this hasher as a 16 byte vector\n pub fn key(&self) -> [u8; 16] {\n let mut bytes = [0u8; 16];\n bytes[0..8].copy_from_slice(&self.hasher.k0.to_le_bytes());\n bytes[8..16].copy_from_slice(&self.hasher.k1.to_le_bytes());\n bytes\n }\n\n /// Hash a byte array - This is the easiest and safest way to use SipHash.\n #[inline]\n pub fn hash(&self, bytes: &[u8]) -> u64 {\n let mut hasher = self.hasher;\n hasher.write(bytes);\n hasher.finish()\n }\n}\n\nimpl Hasher {\n #[inline]\n fn new_with_keys(key0: u64, key1: u64) -> Hasher {\n let mut state = Hasher {\n k0: key0,\n k1: key1,\n length: 0,\n state: State {\n v0: 0,\n v1: 0,\n v2: 0,\n v3: 0,\n },\n tail: 0,\n ntail: 0,\n _marker: PhantomData,\n };\n state.reset();\n state\n }\n\n #[inline]\n fn reset(&mut self) {\n self.length = 0;\n self.state.v0 = self.k0 ^ 0x736f6d6570736575;\n self.state.v1 = self.k1 ^ 0x646f72616e646f6d;\n self.state.v2 = self.k0 ^ 0x6c7967656e657261;\n self.state.v3 = self.k1 ^ 0x7465646279746573;\n self.ntail = 0;\n }\n\n // A specialized write function for values with size <= 8.\n //\n // The hashing of multi-byte integers depends on endianness. E.g.:\n // - little-endian: `write_u32(0xDDCCBBAA)` == `write([0xAA, 0xBB, 0xCC, 0xDD])`\n // - big-endian: `write_u32(0xDDCCBBAA)` == `write([0xDD, 0xCC, 0xBB, 0xAA])`\n //\n // This function does the right thing for little-endian hardware. On\n // big-endian hardware `x` must be byte-swapped first to give the right\n // behaviour. After any byte-swapping, the input must be zero-extended to\n // 64-bits. The caller is responsible for the byte-swapping and\n // zero-extension.\n #[inline]\n fn short_write(&mut self, _x: T, x: u64) {\n let size = mem::size_of::();\n self.length += size;\n\n // The original number must be zero-extended, not sign-extended.\n debug_assert!(if size < 8 { x >> (8 * size) == 0 } else { true });\n\n // The number of bytes needed to fill `self.tail`.\n let needed = 8 - self.ntail;\n\n self.tail |= x << (8 * self.ntail);\n if size < needed {\n self.ntail += size;\n return;\n }\n\n // `self.tail` is full, process it.\n self.state.v3 ^= self.tail;\n S::c_rounds(&mut self.state);\n self.state.v0 ^= self.tail;\n\n self.ntail = size - needed;\n self.tail = if needed < 8 { x >> (8 * needed) } else { 0 };\n }\n}\n\nimpl hash::Hasher for SipHasher {\n #[inline]\n fn write(&mut self, msg: &[u8]) {\n self.0.write(msg)\n }\n\n #[inline]\n fn finish(&self) -> u64 {\n self.0.finish()\n }\n\n #[inline]\n fn write_usize(&mut self, i: usize) {\n self.0.write_usize(i);\n }\n\n #[inline]\n fn write_u8(&mut self, i: u8) {\n self.0.write_u8(i);\n }\n\n #[inline]\n fn write_u16(&mut self, i: u16) {\n self.0.write_u16(i);\n }\n\n #[inline]\n fn write_u32(&mut self, i: u32) {\n self.0.write_u32(i);\n }\n\n #[inline]\n fn write_u64(&mut self, i: u64) {\n self.0.write_u64(i);\n }\n}\n\nimpl hash::Hasher for SipHasher13 {\n #[inline]\n fn write(&mut self, msg: &[u8]) {\n self.hasher.write(msg)\n }\n\n #[inline]\n fn finish(&self) -> u64 {\n self.hasher.finish()\n }\n\n #[inline]\n fn write_usize(&mut self, i: usize) {\n self.hasher.write_usize(i);\n }\n\n #[inline]\n fn write_u8(&mut self, i: u8) {\n self.hasher.write_u8(i);\n }\n\n #[inline]\n fn write_u16(&mut self, i: u16) {\n self.hasher.write_u16(i);\n }\n\n #[inline]\n fn write_u32(&mut self, i: u32) {\n self.hasher.write_u32(i);\n }\n\n #[inline]\n fn write_u64(&mut self, i: u64) {\n self.hasher.write_u64(i);\n }\n}\n\nimpl hash::Hasher for SipHasher24 {\n #[inline]\n fn write(&mut self, msg: &[u8]) {\n self.hasher.write(msg)\n }\n\n #[inline]\n fn finish(&self) -> u64 {\n self.hasher.finish()\n }\n\n #[inline]\n fn write_usize(&mut self, i: usize) {\n self.hasher.write_usize(i);\n }\n\n #[inline]\n fn write_u8(&mut self, i: u8) {\n self.hasher.write_u8(i);\n }\n\n #[inline]\n fn write_u16(&mut self, i: u16) {\n self.hasher.write_u16(i);\n }\n\n #[inline]\n fn write_u32(&mut self, i: u32) {\n self.hasher.write_u32(i);\n }\n\n #[inline]\n fn write_u64(&mut self, i: u64) {\n self.hasher.write_u64(i);\n }\n}\n\nimpl hash::Hasher for Hasher {\n #[inline]\n fn write_usize(&mut self, i: usize) {\n self.short_write(i, i.to_le() as u64);\n }\n\n #[inline]\n fn write_u8(&mut self, i: u8) {\n self.short_write(i, i as u64);\n }\n\n #[inline]\n fn write_u32(&mut self, i: u32) {\n self.short_write(i, i.to_le() as u64);\n }\n\n #[inline]\n fn write_u64(&mut self, i: u64) {\n self.short_write(i, i.to_le());\n }\n\n #[inline]\n fn write(&mut self, msg: &[u8]) {\n let length = msg.len();\n self.length += length;\n\n let mut needed = 0;\n\n if self.ntail != 0 {\n needed = 8 - self.ntail;\n self.tail |= unsafe { u8to64_le(msg, 0, cmp::min(length, needed)) } << (8 * self.ntail);\n if length < needed {\n self.ntail += length;\n return;\n } else {\n self.state.v3 ^= self.tail;\n S::c_rounds(&mut self.state);\n self.state.v0 ^= self.tail;\n self.ntail = 0;\n }\n }\n\n // Buffered tail is now flushed, process new input.\n let len = length - needed;\n let left = len & 0x7;\n\n let mut i = needed;\n while i < len - left {\n let mi = unsafe { load_int_le!(msg, i, u64) };\n\n self.state.v3 ^= mi;\n S::c_rounds(&mut self.state);\n self.state.v0 ^= mi;\n\n i += 8;\n }\n\n self.tail = unsafe { u8to64_le(msg, i, left) };\n self.ntail = left;\n }\n\n #[inline]\n fn finish(&self) -> u64 {\n let mut state = self.state;\n\n let b: u64 = ((self.length as u64 & 0xff) << 56) | self.tail;\n\n state.v3 ^= b;\n S::c_rounds(&mut state);\n state.v0 ^= b;\n\n state.v2 ^= 0xff;\n S::d_rounds(&mut state);\n\n state.v0 ^ state.v1 ^ state.v2 ^ state.v3\n }\n}\n\nimpl Default for Hasher {\n /// Creates a `Hasher` with the two initial keys set to 0.\n #[inline]\n fn default() -> Hasher {\n Hasher::new_with_keys(0, 0)\n }\n}\n\n#[doc(hidden)]\ntrait Sip {\n fn c_rounds(_: &mut State);\n fn d_rounds(_: &mut State);\n}\n\n#[derive(Debug, Clone, Copy, Default)]\nstruct Sip13Rounds;\n\nimpl Sip for Sip13Rounds {\n #[inline]\n fn c_rounds(state: &mut State) {\n compress!(state);\n }\n\n #[inline]\n fn d_rounds(state: &mut State) {\n compress!(state);\n compress!(state);\n compress!(state);\n }\n}\n\n#[derive(Debug, Clone, Copy, Default)]\nstruct Sip24Rounds;\n\nimpl Sip for Sip24Rounds {\n #[inline]\n fn c_rounds(state: &mut State) {\n compress!(state);\n compress!(state);\n }\n\n #[inline]\n fn d_rounds(state: &mut State) {\n compress!(state);\n compress!(state);\n compress!(state);\n compress!(state);\n }\n}\n" }, { "package": "siphasher@0.3.11", "sourcePath": "src/sip128.rs", "sha256": "eccfff86293b02af3e22a087592afcea08995f54c551e9fe0ec087e13d9e6eea", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! An implementation of SipHash with a 128-bit output.\n\nuse core::cmp;\nuse core::hash;\nuse core::hash::Hasher as _;\nuse core::marker::PhantomData;\nuse core::mem;\nuse core::ptr;\nuse core::u64;\n\n/// A 128-bit (2x64) hash output\n#[derive(Debug, Clone, Copy, Default)]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct Hash128 {\n pub h1: u64,\n pub h2: u64,\n}\n\nimpl From for Hash128 {\n fn from(v: u128) -> Self {\n Hash128 {\n h1: v as u64,\n h2: (v >> 64) as u64,\n }\n }\n}\n\nimpl From for u128 {\n fn from(h: Hash128) -> u128 {\n (h.h1 as u128) | ((h.h2 as u128) << 64)\n }\n}\n\n/// An implementation of SipHash128 1-3.\n#[derive(Debug, Clone, Copy, Default)]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct SipHasher13 {\n hasher: Hasher,\n}\n\n/// An implementation of SipHash128 2-4.\n#[derive(Debug, Clone, Copy, Default)]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\npub struct SipHasher24 {\n hasher: Hasher,\n}\n\n/// An implementation of SipHash128 2-4.\n///\n/// SipHash is a general-purpose hashing function: it runs at a good\n/// speed (competitive with Spooky and City) and permits strong _keyed_\n/// hashing. This lets you key your hashtables from a strong RNG, such as\n/// [`rand::os::OsRng`](https://doc.rust-lang.org/rand/rand/os/struct.OsRng.html).\n///\n/// Although the SipHash algorithm is considered to be generally strong,\n/// it is not intended for cryptographic purposes. As such, all\n/// cryptographic uses of this implementation are _strongly discouraged_.\n#[derive(Debug, Clone, Copy, Default)]\npub struct SipHasher(SipHasher24);\n\n#[derive(Debug, Copy)]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\nstruct Hasher {\n k0: u64,\n k1: u64,\n length: usize, // how many bytes we've processed\n state: State, // hash State\n tail: u64, // unprocessed bytes le\n ntail: usize, // how many bytes in tail are valid\n _marker: PhantomData,\n}\n\n#[derive(Debug, Clone, Copy)]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\nstruct State {\n // v0, v2 and v1, v3 show up in pairs in the algorithm,\n // and simd implementations of SipHash will use vectors\n // of v02 and v13. By placing them in this order in the struct,\n // the compiler can pick up on just a few simd optimizations by itself.\n v0: u64,\n v2: u64,\n v1: u64,\n v3: u64,\n}\n\nmacro_rules! compress {\n ($state:expr) => {{\n compress!($state.v0, $state.v1, $state.v2, $state.v3)\n }};\n ($v0:expr, $v1:expr, $v2:expr, $v3:expr) => {{\n $v0 = $v0.wrapping_add($v1);\n $v1 = $v1.rotate_left(13);\n $v1 ^= $v0;\n $v0 = $v0.rotate_left(32);\n $v2 = $v2.wrapping_add($v3);\n $v3 = $v3.rotate_left(16);\n $v3 ^= $v2;\n $v0 = $v0.wrapping_add($v3);\n $v3 = $v3.rotate_left(21);\n $v3 ^= $v0;\n $v2 = $v2.wrapping_add($v1);\n $v1 = $v1.rotate_left(17);\n $v1 ^= $v2;\n $v2 = $v2.rotate_left(32);\n }};\n}\n\n/// Loads an integer of the desired type from a byte stream, in LE order. Uses\n/// `copy_nonoverlapping` to let the compiler generate the most efficient way\n/// to load it from a possibly unaligned address.\n///\n/// Unsafe because: unchecked indexing at `i..i+size_of(int_ty)`\nmacro_rules! load_int_le {\n ($buf:expr, $i:expr, $int_ty:ident) => {{\n debug_assert!($i + mem::size_of::<$int_ty>() <= $buf.len());\n let mut data = 0 as $int_ty;\n ptr::copy_nonoverlapping(\n $buf.as_ptr().add($i),\n &mut data as *mut _ as *mut u8,\n mem::size_of::<$int_ty>(),\n );\n data.to_le()\n }};\n}\n\n/// Loads a u64 using up to 7 bytes of a byte slice. It looks clumsy but the\n/// `copy_nonoverlapping` calls that occur (via `load_int_le!`) all have fixed\n/// sizes and avoid calling `memcpy`, which is good for speed.\n///\n/// Unsafe because: unchecked indexing at start..start+len\n#[inline]\nunsafe fn u8to64_le(buf: &[u8], start: usize, len: usize) -> u64 {\n debug_assert!(len < 8);\n let mut i = 0; // current byte index (from LSB) in the output u64\n let mut out = 0;\n if i + 3 < len {\n out = load_int_le!(buf, start + i, u32) as u64;\n i += 4;\n }\n if i + 1 < len {\n out |= (load_int_le!(buf, start + i, u16) as u64) << (i * 8);\n i += 2\n }\n if i < len {\n out |= (*buf.get_unchecked(start + i) as u64) << (i * 8);\n i += 1;\n }\n debug_assert_eq!(i, len);\n out\n}\n\npub trait Hasher128 {\n /// Return a 128-bit hash\n fn finish128(&self) -> Hash128;\n}\n\nimpl SipHasher {\n /// Creates a new `SipHasher` with the two initial keys set to 0.\n #[inline]\n pub fn new() -> SipHasher {\n SipHasher::new_with_keys(0, 0)\n }\n\n /// Creates a `SipHasher` that is keyed off the provided keys.\n #[inline]\n pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher {\n SipHasher(SipHasher24::new_with_keys(key0, key1))\n }\n\n /// Creates a `SipHasher` from a 16 byte key.\n pub fn new_with_key(key: &[u8; 16]) -> SipHasher {\n let mut b0 = [0u8; 8];\n let mut b1 = [0u8; 8];\n b0.copy_from_slice(&key[0..8]);\n b1.copy_from_slice(&key[8..16]);\n let key0 = u64::from_le_bytes(b0);\n let key1 = u64::from_le_bytes(b1);\n Self::new_with_keys(key0, key1)\n }\n\n /// Get the keys used by this hasher\n pub fn keys(&self) -> (u64, u64) {\n (self.0.hasher.k0, self.0.hasher.k1)\n }\n\n /// Get the key used by this hasher as a 16 byte vector\n pub fn key(&self) -> [u8; 16] {\n let mut bytes = [0u8; 16];\n bytes[0..8].copy_from_slice(&self.0.hasher.k0.to_le_bytes());\n bytes[8..16].copy_from_slice(&self.0.hasher.k1.to_le_bytes());\n bytes\n }\n\n /// Hash a byte array - This is the easiest and safest way to use SipHash.\n #[inline]\n pub fn hash(&self, bytes: &[u8]) -> Hash128 {\n let mut hasher = self.0.hasher;\n hasher.write(bytes);\n hasher.finish128()\n }\n}\n\nimpl Hasher128 for SipHasher {\n /// Return a 128-bit hash\n #[inline]\n fn finish128(&self) -> Hash128 {\n self.0.finish128()\n }\n}\n\nimpl SipHasher13 {\n /// Creates a new `SipHasher13` with the two initial keys set to 0.\n #[inline]\n pub fn new() -> SipHasher13 {\n SipHasher13::new_with_keys(0, 0)\n }\n\n /// Creates a `SipHasher13` that is keyed off the provided keys.\n #[inline]\n pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher13 {\n SipHasher13 {\n hasher: Hasher::new_with_keys(key0, key1),\n }\n }\n\n /// Creates a `SipHasher13` from a 16 byte key.\n pub fn new_with_key(key: &[u8; 16]) -> SipHasher13 {\n let mut b0 = [0u8; 8];\n let mut b1 = [0u8; 8];\n b0.copy_from_slice(&key[0..8]);\n b1.copy_from_slice(&key[8..16]);\n let key0 = u64::from_le_bytes(b0);\n let key1 = u64::from_le_bytes(b1);\n Self::new_with_keys(key0, key1)\n }\n\n /// Get the keys used by this hasher\n pub fn keys(&self) -> (u64, u64) {\n (self.hasher.k0, self.hasher.k1)\n }\n\n /// Get the key used by this hasher as a 16 byte vector\n pub fn key(&self) -> [u8; 16] {\n let mut bytes = [0u8; 16];\n bytes[0..8].copy_from_slice(&self.hasher.k0.to_le_bytes());\n bytes[8..16].copy_from_slice(&self.hasher.k1.to_le_bytes());\n bytes\n }\n\n /// Hash a byte array - This is the easiest and safest way to use SipHash.\n #[inline]\n pub fn hash(&self, bytes: &[u8]) -> Hash128 {\n let mut hasher = self.hasher;\n hasher.write(bytes);\n hasher.finish128()\n }\n}\n\nimpl Hasher128 for SipHasher13 {\n /// Return a 128-bit hash\n #[inline]\n fn finish128(&self) -> Hash128 {\n self.hasher.finish128()\n }\n}\n\nimpl SipHasher24 {\n /// Creates a new `SipHasher24` with the two initial keys set to 0.\n #[inline]\n pub fn new() -> SipHasher24 {\n SipHasher24::new_with_keys(0, 0)\n }\n\n /// Creates a `SipHasher24` that is keyed off the provided keys.\n #[inline]\n pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher24 {\n SipHasher24 {\n hasher: Hasher::new_with_keys(key0, key1),\n }\n }\n\n /// Creates a `SipHasher24` from a 16 byte key.\n pub fn new_with_key(key: &[u8; 16]) -> SipHasher24 {\n let mut b0 = [0u8; 8];\n let mut b1 = [0u8; 8];\n b0.copy_from_slice(&key[0..8]);\n b1.copy_from_slice(&key[8..16]);\n let key0 = u64::from_le_bytes(b0);\n let key1 = u64::from_le_bytes(b1);\n Self::new_with_keys(key0, key1)\n }\n\n /// Get the keys used by this hasher\n pub fn keys(&self) -> (u64, u64) {\n (self.hasher.k0, self.hasher.k1)\n }\n\n /// Get the key used by this hasher as a 16 byte vector\n pub fn key(&self) -> [u8; 16] {\n let mut bytes = [0u8; 16];\n bytes[0..8].copy_from_slice(&self.hasher.k0.to_le_bytes());\n bytes[8..16].copy_from_slice(&self.hasher.k1.to_le_bytes());\n bytes\n }\n\n /// Hash a byte array - This is the easiest and safest way to use SipHash.\n #[inline]\n pub fn hash(&self, bytes: &[u8]) -> Hash128 {\n let mut hasher = self.hasher;\n hasher.write(bytes);\n hasher.finish128()\n }\n}\n\nimpl Hasher128 for SipHasher24 {\n /// Return a 128-bit hash\n #[inline]\n fn finish128(&self) -> Hash128 {\n self.hasher.finish128()\n }\n}\n\nimpl Hasher {\n #[inline]\n fn new_with_keys(key0: u64, key1: u64) -> Hasher {\n let mut state = Hasher {\n k0: key0,\n k1: key1,\n length: 0,\n state: State {\n v0: 0,\n v1: 0xee,\n v2: 0,\n v3: 0,\n },\n tail: 0,\n ntail: 0,\n _marker: PhantomData,\n };\n state.reset();\n state\n }\n\n #[inline]\n fn reset(&mut self) {\n self.length = 0;\n self.state.v0 = self.k0 ^ 0x736f6d6570736575;\n self.state.v1 = self.k1 ^ 0x646f72616e646f83;\n self.state.v2 = self.k0 ^ 0x6c7967656e657261;\n self.state.v3 = self.k1 ^ 0x7465646279746573;\n self.ntail = 0;\n }\n\n // A specialized write function for values with size <= 8.\n //\n // The hashing of multi-byte integers depends on endianness. E.g.:\n // - little-endian: `write_u32(0xDDCCBBAA)` == `write([0xAA, 0xBB, 0xCC, 0xDD])`\n // - big-endian: `write_u32(0xDDCCBBAA)` == `write([0xDD, 0xCC, 0xBB, 0xAA])`\n //\n // This function does the right thing for little-endian hardware. On\n // big-endian hardware `x` must be byte-swapped first to give the right\n // behaviour. After any byte-swapping, the input must be zero-extended to\n // 64-bits. The caller is responsible for the byte-swapping and\n // zero-extension.\n #[inline]\n fn short_write(&mut self, _x: T, x: u64) {\n let size = mem::size_of::();\n self.length += size;\n\n // The original number must be zero-extended, not sign-extended.\n debug_assert!(if size < 8 { x >> (8 * size) == 0 } else { true });\n\n // The number of bytes needed to fill `self.tail`.\n let needed = 8 - self.ntail;\n\n self.tail |= x << (8 * self.ntail);\n if size < needed {\n self.ntail += size;\n return;\n }\n\n // `self.tail` is full, process it.\n self.state.v3 ^= self.tail;\n S::c_rounds(&mut self.state);\n self.state.v0 ^= self.tail;\n\n self.ntail = size - needed;\n self.tail = if needed < 8 { x >> (8 * needed) } else { 0 };\n }\n}\n\nimpl Hasher {\n #[inline]\n pub fn finish128(&self) -> Hash128 {\n let mut state = self.state;\n\n let b: u64 = ((self.length as u64 & 0xff) << 56) | self.tail;\n\n state.v3 ^= b;\n S::c_rounds(&mut state);\n state.v0 ^= b;\n\n state.v2 ^= 0xee;\n S::d_rounds(&mut state);\n let h1 = state.v0 ^ state.v1 ^ state.v2 ^ state.v3;\n\n state.v1 ^= 0xdd;\n S::d_rounds(&mut state);\n let h2 = state.v0 ^ state.v1 ^ state.v2 ^ state.v3;\n\n Hash128 { h1, h2 }\n }\n}\n\nimpl hash::Hasher for SipHasher {\n #[inline]\n fn write(&mut self, msg: &[u8]) {\n self.0.write(msg)\n }\n\n #[inline]\n fn finish(&self) -> u64 {\n self.0.finish()\n }\n\n #[inline]\n fn write_usize(&mut self, i: usize) {\n self.0.write_usize(i);\n }\n\n #[inline]\n fn write_u8(&mut self, i: u8) {\n self.0.write_u8(i);\n }\n\n #[inline]\n fn write_u16(&mut self, i: u16) {\n self.0.write_u16(i);\n }\n\n #[inline]\n fn write_u32(&mut self, i: u32) {\n self.0.write_u32(i);\n }\n\n #[inline]\n fn write_u64(&mut self, i: u64) {\n self.0.write_u64(i);\n }\n}\n\nimpl hash::Hasher for SipHasher13 {\n #[inline]\n fn write(&mut self, msg: &[u8]) {\n self.hasher.write(msg)\n }\n\n #[inline]\n fn finish(&self) -> u64 {\n self.hasher.finish()\n }\n\n #[inline]\n fn write_usize(&mut self, i: usize) {\n self.hasher.write_usize(i);\n }\n\n #[inline]\n fn write_u8(&mut self, i: u8) {\n self.hasher.write_u8(i);\n }\n\n #[inline]\n fn write_u16(&mut self, i: u16) {\n self.hasher.write_u16(i);\n }\n\n #[inline]\n fn write_u32(&mut self, i: u32) {\n self.hasher.write_u32(i);\n }\n\n #[inline]\n fn write_u64(&mut self, i: u64) {\n self.hasher.write_u64(i);\n }\n}\n\nimpl hash::Hasher for SipHasher24 {\n #[inline]\n fn write(&mut self, msg: &[u8]) {\n self.hasher.write(msg)\n }\n\n #[inline]\n fn finish(&self) -> u64 {\n self.hasher.finish()\n }\n\n #[inline]\n fn write_usize(&mut self, i: usize) {\n self.hasher.write_usize(i);\n }\n\n #[inline]\n fn write_u8(&mut self, i: u8) {\n self.hasher.write_u8(i);\n }\n\n #[inline]\n fn write_u16(&mut self, i: u16) {\n self.hasher.write_u16(i);\n }\n\n #[inline]\n fn write_u32(&mut self, i: u32) {\n self.hasher.write_u32(i);\n }\n\n #[inline]\n fn write_u64(&mut self, i: u64) {\n self.hasher.write_u64(i);\n }\n}\n\nimpl hash::Hasher for Hasher {\n #[inline]\n fn write_usize(&mut self, i: usize) {\n self.short_write(i, i.to_le() as u64);\n }\n\n #[inline]\n fn write_u8(&mut self, i: u8) {\n self.short_write(i, i as u64);\n }\n\n #[inline]\n fn write_u32(&mut self, i: u32) {\n self.short_write(i, i.to_le() as u64);\n }\n\n #[inline]\n fn write_u64(&mut self, i: u64) {\n self.short_write(i, i.to_le());\n }\n\n #[inline]\n fn write(&mut self, msg: &[u8]) {\n let length = msg.len();\n self.length += length;\n\n let mut needed = 0;\n\n if self.ntail != 0 {\n needed = 8 - self.ntail;\n self.tail |= unsafe { u8to64_le(msg, 0, cmp::min(length, needed)) } << (8 * self.ntail);\n if length < needed {\n self.ntail += length;\n return;\n } else {\n self.state.v3 ^= self.tail;\n S::c_rounds(&mut self.state);\n self.state.v0 ^= self.tail;\n self.ntail = 0;\n }\n }\n\n // Buffered tail is now flushed, process new input.\n let len = length - needed;\n let left = len & 0x7;\n\n let mut i = needed;\n while i < len - left {\n let mi = unsafe { load_int_le!(msg, i, u64) };\n\n self.state.v3 ^= mi;\n S::c_rounds(&mut self.state);\n self.state.v0 ^= mi;\n\n i += 8;\n }\n\n self.tail = unsafe { u8to64_le(msg, i, left) };\n self.ntail = left;\n }\n\n #[inline]\n fn finish(&self) -> u64 {\n self.finish128().h2\n }\n}\n\nimpl Clone for Hasher {\n #[inline]\n fn clone(&self) -> Hasher {\n Hasher {\n k0: self.k0,\n k1: self.k1,\n length: self.length,\n state: self.state,\n tail: self.tail,\n ntail: self.ntail,\n _marker: self._marker,\n }\n }\n}\n\nimpl Default for Hasher {\n /// Creates a `Hasher` with the two initial keys set to 0.\n #[inline]\n fn default() -> Hasher {\n Hasher::new_with_keys(0, 0)\n }\n}\n\n#[doc(hidden)]\ntrait Sip {\n fn c_rounds(_: &mut State);\n fn d_rounds(_: &mut State);\n}\n\n#[derive(Debug, Clone, Copy, Default)]\nstruct Sip13Rounds;\n\nimpl Sip for Sip13Rounds {\n #[inline]\n fn c_rounds(state: &mut State) {\n compress!(state);\n }\n\n #[inline]\n fn d_rounds(state: &mut State) {\n compress!(state);\n compress!(state);\n compress!(state);\n }\n}\n\n#[derive(Debug, Clone, Copy, Default)]\nstruct Sip24Rounds;\n\nimpl Sip for Sip24Rounds {\n #[inline]\n fn c_rounds(state: &mut State) {\n compress!(state);\n compress!(state);\n }\n\n #[inline]\n fn d_rounds(state: &mut State) {\n compress!(state);\n compress!(state);\n compress!(state);\n compress!(state);\n }\n}\n\nimpl Hash128 {\n /// Convert into a 16-bytes vector\n pub fn as_bytes(&self) -> [u8; 16] {\n let mut bytes = [0u8; 16];\n let h1 = self.h1.to_le();\n let h2 = self.h2.to_le();\n unsafe {\n ptr::copy_nonoverlapping(&h1 as *const _ as *const u8, bytes.as_mut_ptr(), 8);\n ptr::copy_nonoverlapping(&h2 as *const _ as *const u8, bytes.as_mut_ptr().add(8), 8);\n }\n bytes\n }\n\n /// Convert into a `u128`\n #[inline]\n pub fn as_u128(&self) -> u128 {\n let h1 = self.h1.to_le();\n let h2 = self.h2.to_le();\n h1 as u128 | ((h2 as u128) << 64)\n }\n\n /// Convert into `(u64, u64)`\n #[inline]\n pub fn as_u64(&self) -> (u64, u64) {\n let h1 = self.h1.to_le();\n let h2 = self.h2.to_le();\n (h1, h2)\n }\n}\n" }, { "package": "siphasher@0.3.11", "sourcePath": "src/tests.rs", "sha256": "99c299d1951aee4c70cd01bea8baabc151a7b316904e5d31de847807a671eaf5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse std::hash::{Hash, Hasher};\n\nuse super::sip::{SipHasher, SipHasher13, SipHasher24};\n\n// Hash just the bytes of the slice, without length prefix\nstruct Bytes<'a>(&'a [u8]);\n\nimpl<'a> Hash for Bytes<'a> {\n #[allow(unused_must_use)]\n fn hash(&self, state: &mut H) {\n let Bytes(v) = *self;\n state.write(v);\n }\n}\n\nmacro_rules! u8to64_le {\n ($buf:expr, $i:expr) => {\n $buf[0 + $i] as u64\n | ($buf[1 + $i] as u64) << 8\n | ($buf[2 + $i] as u64) << 16\n | ($buf[3 + $i] as u64) << 24\n | ($buf[4 + $i] as u64) << 32\n | ($buf[5 + $i] as u64) << 40\n | ($buf[6 + $i] as u64) << 48\n | ($buf[7 + $i] as u64) << 56\n };\n ($buf:expr, $i:expr, $len:expr) => {{\n let mut t = 0;\n let mut out = 0;\n while t < $len {\n out |= ($buf[t + $i] as u64) << t * 8;\n t += 1;\n }\n out\n }};\n}\n\nfn hash_with(mut st: H, x: &T) -> u64 {\n x.hash(&mut st);\n st.finish()\n}\n\nfn hash(x: &T) -> u64 {\n hash_with(SipHasher::new(), x)\n}\n\n#[test]\n#[allow(unused_must_use)]\nfn test_siphash_1_3() {\n let vecs: [[u8; 8]; 64] = [\n [0xdc, 0xc4, 0x0f, 0x05, 0x58, 0x01, 0xac, 0xab],\n [0x93, 0xca, 0x57, 0x7d, 0xf3, 0x9b, 0xf4, 0xc9],\n [0x4d, 0xd4, 0xc7, 0x4d, 0x02, 0x9b, 0xcb, 0x82],\n [0xfb, 0xf7, 0xdd, 0xe7, 0xb8, 0x0a, 0xf8, 0x8b],\n [0x28, 0x83, 0xd3, 0x88, 0x60, 0x57, 0x75, 0xcf],\n [0x67, 0x3b, 0x53, 0x49, 0x2f, 0xd5, 0xf9, 0xde],\n [0xa7, 0x22, 0x9f, 0xc5, 0x50, 0x2b, 0x0d, 0xc5],\n [0x40, 0x11, 0xb1, 0x9b, 0x98, 0x7d, 0x92, 0xd3],\n [0x8e, 0x9a, 0x29, 0x8d, 0x11, 0x95, 0x90, 0x36],\n [0xe4, 0x3d, 0x06, 0x6c, 0xb3, 0x8e, 0xa4, 0x25],\n [0x7f, 0x09, 0xff, 0x92, 0xee, 0x85, 0xde, 0x79],\n [0x52, 0xc3, 0x4d, 0xf9, 0xc1, 0x18, 0xc1, 0x70],\n [0xa2, 0xd9, 0xb4, 0x57, 0xb1, 0x84, 0xa3, 0x78],\n [0xa7, 0xff, 0x29, 0x12, 0x0c, 0x76, 0x6f, 0x30],\n [0x34, 0x5d, 0xf9, 0xc0, 0x11, 0xa1, 0x5a, 0x60],\n [0x56, 0x99, 0x51, 0x2a, 0x6d, 0xd8, 0x20, 0xd3],\n [0x66, 0x8b, 0x90, 0x7d, 0x1a, 0xdd, 0x4f, 0xcc],\n [0x0c, 0xd8, 0xdb, 0x63, 0x90, 0x68, 0xf2, 0x9c],\n [0x3e, 0xe6, 0x73, 0xb4, 0x9c, 0x38, 0xfc, 0x8f],\n [0x1c, 0x7d, 0x29, 0x8d, 0xe5, 0x9d, 0x1f, 0xf2],\n [0x40, 0xe0, 0xcc, 0xa6, 0x46, 0x2f, 0xdc, 0xc0],\n [0x44, 0xf8, 0x45, 0x2b, 0xfe, 0xab, 0x92, 0xb9],\n [0x2e, 0x87, 0x20, 0xa3, 0x9b, 0x7b, 0xfe, 0x7f],\n [0x23, 0xc1, 0xe6, 0xda, 0x7f, 0x0e, 0x5a, 0x52],\n [0x8c, 0x9c, 0x34, 0x67, 0xb2, 0xae, 0x64, 0xf4],\n [0x79, 0x09, 0x5b, 0x70, 0x28, 0x59, 0xcd, 0x45],\n [0xa5, 0x13, 0x99, 0xca, 0xe3, 0x35, 0x3e, 0x3a],\n [0x35, 0x3b, 0xde, 0x4a, 0x4e, 0xc7, 0x1d, 0xa9],\n [0x0d, 0xd0, 0x6c, 0xef, 0x02, 0xed, 0x0b, 0xfb],\n [0xf4, 0xe1, 0xb1, 0x4a, 0xb4, 0x3c, 0xd9, 0x88],\n [0x63, 0xe6, 0xc5, 0x43, 0xd6, 0x11, 0x0f, 0x54],\n [0xbc, 0xd1, 0x21, 0x8c, 0x1f, 0xdd, 0x70, 0x23],\n [0x0d, 0xb6, 0xa7, 0x16, 0x6c, 0x7b, 0x15, 0x81],\n [0xbf, 0xf9, 0x8f, 0x7a, 0xe5, 0xb9, 0x54, 0x4d],\n [0x3e, 0x75, 0x2a, 0x1f, 0x78, 0x12, 0x9f, 0x75],\n [0x91, 0x6b, 0x18, 0xbf, 0xbe, 0xa3, 0xa1, 0xce],\n [0x06, 0x62, 0xa2, 0xad, 0xd3, 0x08, 0xf5, 0x2c],\n [0x57, 0x30, 0xc3, 0xa3, 0x2d, 0x1c, 0x10, 0xb6],\n [0xa1, 0x36, 0x3a, 0xae, 0x96, 0x74, 0xf4, 0xb3],\n [0x92, 0x83, 0x10, 0x7b, 0x54, 0x57, 0x6b, 0x62],\n [0x31, 0x15, 0xe4, 0x99, 0x32, 0x36, 0xd2, 0xc1],\n [0x44, 0xd9, 0x1a, 0x3f, 0x92, 0xc1, 0x7c, 0x66],\n [0x25, 0x88, 0x13, 0xc8, 0xfe, 0x4f, 0x70, 0x65],\n [0xa6, 0x49, 0x89, 0xc2, 0xd1, 0x80, 0xf2, 0x24],\n [0x6b, 0x87, 0xf8, 0xfa, 0xed, 0x1c, 0xca, 0xc2],\n [0x96, 0x21, 0x04, 0x9f, 0xfc, 0x4b, 0x16, 0xc2],\n [0x23, 0xd6, 0xb1, 0x68, 0x93, 0x9c, 0x6e, 0xa1],\n [0xfd, 0x14, 0x51, 0x8b, 0x9c, 0x16, 0xfb, 0x49],\n [0x46, 0x4c, 0x07, 0xdf, 0xf8, 0x43, 0x31, 0x9f],\n [0xb3, 0x86, 0xcc, 0x12, 0x24, 0xaf, 0xfd, 0xc6],\n [0x8f, 0x09, 0x52, 0x0a, 0xd1, 0x49, 0xaf, 0x7e],\n [0x9a, 0x2f, 0x29, 0x9d, 0x55, 0x13, 0xf3, 0x1c],\n [0x12, 0x1f, 0xf4, 0xa2, 0xdd, 0x30, 0x4a, 0xc4],\n [0xd0, 0x1e, 0xa7, 0x43, 0x89, 0xe9, 0xfa, 0x36],\n [0xe6, 0xbc, 0xf0, 0x73, 0x4c, 0xb3, 0x8f, 0x31],\n [0x80, 0xe9, 0xa7, 0x70, 0x36, 0xbf, 0x7a, 0xa2],\n [0x75, 0x6d, 0x3c, 0x24, 0xdb, 0xc0, 0xbc, 0xb4],\n [0x13, 0x15, 0xb7, 0xfd, 0x52, 0xd8, 0xf8, 0x23],\n [0x08, 0x8a, 0x7d, 0xa6, 0x4d, 0x5f, 0x03, 0x8f],\n [0x48, 0xf1, 0xe8, 0xb7, 0xe5, 0xd0, 0x9c, 0xd8],\n [0xee, 0x44, 0xa6, 0xf7, 0xbc, 0xe6, 0xf4, 0xf6],\n [0xf2, 0x37, 0x18, 0x0f, 0xd8, 0x9a, 0xc5, 0xae],\n [0xe0, 0x94, 0x66, 0x4b, 0x15, 0xf6, 0xb2, 0xc3],\n [0xa8, 0xb3, 0xbb, 0xb7, 0x62, 0x90, 0x19, 0x9d],\n ];\n\n let k0 = 0x_07_06_05_04_03_02_01_00;\n let k1 = 0x_0f_0e_0d_0c_0b_0a_09_08;\n let mut buf = Vec::new();\n let mut t = 0;\n let mut state_inc = SipHasher13::new_with_keys(k0, k1);\n\n while t < 64 {\n let vec = u8to64_le!(vecs[t], 0);\n let out = hash_with(SipHasher13::new_with_keys(k0, k1), &Bytes(&buf));\n assert_eq!(vec, out);\n\n let full = hash_with(SipHasher13::new_with_keys(k0, k1), &Bytes(&buf));\n let i = state_inc.finish();\n\n assert_eq!(full, i);\n assert_eq!(full, vec);\n\n buf.push(t as u8);\n Hasher::write(&mut state_inc, &[t as u8]);\n\n t += 1;\n }\n}\n\n#[test]\n#[allow(unused_must_use)]\nfn test_siphash_2_4() {\n let vecs: [[u8; 8]; 64] = [\n [0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72],\n [0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74],\n [0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d],\n [0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85],\n [0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf],\n [0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18],\n [0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb],\n [0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab],\n [0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93],\n [0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e],\n [0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a],\n [0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4],\n [0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75],\n [0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14],\n [0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7],\n [0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1],\n [0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f],\n [0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69],\n [0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b],\n [0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb],\n [0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe],\n [0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0],\n [0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93],\n [0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8],\n [0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8],\n [0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc],\n [0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17],\n [0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f],\n [0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde],\n [0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6],\n [0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad],\n [0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32],\n [0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71],\n [0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7],\n [0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12],\n [0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15],\n [0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31],\n [0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02],\n [0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca],\n [0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a],\n [0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e],\n [0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad],\n [0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18],\n [0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4],\n [0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9],\n [0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9],\n [0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb],\n [0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0],\n [0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6],\n [0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7],\n [0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee],\n [0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1],\n [0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a],\n [0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81],\n [0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f],\n [0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24],\n [0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7],\n [0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea],\n [0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60],\n [0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66],\n [0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c],\n [0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f],\n [0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5],\n [0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95],\n ];\n\n let k0 = 0x_07_06_05_04_03_02_01_00;\n let k1 = 0x_0f_0e_0d_0c_0b_0a_09_08;\n let mut buf = Vec::new();\n let mut t = 0;\n let mut state_inc = SipHasher24::new_with_keys(k0, k1);\n\n while t < 64 {\n let vec = u8to64_le!(vecs[t], 0);\n let out = hash_with(SipHasher24::new_with_keys(k0, k1), &Bytes(&buf));\n assert_eq!(vec, out);\n\n let full = hash_with(SipHasher24::new_with_keys(k0, k1), &Bytes(&buf));\n let i = state_inc.finish();\n\n assert_eq!(full, i);\n assert_eq!(full, vec);\n\n buf.push(t as u8);\n Hasher::write(&mut state_inc, &[t as u8]);\n\n t += 1;\n }\n}\n#[test]\nfn test_hash_idempotent() {\n let val64 = 0xdead_beef_dead_beef_u64;\n assert_eq!(hash(&val64), hash(&val64));\n let val32 = 0xdeadbeef_u32;\n assert_eq!(hash(&val32), hash(&val32));\n}\n\n#[test]\nfn test_hash_no_bytes_dropped_64() {\n let val = 0xdead_beef_dead_beef_u64;\n\n assert_ne!(hash(&val), hash(&zero_byte(val, 0)));\n assert_ne!(hash(&val), hash(&zero_byte(val, 1)));\n assert_ne!(hash(&val), hash(&zero_byte(val, 2)));\n assert_ne!(hash(&val), hash(&zero_byte(val, 3)));\n assert_ne!(hash(&val), hash(&zero_byte(val, 4)));\n assert_ne!(hash(&val), hash(&zero_byte(val, 5)));\n assert_ne!(hash(&val), hash(&zero_byte(val, 6)));\n assert_ne!(hash(&val), hash(&zero_byte(val, 7)));\n\n fn zero_byte(val: u64, byte: usize) -> u64 {\n assert!(byte < 8);\n val & !(0xff << (byte * 8))\n }\n}\n\n#[test]\nfn test_hash_no_bytes_dropped_32() {\n let val = 0xdeadbeef_u32;\n\n assert_ne!(hash(&val), hash(&zero_byte(val, 0)));\n assert_ne!(hash(&val), hash(&zero_byte(val, 1)));\n assert_ne!(hash(&val), hash(&zero_byte(val, 2)));\n assert_ne!(hash(&val), hash(&zero_byte(val, 3)));\n\n fn zero_byte(val: u32, byte: usize) -> u32 {\n assert!(byte < 4);\n val & !(0xff << (byte * 8))\n }\n}\n\n#[test]\nfn test_hash_no_concat_alias() {\n let s = (\"aa\", \"bb\");\n let t = (\"aabb\", \"\");\n let u = (\"a\", \"abb\");\n\n assert!(s != t && t != u);\n assert!(hash(&s) != hash(&t) && hash(&s) != hash(&u));\n\n let u = [1, 0, 0, 0];\n let v = (&u[..1], &u[1..3], &u[3..]);\n let w = (&u[..], &u[4..4], &u[4..4]);\n\n assert_ne!(v, w);\n assert_ne!(hash(&v), hash(&w));\n}\n\n#[test]\nfn test_hash_simple() {\n let array: &[u8] = &[1, 2, 3];\n let key: &[u8; 16] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];\n let hasher = SipHasher13::new_with_key(key);\n let h = hasher.hash(array);\n _ = h;\n}\n\n#[test]\nfn test_hash_incremental() {\n let array1: &[u8] = &[1, 2, 3];\n let array2: &[u8] = &[4, 5, 6];\n let key: &[u8; 16] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];\n let mut hasher = SipHasher13::new_with_key(key);\n hasher.write(array1);\n hasher.write(array2);\n let h = hasher.finish();\n _ = h;\n}\n\n#[test]\n#[cfg(all(feature = \"serde\", feature = \"serde_json\"))]\nfn test_hash_serde() {\n let val64 = 0xdead_beef_dead_beef_u64;\n let hash = hash(&val64);\n let serialized = serde_json::to_string(&hash).unwrap();\n let deserialized: u64 = serde_json::from_str(&serialized).unwrap();\n assert_eq!(hash, deserialized);\n}\n" }, { "package": "siphasher@0.3.11", "sourcePath": "src/tests128.rs", "sha256": "227986a432d65be2c947a500209adb7767a376b49d82e448487db6a2215476cc", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse std::hash::{Hash, Hasher};\n\nuse super::sip128::{Hasher128, SipHasher, SipHasher13, SipHasher24};\n\n// Hash just the bytes of the slice, without length prefix\nstruct Bytes<'a>(&'a [u8]);\n\nimpl<'a> Hash for Bytes<'a> {\n #[allow(unused_must_use)]\n fn hash(&self, state: &mut H) {\n let Bytes(v) = *self;\n state.write(v);\n }\n}\n\nfn hash_with(mut st: H, x: &T) -> [u8; 16] {\n x.hash(&mut st);\n st.finish128().as_bytes()\n}\n\nfn hash(x: &T) -> [u8; 16] {\n hash_with(SipHasher::new(), x)\n}\n\n#[test]\nfn test_siphash128_idempotent() {\n let val64 = 0xdead_beef_dead_beef_u64;\n assert_eq!(hash(&val64), hash(&val64));\n let val32 = 0xdeadbeef_u32;\n assert_eq!(hash(&val32), hash(&val32));\n}\n\n#[test]\n#[allow(unused_must_use)]\nfn test_siphash128_1_3() {\n let vecs: [[u8; 16]; 1] = [[\n 231, 126, 188, 178, 39, 136, 165, 190, 253, 98, 219, 106, 221, 48, 48, 1,\n ]];\n\n let k0 = 0x_07_06_05_04_03_02_01_00;\n let k1 = 0x_0f_0e_0d_0c_0b_0a_09_08;\n let mut buf = Vec::new();\n let mut t = 0;\n let mut state_inc = SipHasher13::new_with_keys(k0, k1);\n\n while t < 1 {\n let vec = vecs[t];\n let out = hash_with(SipHasher13::new_with_keys(k0, k1), &Bytes(&buf));\n assert_eq!(vec, out[..]);\n\n let full = hash_with(SipHasher13::new_with_keys(k0, k1), &Bytes(&buf));\n let i = state_inc.finish128().as_bytes();\n\n assert_eq!(full, i);\n assert_eq!(full, vec);\n\n buf.push(t as u8);\n Hasher::write(&mut state_inc, &[t as u8]);\n\n t += 1;\n }\n}\n\n#[test]\n#[allow(unused_must_use)]\nfn test_siphash128_2_4() {\n let vecs: [[u8; 16]; 1] = [[\n 163, 129, 127, 4, 186, 37, 168, 230, 109, 246, 114, 20, 199, 85, 2, 147,\n ]];\n\n let k0 = 0x_07_06_05_04_03_02_01_00;\n let k1 = 0x_0f_0e_0d_0c_0b_0a_09_08;\n let mut buf = Vec::new();\n let mut t = 0;\n let mut state_inc = SipHasher24::new_with_keys(k0, k1);\n\n while t < 1 {\n let vec = vecs[t];\n let out = hash_with(SipHasher24::new_with_keys(k0, k1), &Bytes(&buf));\n assert_eq!(vec, out[..]);\n\n let full = hash_with(SipHasher24::new_with_keys(k0, k1), &Bytes(&buf));\n let i = state_inc.finish128().as_bytes();\n\n assert_eq!(full, i);\n assert_eq!(full, vec);\n\n buf.push(t as u8);\n Hasher::write(&mut state_inc, &[t as u8]);\n\n t += 1;\n }\n}\n\n#[test]\nfn test_siphash128_simple() {\n let array: &[u8] = &[1, 2, 3];\n let key: &[u8; 16] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];\n let hasher = SipHasher13::new_with_key(key);\n let h = hasher.hash(array).as_bytes();\n _ = h;\n}\n\n#[test]\nfn test_siphash128_incremental() {\n let array1: &[u8] = &[1, 2, 3];\n let array2: &[u8] = &[4, 5, 6];\n let key: &[u8; 16] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];\n let mut hasher = SipHasher13::new_with_key(key);\n hasher.write(array1);\n hasher.write(array2);\n let h = hasher.finish128().as_bytes();\n _ = h;\n}\n\n#[test]\n#[cfg(all(feature = \"serde\", feature = \"serde_json\"))]\nfn test_siphash128_serde() {\n let val64 = 0xdead_beef_dead_beef_u64;\n let hash = hash(&val64);\n let serialized = serde_json::to_string(&hash).unwrap();\n let deserialized: [u8; 16] = serde_json::from_str(&serialized).unwrap();\n assert_eq!(hash, deserialized);\n}\n" }, { "package": "slab@0.4.12", "sourcePath": "README.md", "sha256": "6e8d6d493aec6526593ae9b05e3b21d3f878c5816a94af9d372598e03406b35f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "# Slab\n\nPre-allocated storage for a uniform data type.\n\n[![Crates.io][crates-badge]][crates-url]\n[![Build Status][ci-badge]][ci-url]\n\n[crates-badge]: https://img.shields.io/crates/v/slab\n[crates-url]: https://crates.io/crates/slab\n[ci-badge]: https://img.shields.io/github/actions/workflow/status/tokio-rs/slab/ci.yml?branch=master\n[ci-url]: https://github.com/tokio-rs/slab/actions\n\n[Documentation](https://docs.rs/slab)\n\n## Usage\n\nTo use `slab`, first add this to your `Cargo.toml`:\n\n```toml\n[dependencies]\nslab = \"0.4\"\n```\n\nNext, add this to your crate:\n\n```rust\nuse slab::Slab;\n\nlet mut slab = Slab::new();\n\nlet hello = slab.insert(\"hello\");\nlet world = slab.insert(\"world\");\n\nassert_eq!(slab[hello], \"hello\");\nassert_eq!(slab[world], \"world\");\n\nslab[world] = \"earth\";\nassert_eq!(slab[world], \"earth\");\n```\n\nSee [documentation](https://docs.rs/slab) for more details.\n\n## License\n\nThis project is licensed under the [MIT license](LICENSE).\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in `slab` by you, shall be licensed as MIT, without any additional\nterms or conditions.\n" }, { "package": "smallvec@1.16.1", "sourcePath": "src/lib.rs", "sha256": "5b134e57ce9dfa4ef8bfaac59bd0f73250123fed74f315030cd052a2ae64a41d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Small vectors in various sizes. These store a certain number of elements\n//! inline, and fall back to the heap for larger allocations. This can be a\n//! useful optimization for improving cache locality and reducing allocator\n//! traffic for workloads that fit within the inline buffer.\n//!\n//! ## `no_std` support\n//!\n//! By default, `smallvec` does not depend on `std`. However, the optional\n//! `write` feature implements the `std::io::Write` trait for vectors of `u8`.\n//! When this feature is enabled, `smallvec` depends on `std`.\n//!\n//! ## Optional features\n//!\n//! ### `serde`\n//!\n//! When this optional dependency is enabled, `SmallVec` implements the\n//! `serde::Serialize` and `serde::Deserialize` traits.\n//!\n//! ### `write`\n//!\n//! When this feature is enabled, `SmallVec<[u8; _]>` implements the\n//! `std::io::Write` trait. This feature is not compatible with `#![no_std]`\n//! programs.\n//!\n//! ### `union`\n//!\n//! **This feature requires Rust 1.49.**\n//!\n//! When the `union` feature is enabled `smallvec` will track its state (inline\n//! or spilled) without the use of an enum tag, reducing the size of the\n//! `smallvec` by one machine word. This means that there is potentially no\n//! space overhead compared to `Vec`. Note that `smallvec` can still be larger\n//! than `Vec` if the inline buffer is larger than two machine words.\n//!\n//! To use this feature add `features = [\"union\"]` in the `smallvec` section of\n//! Cargo.toml. Note that this feature requires Rust 1.49.\n//!\n//! Tracking issue: [rust-lang/rust#55149](https://github.com/rust-lang/rust/issues/55149)\n//!\n//! ### `const_generics`\n//!\n//! **This feature requires Rust 1.51.**\n//!\n//! When this feature is enabled, `SmallVec` works with any arrays of any size,\n//! not just a fixed list of sizes.\n//!\n//! ### `const_new`\n//!\n//! **This feature requires Rust 1.51.**\n//!\n//! This feature exposes the functions [`SmallVec::new_const`],\n//! [`SmallVec::from_const`], and [`smallvec_inline`] which enables the\n//! `SmallVec` to be initialized from a const context. For details, see the\n//! [Rust Reference](https://doc.rust-lang.org/reference/const_eval.html#const-functions).\n//!\n//! ### `drain_filter`\n//!\n//! **This feature is unstable.** It may change to match the unstable\n//! `drain_filter` method in libstd.\n//!\n//! Enables the `drain_filter` method, which produces an iterator that calls a\n//! user-provided closure to determine which elements of the vector to remove\n//! and yield from the iterator.\n//!\n//! ### `drain_keep_rest`\n//!\n//! **This feature is unstable.** It may change to match the unstable\n//! `drain_keep_rest` method in libstd.\n//!\n//! Enables the `DrainFilter::keep_rest` method.\n//!\n//! ### `specialization`\n//!\n//! **This feature is unstable and requires a nightly build of the Rust\n//! toolchain.**\n//!\n//! When this feature is enabled, `SmallVec::from(slice)` has improved\n//! performance for slices of `Copy` types. (Without this feature, you can use\n//! `SmallVec::from_slice` to get optimal performance for `Copy` types.)\n//!\n//! Tracking issue: [rust-lang/rust#31844](https://github.com/rust-lang/rust/issues/31844)\n//!\n//! ### `may_dangle`\n//!\n//! **This feature is unstable and requires a nightly build of the Rust\n//! toolchain.**\n//!\n//! This feature makes the Rust compiler less strict about use of vectors that\n//! contain borrowed references. For details, see the\n//! [Rustonomicon](https://doc.rust-lang.org/1.42.0/nomicon/dropck.html#an-escape-hatch).\n//!\n//! Tracking issue: [rust-lang/rust#34761](https://github.com/rust-lang/rust/issues/34761)\n\n#![no_std]\n#![cfg_attr(docsrs, feature(doc_cfg))]\n#![cfg_attr(feature = \"specialization\", allow(incomplete_features))]\n#![cfg_attr(feature = \"specialization\", feature(specialization))]\n#![cfg_attr(feature = \"may_dangle\", feature(dropck_eyepatch))]\n#![deny(missing_docs)]\n\n#[doc(hidden)]\npub extern crate alloc;\n\n#[cfg(any(test, feature = \"write\"))]\nextern crate std;\n\n#[cfg(test)]\nmod tests;\n\n#[cfg(feature = \"serde\")]\nuse core::marker::PhantomData;\n#[cfg(feature = \"drain_keep_rest\")]\nuse core::mem::ManuallyDrop;\n#[cfg(feature = \"malloc_size_of\")]\nuse malloc_size_of::{MallocShallowSizeOf, MallocSizeOf, MallocSizeOfOps};\n#[cfg(feature = \"serde\")]\nuse serde::{\n de::{Deserialize, Deserializer, SeqAccess, Visitor},\n ser::{Serialize, SerializeSeq, Serializer},\n};\n#[cfg(feature = \"write\")]\nuse std::io;\n#[allow(deprecated)]\nuse {\n alloc::{\n alloc::{Layout, LayoutErr},\n boxed::Box,\n vec,\n vec::Vec,\n },\n core::{\n borrow::{Borrow, BorrowMut},\n cmp, fmt,\n hash::{Hash, Hasher},\n hint::unreachable_unchecked,\n iter::{repeat, FromIterator, FusedIterator, IntoIterator},\n mem::{self, MaybeUninit},\n ops::{self, Range, RangeBounds},\n ptr::{self, NonNull},\n slice::{self, SliceIndex},\n },\n};\n\n/// Creates a [`SmallVec`] containing the arguments.\n///\n/// `smallvec!` allows `SmallVec`s to be defined with the same syntax as array\n/// expressions. There are two forms of this macro:\n///\n/// - Create a [`SmallVec`] containing a given list of elements:\n///\n/// ```\n/// # use smallvec::{smallvec, SmallVec};\n/// # fn main() {\n/// let v: SmallVec<[_; 128]> = smallvec![1, 2, 3];\n/// assert_eq!(v[0], 1);\n/// assert_eq!(v[1], 2);\n/// assert_eq!(v[2], 3);\n/// # }\n/// ```\n///\n/// - Create a [`SmallVec`] from a given element and size:\n///\n/// ```\n/// # use smallvec::{smallvec, SmallVec};\n/// # fn main() {\n/// let v: SmallVec<[_; 10]> = smallvec![1; 3];\n/// assert_eq!(v, SmallVec::from_buf([1, 1, 1]));\n/// # }\n/// ```\n///\n/// Note that unlike array expressions this syntax supports all elements\n/// which implement [`Clone`] and the number of elements doesn't have to be\n/// a constant.\n///\n/// This will use `clone` to duplicate an expression, so one should be careful\n/// using this with types having a nonstandard `Clone` implementation. For\n/// example, `smallvec![Rc::new(1); 5]` will create a vector of five references\n/// to the same boxed integer value, not five references pointing to\n/// independently boxed integers.\n#[macro_export]\nmacro_rules! smallvec {\n // count helper: transform any expression into 1\n (@one $x:expr) => (1usize);\n () => (\n $crate::SmallVec::new()\n );\n ($elem:expr; $n:expr) => ({\n $crate::SmallVec::from_elem($elem, $n)\n });\n ($($x:expr),+$(,)?) => ({\n let count = 0usize $(+ $crate::smallvec!(@one $x))+;\n let mut vec = $crate::SmallVec::new();\n if count <= vec.inline_size() {\n $(vec.push($x);)*\n vec\n } else {\n $crate::SmallVec::from_vec($crate::alloc::vec![$($x,)+])\n }\n });\n}\n\n/// Creates an inline [`SmallVec`] containing the arguments. This macro is\n/// enabled by the feature `const_new`.\n///\n/// `smallvec_inline!` allows `SmallVec`s to be defined with the same syntax as\n/// array expressions in `const` contexts. The inline storage `A` will always be\n/// an array of the size specified by the arguments. There are two forms of this\n/// macro:\n///\n/// - Create a [`SmallVec`] containing a given list of elements:\n///\n/// ```\n/// # use smallvec::{smallvec_inline, SmallVec};\n/// # fn main() {\n/// const V: SmallVec<[i32; 3]> = smallvec_inline![1, 2, 3];\n/// assert_eq!(V[0], 1);\n/// assert_eq!(V[1], 2);\n/// assert_eq!(V[2], 3);\n/// # }\n/// ```\n///\n/// - Create a [`SmallVec`] from a given element and size:\n///\n/// ```\n/// # use smallvec::{smallvec_inline, SmallVec};\n/// # fn main() {\n/// const V: SmallVec<[i32; 3]> = smallvec_inline![1; 3];\n/// assert_eq!(V, SmallVec::from_buf([1, 1, 1]));\n/// # }\n/// ```\n///\n/// Note that the behavior mimics that of array expressions, in contrast to\n/// [`smallvec`].\n#[cfg(feature = \"const_new\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"const_new\")))]\n#[macro_export]\nmacro_rules! smallvec_inline {\n // count helper: transform any expression into 1\n (@one $x:expr) => (1usize);\n ($elem:expr; $n:expr) => ({\n $crate::SmallVec::<[_; $n]>::from_const([$elem; $n])\n });\n ($($x:expr),+ $(,)?) => ({\n const N: usize = 0usize $(+ $crate::smallvec_inline!(@one $x))*;\n $crate::SmallVec::<[_; N]>::from_const([$($x,)*])\n });\n}\n\n/// `panic!()` in debug builds, optimization hint in release.\n#[cfg(not(feature = \"union\"))]\nmacro_rules! debug_unreachable {\n () => {\n debug_unreachable!(\"entered unreachable code\")\n };\n ($e:expr) => {\n if cfg!(debug_assertions) {\n panic!($e);\n } else {\n unreachable_unchecked();\n }\n };\n}\n\n/// Trait to be implemented by a collection that can be extended from a slice\n///\n/// ## Example\n///\n/// ```rust\n/// use smallvec::{ExtendFromSlice, SmallVec};\n///\n/// fn initialize>(v: &mut V) {\n/// v.extend_from_slice(b\"Test!\");\n/// }\n///\n/// let mut vec = Vec::new();\n/// initialize(&mut vec);\n/// assert_eq!(&vec, b\"Test!\");\n///\n/// let mut small_vec = SmallVec::<[u8; 8]>::new();\n/// initialize(&mut small_vec);\n/// assert_eq!(&small_vec as &[_], b\"Test!\");\n/// ```\n#[doc(hidden)]\n#[deprecated]\npub trait ExtendFromSlice {\n /// Extends a collection from a slice of its element type\n fn extend_from_slice(&mut self, other: &[T]);\n}\n\n#[allow(deprecated)]\nimpl ExtendFromSlice for Vec {\n fn extend_from_slice(&mut self, other: &[T]) {\n Vec::extend_from_slice(self, other)\n }\n}\n\n/// Error type for APIs with fallible heap allocation\n#[derive(Debug)]\npub enum CollectionAllocErr {\n /// Overflow `usize::MAX` or other error during size computation\n CapacityOverflow,\n /// The allocator return an error\n AllocErr {\n /// The layout that was passed to the allocator\n layout: Layout,\n },\n}\n\nimpl fmt::Display for CollectionAllocErr {\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n write!(f, \"Allocation error: {:?}\", self)\n }\n}\n\n#[allow(deprecated)]\nimpl From for CollectionAllocErr {\n fn from(_: LayoutErr) -> Self {\n CollectionAllocErr::CapacityOverflow\n }\n}\n\nfn infallible(result: Result) -> T {\n match result {\n Ok(x) => x,\n Err(CollectionAllocErr::CapacityOverflow) => panic!(\"capacity overflow\"),\n Err(CollectionAllocErr::AllocErr { layout }) => alloc::alloc::handle_alloc_error(layout),\n }\n}\n\n/// FIXME: use `Layout::array` when we require a Rust version where it’s stable\n/// \nfn layout_array(n: usize) -> Result {\n let size = mem::size_of::()\n .checked_mul(n)\n .ok_or(CollectionAllocErr::CapacityOverflow)?;\n let align = mem::align_of::();\n Layout::from_size_align(size, align).map_err(|_| CollectionAllocErr::CapacityOverflow)\n}\n\nunsafe fn deallocate(ptr: NonNull, capacity: usize) {\n // This unwrap should succeed since the same did when allocating.\n let layout = layout_array::(capacity).unwrap();\n alloc::alloc::dealloc(ptr.as_ptr() as *mut u8, layout)\n}\n\n/// An iterator that removes the items from a `SmallVec` and yields them by\n/// value.\n///\n/// Returned from [`SmallVec::drain`][1].\n///\n/// [1]: struct.SmallVec.html#method.drain\npub struct Drain<'a, T: 'a + Array> {\n tail_start: usize,\n tail_len: usize,\n iter: slice::Iter<'a, T::Item>,\n vec: NonNull>,\n}\n\nimpl<'a, T: 'a + Array> fmt::Debug for Drain<'a, T>\nwhere\n T::Item: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_tuple(\"Drain\").field(&self.iter.as_slice()).finish()\n }\n}\n\nunsafe impl<'a, T: Sync + Array> Sync for Drain<'a, T> {}\nunsafe impl<'a, T: Send + Array> Send for Drain<'a, T> {}\n\nimpl<'a, T: 'a + Array> Iterator for Drain<'a, T> {\n type Item = T::Item;\n\n #[inline]\n fn next(&mut self) -> Option {\n self.iter\n .next()\n .map(|reference| unsafe { ptr::read(reference) })\n }\n\n #[inline]\n fn size_hint(&self) -> (usize, Option) {\n self.iter.size_hint()\n }\n}\n\nimpl<'a, T: 'a + Array> DoubleEndedIterator for Drain<'a, T> {\n #[inline]\n fn next_back(&mut self) -> Option {\n self.iter\n .next_back()\n .map(|reference| unsafe { ptr::read(reference) })\n }\n}\n\nimpl<'a, T: Array> ExactSizeIterator for Drain<'a, T> {\n #[inline]\n fn len(&self) -> usize {\n self.iter.len()\n }\n}\n\nimpl<'a, T: Array> FusedIterator for Drain<'a, T> {}\n\nimpl<'a, T: 'a + Array> Drop for Drain<'a, T> {\n fn drop(&mut self) {\n self.for_each(drop);\n\n if self.tail_len > 0 {\n unsafe {\n let source_vec = self.vec.as_mut();\n\n // memmove back untouched tail, update to new length\n let start = source_vec.len();\n let tail = self.tail_start;\n if tail != start {\n // as_mut_ptr creates a &mut, invalidating other pointers.\n // This pattern avoids calling it with a pointer already\n // present.\n let ptr = source_vec.as_mut_ptr();\n let src = ptr.add(tail);\n let dst = ptr.add(start);\n ptr::copy(src, dst, self.tail_len);\n }\n source_vec.set_len(start + self.tail_len);\n }\n }\n }\n}\n\n#[cfg(feature = \"drain_filter\")]\n/// An iterator which uses a closure to determine if an element should be\n/// removed.\n///\n/// Returned from [`SmallVec::drain_filter`][1].\n///\n/// [1]: struct.SmallVec.html#method.drain_filter\npub struct DrainFilter<'a, T, F>\nwhere\n F: FnMut(&mut T::Item) -> bool,\n T: Array,\n{\n vec: &'a mut SmallVec,\n /// The index of the item that will be inspected by the next call to `next`.\n idx: usize,\n /// The number of items that have been drained (removed) thus far.\n del: usize,\n /// The original length of `vec` prior to draining.\n old_len: usize,\n /// The filter test predicate.\n pred: F,\n /// A flag that indicates a panic has occurred in the filter test predicate.\n /// This is used as a hint in the drop implementation to prevent consumption\n /// of the remainder of the `DrainFilter`. Any unprocessed items will be\n /// backshifted in the `vec`, but no further items will be dropped or\n /// tested by the filter predicate.\n panic_flag: bool,\n}\n\n#[cfg(feature = \"drain_filter\")]\nimpl fmt::Debug for DrainFilter<'_, T, F>\nwhere\n F: FnMut(&mut T::Item) -> bool,\n T: Array,\n T::Item: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_tuple(\"DrainFilter\")\n .field(&self.vec.as_slice())\n .finish()\n }\n}\n\n#[cfg(feature = \"drain_filter\")]\nimpl Iterator for DrainFilter<'_, T, F>\nwhere\n F: FnMut(&mut T::Item) -> bool,\n T: Array,\n{\n type Item = T::Item;\n\n fn next(&mut self) -> Option {\n unsafe {\n while self.idx < self.old_len {\n let i = self.idx;\n let v = slice::from_raw_parts_mut(self.vec.as_mut_ptr(), self.old_len);\n self.panic_flag = true;\n let drained = (self.pred)(&mut v[i]);\n self.panic_flag = false;\n // Update the index *after* the predicate is called. If the\n // index is updated prior and the predicate\n // panics, the element at this index would be\n // leaked.\n self.idx += 1;\n if drained {\n self.del += 1;\n return Some(ptr::read(&v[i]));\n } else if self.del > 0 {\n let del = self.del;\n let src: *const Self::Item = &v[i];\n let dst: *mut Self::Item = &mut v[i - del];\n ptr::copy_nonoverlapping(src, dst, 1);\n }\n }\n None\n }\n }\n\n fn size_hint(&self) -> (usize, Option) {\n (0, Some(self.old_len - self.idx))\n }\n}\n\n#[cfg(feature = \"drain_filter\")]\nimpl Drop for DrainFilter<'_, T, F>\nwhere\n F: FnMut(&mut T::Item) -> bool,\n T: Array,\n{\n fn drop(&mut self) {\n struct BackshiftOnDrop<'a, 'b, T, F>\n where\n F: FnMut(&mut T::Item) -> bool,\n T: Array,\n {\n drain: &'b mut DrainFilter<'a, T, F>,\n }\n\n impl<'a, 'b, T, F> Drop for BackshiftOnDrop<'a, 'b, T, F>\n where\n F: FnMut(&mut T::Item) -> bool,\n T: Array,\n {\n fn drop(&mut self) {\n unsafe {\n if self.drain.idx < self.drain.old_len && self.drain.del > 0 {\n // This is a pretty messed up state, and there isn't\n // really an obviously right\n // thing to do. We don't want to keep trying\n // to execute `pred`, so we just backshift all the\n // unprocessed elements and tell\n // the vec that they still exist. The backshift\n // is required to prevent a double-drop of the last\n // successfully drained item\n // prior to a panic in the predicate.\n let ptr = self.drain.vec.as_mut_ptr();\n let src = ptr.add(self.drain.idx);\n let dst = src.sub(self.drain.del);\n let tail_len = self.drain.old_len - self.drain.idx;\n src.copy_to(dst, tail_len);\n }\n self.drain.vec.set_len(self.drain.old_len - self.drain.del);\n }\n }\n }\n\n let backshift = BackshiftOnDrop { drain: self };\n\n // Attempt to consume any remaining elements if the filter predicate\n // has not yet panicked. We'll backshift any remaining elements\n // whether we've already panicked or if the consumption here panics.\n if !backshift.drain.panic_flag {\n backshift.drain.for_each(drop);\n }\n }\n}\n\n#[cfg(feature = \"drain_keep_rest\")]\nimpl DrainFilter<'_, T, F>\nwhere\n F: FnMut(&mut T::Item) -> bool,\n T: Array,\n{\n /// Keep unyielded elements in the source `Vec`.\n ///\n /// # Examples\n ///\n /// ```\n /// # use smallvec::{smallvec, SmallVec};\n ///\n /// let mut vec: SmallVec<[char; 2]> = smallvec!['a', 'b', 'c'];\n /// let mut drain = vec.drain_filter(|_| true);\n ///\n /// assert_eq!(drain.next().unwrap(), 'a');\n ///\n /// // This call keeps 'b' and 'c' in the vec.\n /// drain.keep_rest();\n ///\n /// // If we wouldn't call `keep_rest()`,\n /// // `vec` would be empty.\n /// assert_eq!(vec, SmallVec::<[char; 2]>::from_slice(&['b', 'c']));\n /// ```\n pub fn keep_rest(self) {\n // At this moment layout looks like this:\n //\n // _____________________/-- old_len\n // / \\\n // [kept] [yielded] [tail]\n // \\_______/ ^-- idx\n // \\-- del\n //\n // Normally `Drop` impl would drop [tail] (via .for_each(drop), ie still\n // calling `pred`)\n //\n // 1. Move [tail] after [kept]\n // 2. Update length of the original vec to `old_len - del` a. In case of\n // ZST, this is the only thing we want to do\n // 3. Do *not* drop self, as everything is put in a consistent state\n // already, there is nothing to do\n let mut this = ManuallyDrop::new(self);\n\n unsafe {\n // ZSTs have no identity, so we don't need to move them around.\n let needs_move = mem::size_of::() != 0;\n\n if needs_move && this.idx < this.old_len && this.del > 0 {\n let ptr = this.vec.as_mut_ptr();\n let src = ptr.add(this.idx);\n let dst = src.sub(this.del);\n let tail_len = this.old_len - this.idx;\n src.copy_to(dst, tail_len);\n }\n\n let new_len = this.old_len - this.del;\n this.vec.set_len(new_len);\n }\n }\n}\n\n#[cfg(feature = \"union\")]\nunion SmallVecData {\n inline: core::mem::ManuallyDrop>,\n heap: (NonNull, usize),\n}\n\n#[cfg(all(feature = \"union\", feature = \"const_new\"))]\nimpl SmallVecData<[T; N]> {\n #[cfg_attr(docsrs, doc(cfg(feature = \"const_new\")))]\n #[inline]\n const fn from_const(inline: MaybeUninit<[T; N]>) -> Self {\n SmallVecData {\n inline: core::mem::ManuallyDrop::new(inline),\n }\n }\n}\n\n#[cfg(feature = \"union\")]\nimpl SmallVecData {\n #[inline]\n unsafe fn inline(&self) -> ConstNonNull {\n ConstNonNull::new(self.inline.as_ptr() as *const A::Item).unwrap()\n }\n #[inline]\n unsafe fn inline_mut(&mut self) -> NonNull {\n NonNull::new(self.inline.as_mut_ptr() as *mut A::Item).unwrap()\n }\n #[inline]\n fn from_inline(inline: MaybeUninit) -> SmallVecData {\n SmallVecData {\n inline: core::mem::ManuallyDrop::new(inline),\n }\n }\n // Workaround for https://github.com/rust-lang/rust/issues/157743: when from_inline is\n // called with MaybeUninit::uninit(), rustc 1.93+ GVN propagates const\n // into the ManuallyDrop::new() aggregate, causing LLVM to\n // materialize a global constant that MemCpyOpt then collapses into a\n // memset over the whole struct. Using assume_init() of a doubly-wrapped\n // MaybeUninit produces Immediate::Uninit instead of const ,\n // which codegen handles as undef without emitting any global. This\n // function also avoids introducing an intermediate local that would\n // inflate stack frames in debug builds.\n #[inline]\n fn empty() -> SmallVecData {\n // SAFETY: ManuallyDrop> is valid for any bit pattern\n // including uninitialized bytes, so assume_init() on a\n // MaybeUninit of that type is sound.\n SmallVecData {\n inline: unsafe { MaybeUninit::uninit().assume_init() },\n }\n }\n #[inline]\n unsafe fn into_inline(self) -> MaybeUninit {\n core::mem::ManuallyDrop::into_inner(self.inline)\n }\n #[inline]\n unsafe fn heap(&self) -> (ConstNonNull, usize) {\n (ConstNonNull(self.heap.0), self.heap.1)\n }\n #[inline]\n unsafe fn heap_mut(&mut self) -> (NonNull, &mut usize) {\n let h = &mut self.heap;\n (h.0, &mut h.1)\n }\n #[inline]\n fn from_heap(ptr: NonNull, len: usize) -> SmallVecData {\n SmallVecData { heap: (ptr, len) }\n }\n}\n\n#[cfg(not(feature = \"union\"))]\nenum SmallVecData {\n Inline(MaybeUninit),\n // Using NonNull and NonZero here allows to reduce size of `SmallVec`.\n Heap {\n // Since we never allocate on heap\n // unless our capacity is bigger than inline capacity\n // heap capacity cannot be less than 1.\n // Therefore, pointer cannot be null too.\n ptr: NonNull,\n len: usize,\n },\n}\n\n#[cfg(all(not(feature = \"union\"), feature = \"const_new\"))]\nimpl SmallVecData<[T; N]> {\n #[cfg_attr(docsrs, doc(cfg(feature = \"const_new\")))]\n #[inline]\n const fn from_const(inline: MaybeUninit<[T; N]>) -> Self {\n SmallVecData::Inline(inline)\n }\n}\n\n#[cfg(not(feature = \"union\"))]\nimpl SmallVecData {\n #[inline]\n unsafe fn inline(&self) -> ConstNonNull {\n match self {\n SmallVecData::Inline(a) => ConstNonNull::new(a.as_ptr() as *const A::Item).unwrap(),\n _ => debug_unreachable!(),\n }\n }\n #[inline]\n unsafe fn inline_mut(&mut self) -> NonNull {\n match self {\n SmallVecData::Inline(a) => NonNull::new(a.as_mut_ptr() as *mut A::Item).unwrap(),\n _ => debug_unreachable!(),\n }\n }\n #[inline]\n fn from_inline(inline: MaybeUninit) -> SmallVecData {\n SmallVecData::Inline(inline)\n }\n // See the comment on the union variant's empty() for why this exists.\n #[inline]\n fn empty() -> SmallVecData {\n // SAFETY: MaybeUninit is valid for any bit pattern including\n // uninitialized bytes, so assume_init() on a MaybeUninit of\n // that type is sound.\n SmallVecData::Inline(unsafe { MaybeUninit::uninit().assume_init() })\n }\n #[inline]\n unsafe fn into_inline(self) -> MaybeUninit {\n match self {\n SmallVecData::Inline(a) => a,\n _ => debug_unreachable!(),\n }\n }\n #[inline]\n unsafe fn heap(&self) -> (ConstNonNull, usize) {\n match self {\n SmallVecData::Heap { ptr, len } => (ConstNonNull(*ptr), *len),\n _ => debug_unreachable!(),\n }\n }\n #[inline]\n unsafe fn heap_mut(&mut self) -> (NonNull, &mut usize) {\n match self {\n SmallVecData::Heap { ptr, len } => (*ptr, len),\n _ => debug_unreachable!(),\n }\n }\n #[inline]\n fn from_heap(ptr: NonNull, len: usize) -> SmallVecData {\n SmallVecData::Heap { ptr, len }\n }\n}\n\nunsafe impl Send for SmallVecData {}\nunsafe impl Sync for SmallVecData {}\n\n/// A `Vec`-like container that can store a small number of elements inline.\n///\n/// `SmallVec` acts like a vector, but can store a limited amount of data inline\n/// within the `SmallVec` struct rather than in a separate allocation. If the\n/// data exceeds this limit, the `SmallVec` will \"spill\" its data onto the heap,\n/// allocating a new buffer to hold it.\n///\n/// The amount of data that a `SmallVec` can store inline depends on its backing\n/// store. The backing store can be any type that implements the `Array` trait;\n/// usually it is a small fixed-sized array. For example a `SmallVec<[u64; 8]>`\n/// can hold up to eight 64-bit integers inline.\n///\n/// ## Example\n///\n/// ```rust\n/// use smallvec::SmallVec;\n/// let mut v = SmallVec::<[u8; 4]>::new(); // initialize an empty vector\n///\n/// // The vector can hold up to 4 items without spilling onto the heap.\n/// v.extend(0..4);\n/// assert_eq!(v.len(), 4);\n/// assert!(!v.spilled());\n///\n/// // Pushing another element will force the buffer to spill:\n/// v.push(4);\n/// assert_eq!(v.len(), 5);\n/// assert!(v.spilled());\n/// ```\npub struct SmallVec {\n // The capacity field is used to determine which of the storage variants is active:\n // If capacity <= Self::inline_capacity() then the inline variant is used and capacity holds\n // the current length of the vector (number of elements actually in use). If capacity >\n // Self::inline_capacity() then the heap variant is used and capacity holds the size of the\n // memory allocation.\n capacity: usize,\n data: SmallVecData,\n}\n\nimpl SmallVec {\n /// Construct an empty vector\n #[inline]\n pub fn new() -> SmallVec {\n // Try to detect invalid custom implementations of `Array`. Hopefully,\n // this check should be optimized away entirely for valid ones.\n assert!(\n mem::size_of::() == A::size() * mem::size_of::()\n && mem::align_of::() >= mem::align_of::()\n );\n SmallVec {\n capacity: 0,\n data: SmallVecData::empty(),\n }\n }\n\n /// Construct an empty vector with enough capacity pre-allocated to store at\n /// least `n` elements.\n ///\n /// Will create a heap allocation only if `n` is larger than the inline\n /// capacity.\n ///\n /// ```\n /// # use smallvec::SmallVec;\n ///\n /// let v: SmallVec<[u8; 3]> = SmallVec::with_capacity(100);\n ///\n /// assert!(v.is_empty());\n /// assert!(v.capacity() >= 100);\n /// ```\n #[inline]\n pub fn with_capacity(n: usize) -> Self {\n let mut v = SmallVec::new();\n v.reserve_exact(n);\n v\n }\n\n /// Construct a new `SmallVec` from a `Vec`.\n ///\n /// Elements will be copied to the inline buffer if `vec.capacity() <=\n /// Self::inline_capacity()`.\n ///\n /// ```rust\n /// use smallvec::SmallVec;\n ///\n /// let vec = vec![1, 2, 3, 4, 5];\n /// let small_vec: SmallVec<[_; 3]> = SmallVec::from_vec(vec);\n ///\n /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]);\n /// ```\n #[inline]\n pub fn from_vec(mut vec: Vec) -> SmallVec {\n if vec.capacity() <= Self::inline_capacity() {\n // Cannot use Vec with smaller capacity\n // because we use value of `Self::capacity` field as indicator.\n unsafe {\n let mut data = SmallVecData::::empty();\n let len = vec.len();\n vec.set_len(0);\n ptr::copy_nonoverlapping(vec.as_ptr(), data.inline_mut().as_ptr(), len);\n\n SmallVec {\n capacity: len,\n data,\n }\n }\n } else {\n let (ptr, cap, len) = (vec.as_mut_ptr(), vec.capacity(), vec.len());\n mem::forget(vec);\n let ptr = NonNull::new(ptr)\n // See docs: https://doc.rust-lang.org/std/vec/struct.Vec.html#method.as_mut_ptr\n .expect(\"Cannot be null by `Vec` invariant\");\n\n SmallVec {\n capacity: cap,\n data: SmallVecData::from_heap(ptr, len),\n }\n }\n }\n\n /// Constructs a new `SmallVec` on the stack from an `A` without\n /// copying elements.\n ///\n /// ```rust\n /// use smallvec::SmallVec;\n ///\n /// let buf = [1, 2, 3, 4, 5];\n /// let small_vec: SmallVec<_> = SmallVec::from_buf(buf);\n ///\n /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]);\n /// ```\n #[inline]\n pub fn from_buf(buf: A) -> SmallVec {\n SmallVec {\n capacity: A::size(),\n data: SmallVecData::from_inline(MaybeUninit::new(buf)),\n }\n }\n\n /// Constructs a new `SmallVec` on the stack from an `A` without\n /// copying elements. Also sets the length, which must be less or\n /// equal to the size of `buf`.\n ///\n /// ```rust\n /// use smallvec::SmallVec;\n ///\n /// let buf = [1, 2, 3, 4, 5, 0, 0, 0];\n /// let small_vec: SmallVec<_> = SmallVec::from_buf_and_len(buf, 5);\n ///\n /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]);\n /// ```\n #[inline]\n pub fn from_buf_and_len(buf: A, len: usize) -> SmallVec {\n assert!(len <= A::size());\n unsafe { SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), len) }\n }\n\n /// Constructs a new `SmallVec` on the stack from an `A` without\n /// copying elements. Also sets the length. The user is responsible\n /// for ensuring that `len <= A::size()`.\n ///\n /// ```rust\n /// use {smallvec::SmallVec, std::mem::MaybeUninit};\n ///\n /// let buf = [1, 2, 3, 4, 5, 0, 0, 0];\n /// let small_vec: SmallVec<_> =\n /// unsafe { SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), 5) };\n ///\n /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]);\n /// ```\n #[inline]\n pub unsafe fn from_buf_and_len_unchecked(buf: MaybeUninit, len: usize) -> SmallVec {\n SmallVec {\n capacity: len,\n data: SmallVecData::from_inline(buf),\n }\n }\n\n /// Sets the length of a vector.\n ///\n /// This will explicitly set the size of the vector, without actually\n /// modifying its buffers, so it is up to the caller to ensure that the\n /// vector is actually the specified size.\n pub unsafe fn set_len(&mut self, new_len: usize) {\n let (_, len_ptr, _) = self.triple_mut();\n *len_ptr = new_len;\n }\n\n /// The maximum number of elements this vector can hold inline\n #[inline]\n fn inline_capacity() -> usize {\n if mem::size_of::() > 0 {\n A::size()\n } else {\n // For zero-size items code like `ptr.add(offset)` always returns\n // the same pointer. Therefore all items are at the same\n // address, and any array size has capacity for\n // infinitely many items. The capacity is limited by the\n // bit width of the length field.\n //\n // `Vec` also does this:\n // https://github.com/rust-lang/rust/blob/1.44.0/src/liballoc/raw_vec.rs#L186\n //\n // In our case, this also ensures that a smallvec of zero-size items\n // never spills, and we never try to allocate zero bytes\n // which `std::alloc::alloc` disallows.\n #[allow(deprecated)]\n core::usize::MAX\n }\n }\n\n /// The maximum number of elements this vector can hold inline\n #[inline]\n pub fn inline_size(&self) -> usize {\n Self::inline_capacity()\n }\n\n /// The number of elements stored in the vector\n #[inline]\n pub fn len(&self) -> usize {\n self.triple().1\n }\n\n /// Returns `true` if the vector is empty\n #[inline]\n pub fn is_empty(&self) -> bool {\n self.len() == 0\n }\n\n /// The number of items the vector can hold without reallocating\n #[inline]\n pub fn capacity(&self) -> usize {\n self.triple().2\n }\n\n /// Returns a tuple with (data ptr, len, capacity)\n /// Useful to get all `SmallVec` properties with a single check of the\n /// current storage variant.\n #[inline]\n fn triple(&self) -> (ConstNonNull, usize, usize) {\n unsafe {\n if self.spilled() {\n let (ptr, len) = self.data.heap();\n (ptr, len, self.capacity)\n } else {\n (self.data.inline(), self.capacity, Self::inline_capacity())\n }\n }\n }\n\n /// Returns a tuple with (data ptr, len ptr, capacity)\n #[inline]\n fn triple_mut(&mut self) -> (NonNull, &mut usize, usize) {\n unsafe {\n if self.spilled() {\n let (ptr, len_ptr) = self.data.heap_mut();\n (ptr, len_ptr, self.capacity)\n } else {\n (\n self.data.inline_mut(),\n &mut self.capacity,\n Self::inline_capacity(),\n )\n }\n }\n }\n\n /// Returns `true` if the data has spilled into a separate heap-allocated\n /// buffer.\n #[inline]\n pub fn spilled(&self) -> bool {\n self.capacity > Self::inline_capacity()\n }\n\n /// Creates a draining iterator that removes the specified range in the\n /// vector and yields the removed items.\n ///\n /// Note 1: The element range is removed even if the iterator is only\n /// partially consumed or not consumed at all.\n ///\n /// Note 2: It is unspecified how many elements are removed from the vector\n /// if the `Drain` value is leaked.\n ///\n /// # Panics\n ///\n /// Panics if the starting point is greater than the end point or if\n /// the end point is greater than the length of the vector.\n pub fn drain(&mut self, range: R) -> Drain<'_, A>\n where\n R: RangeBounds,\n {\n use core::ops::Bound::*;\n\n let len = self.len();\n let start = match range.start_bound() {\n Included(&n) => n,\n Excluded(&n) => n.checked_add(1).expect(\"Range start out of bounds\"),\n Unbounded => 0,\n };\n let end = match range.end_bound() {\n Included(&n) => n.checked_add(1).expect(\"Range end out of bounds\"),\n Excluded(&n) => n,\n Unbounded => len,\n };\n\n assert!(start <= end);\n assert!(end <= len);\n\n unsafe {\n self.set_len(start);\n\n let range_slice = slice::from_raw_parts(self.as_ptr().add(start), end - start);\n\n Drain {\n tail_start: end,\n tail_len: len - end,\n iter: range_slice.iter(),\n // Since self is a &mut, passing it to a function would invalidate the slice\n // iterator.\n vec: NonNull::new_unchecked(self as *mut _),\n }\n }\n }\n\n #[cfg(feature = \"drain_filter\")]\n /// Creates an iterator which uses a closure to determine if an element\n /// should be removed.\n ///\n /// If the closure returns true, the element is removed and yielded. If the\n /// closure returns false, the element will remain in the vector and\n /// will not be yielded by the iterator.\n ///\n /// Using this method is equivalent to the following code:\n /// ```\n /// # use smallvec::SmallVec;\n /// # let some_predicate = |x: &mut i32| { *x == 2 || *x == 3 || *x == 6 };\n /// # let mut vec: SmallVec<[i32; 8]> = SmallVec::from_slice(&[1i32, 2, 3, 4, 5, 6]);\n /// let mut i = 0;\n /// while i < vec.len() {\n /// if some_predicate(&mut vec[i]) {\n /// let val = vec.remove(i);\n /// // your code here\n /// } else {\n /// i += 1;\n /// }\n /// }\n ///\n /// # assert_eq!(vec, SmallVec::<[i32; 8]>::from_slice(&[1i32, 4, 5]));\n /// ```\n /// ///\n /// But `drain_filter` is easier to use. `drain_filter` is also more\n /// efficient, because it can backshift the elements of the array in\n /// bulk.\n ///\n /// Note that `drain_filter` also lets you mutate every element in the\n /// filter closure, regardless of whether you choose to keep or remove\n /// it.\n ///\n /// # Examples\n ///\n /// Splitting an array into evens and odds, reusing the original allocation:\n ///\n /// ```\n /// # use smallvec::SmallVec;\n /// let mut numbers: SmallVec<[i32; 16]> =\n /// SmallVec::from_slice(&[1i32, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 15]);\n ///\n /// let evens = numbers\n /// .drain_filter(|x| *x % 2 == 0)\n /// .collect::>();\n /// let odds = numbers;\n ///\n /// assert_eq!(\n /// evens,\n /// SmallVec::<[i32; 16]>::from_slice(&[2i32, 4, 6, 8, 14])\n /// );\n /// assert_eq!(\n /// odds,\n /// SmallVec::<[i32; 16]>::from_slice(&[1i32, 3, 5, 9, 11, 13, 15])\n /// );\n /// ```\n pub fn drain_filter(&mut self, filter: F) -> DrainFilter<'_, A, F>\n where\n F: FnMut(&mut A::Item) -> bool,\n {\n let old_len = self.len();\n\n // Guard against us getting leaked (leak amplification)\n unsafe {\n self.set_len(0);\n }\n\n DrainFilter {\n vec: self,\n idx: 0,\n del: 0,\n old_len,\n pred: filter,\n panic_flag: false,\n }\n }\n\n /// Append an item to the vector.\n #[inline]\n pub fn push(&mut self, value: A::Item) {\n unsafe {\n if self.spilled() {\n let (mut ptr, mut len_ptr) = self.data.heap_mut();\n if *len_ptr == self.capacity {\n self.reserve_one_unchecked();\n let (heap_ptr, heap_len) = self.data.heap_mut();\n ptr = heap_ptr;\n len_ptr = heap_len;\n }\n ptr::write(ptr.as_ptr().add(*len_ptr), value);\n *len_ptr += 1;\n } else {\n let mut ptr = self.data.inline_mut();\n let mut len_ptr = &mut self.capacity;\n if *len_ptr == Self::inline_capacity() {\n self.reserve_one_unchecked();\n let (heap_ptr, heap_len) = self.data.heap_mut();\n ptr = heap_ptr;\n len_ptr = heap_len;\n }\n ptr::write(ptr.as_ptr().add(*len_ptr), value);\n *len_ptr += 1;\n };\n }\n }\n\n /// Remove an item from the end of the vector and return it, or None if\n /// empty.\n #[inline]\n pub fn pop(&mut self) -> Option {\n unsafe {\n let (ptr, len_ptr, _) = self.triple_mut();\n let ptr: *const _ = ptr.as_ptr();\n if *len_ptr == 0 {\n return None;\n }\n let last_index = *len_ptr - 1;\n *len_ptr = last_index;\n Some(ptr::read(ptr.add(last_index)))\n }\n }\n\n /// Moves all the elements of `other` into `self`, leaving `other` empty.\n ///\n /// # Example\n ///\n /// ```\n /// # use smallvec::{SmallVec, smallvec};\n /// let mut v0: SmallVec<[u8; 16]> = smallvec![1, 2, 3];\n /// let mut v1: SmallVec<[u8; 32]> = smallvec![4, 5, 6];\n /// v0.append(&mut v1);\n /// assert_eq!(*v0, [1, 2, 3, 4, 5, 6]);\n /// assert_eq!(*v1, []);\n /// ```\n pub fn append(&mut self, other: &mut SmallVec)\n where\n B: Array,\n {\n self.extend(other.drain(..))\n }\n\n /// Re-allocate to set the capacity to `max(new_cap, inline_size())`.\n ///\n /// Panics if `new_cap` is less than the vector's length\n /// or if the capacity computation overflows `usize`.\n pub fn grow(&mut self, new_cap: usize) {\n infallible(self.try_grow(new_cap))\n }\n\n /// Re-allocate to set the capacity to `max(new_cap, inline_size())`.\n ///\n /// Panics if `new_cap` is less than the vector's length\n pub fn try_grow(&mut self, new_cap: usize) -> Result<(), CollectionAllocErr> {\n unsafe {\n let unspilled = !self.spilled();\n let (ptr, &mut len, cap) = self.triple_mut();\n assert!(new_cap >= len);\n if new_cap <= Self::inline_capacity() {\n if unspilled {\n return Ok(());\n }\n self.data = SmallVecData::empty();\n ptr::copy_nonoverlapping(ptr.as_ptr(), self.data.inline_mut().as_ptr(), len);\n self.capacity = len;\n deallocate(ptr, cap);\n } else if new_cap != cap {\n let layout = layout_array::(new_cap)?;\n debug_assert!(layout.size() > 0);\n let new_alloc;\n if unspilled {\n new_alloc = NonNull::new(alloc::alloc::alloc(layout))\n .ok_or(CollectionAllocErr::AllocErr { layout })?\n .cast();\n ptr::copy_nonoverlapping(ptr.as_ptr(), new_alloc.as_ptr(), len);\n } else {\n // This should never fail since the same succeeded\n // when previously allocating `ptr`.\n let old_layout = layout_array::(cap)?;\n\n let new_ptr =\n alloc::alloc::realloc(ptr.as_ptr() as *mut u8, old_layout, layout.size());\n new_alloc = NonNull::new(new_ptr)\n .ok_or(CollectionAllocErr::AllocErr { layout })?\n .cast();\n }\n self.data = SmallVecData::from_heap(new_alloc, len);\n self.capacity = new_cap;\n }\n Ok(())\n }\n }\n\n /// Reserve capacity for `additional` more elements to be inserted.\n ///\n /// May reserve more space to avoid frequent reallocations.\n ///\n /// Panics if the capacity computation overflows `usize`.\n #[inline]\n pub fn reserve(&mut self, additional: usize) {\n infallible(self.try_reserve(additional))\n }\n\n /// Internal method used to grow in push() and insert(), where we know\n /// already we have to grow.\n #[cold]\n fn reserve_one_unchecked(&mut self) {\n debug_assert_eq!(self.len(), self.capacity());\n let new_cap = self\n .len()\n .checked_add(1)\n .and_then(usize::checked_next_power_of_two)\n .expect(\"capacity overflow\");\n infallible(self.try_grow(new_cap))\n }\n\n /// Reserve capacity for `additional` more elements to be inserted.\n ///\n /// May reserve more space to avoid frequent reallocations.\n pub fn try_reserve(&mut self, additional: usize) -> Result<(), CollectionAllocErr> {\n // prefer triple_mut() even if triple() would work so that the optimizer\n // removes duplicated calls to it from callers.\n let (_, &mut len, cap) = self.triple_mut();\n if cap - len >= additional {\n return Ok(());\n }\n let new_cap = len\n .checked_add(additional)\n .and_then(usize::checked_next_power_of_two)\n .ok_or(CollectionAllocErr::CapacityOverflow)?;\n self.try_grow(new_cap)\n }\n\n /// Reserve the minimum capacity for `additional` more elements to be\n /// inserted.\n ///\n /// Panics if the new capacity overflows `usize`.\n pub fn reserve_exact(&mut self, additional: usize) {\n infallible(self.try_reserve_exact(additional))\n }\n\n /// Reserve the minimum capacity for `additional` more elements to be\n /// inserted.\n pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), CollectionAllocErr> {\n let (_, &mut len, cap) = self.triple_mut();\n if cap - len >= additional {\n return Ok(());\n }\n let new_cap = len\n .checked_add(additional)\n .ok_or(CollectionAllocErr::CapacityOverflow)?;\n self.try_grow(new_cap)\n }\n\n /// Shrink the capacity of the vector as much as possible.\n ///\n /// When possible, this will move data from an external heap buffer to the\n /// vector's inline storage.\n pub fn shrink_to_fit(&mut self) {\n if !self.spilled() {\n return;\n }\n let len = self.len();\n if self.inline_size() >= len {\n unsafe {\n let (ptr, len) = self.data.heap();\n self.data = SmallVecData::empty();\n ptr::copy_nonoverlapping(ptr.as_ptr(), self.data.inline_mut().as_ptr(), len);\n deallocate(ptr.0, self.capacity);\n self.capacity = len;\n }\n } else if self.capacity() > len {\n self.grow(len);\n }\n }\n\n /// Shorten the vector, keeping the first `len` elements and dropping the\n /// rest.\n ///\n /// If `len` is greater than or equal to the vector's current length, this\n /// has no effect.\n ///\n /// This does not re-allocate. If you want the vector's capacity to shrink,\n /// call `shrink_to_fit` after truncating.\n pub fn truncate(&mut self, len: usize) {\n unsafe {\n let (ptr, len_ptr, _) = self.triple_mut();\n let ptr = ptr.as_ptr();\n while len < *len_ptr {\n let last_index = *len_ptr - 1;\n *len_ptr = last_index;\n ptr::drop_in_place(ptr.add(last_index));\n }\n }\n }\n\n /// Extracts a slice containing the entire vector.\n ///\n /// Equivalent to `&s[..]`.\n pub fn as_slice(&self) -> &[A::Item] {\n self\n }\n\n /// Extracts a mutable slice of the entire vector.\n ///\n /// Equivalent to `&mut s[..]`.\n pub fn as_mut_slice(&mut self) -> &mut [A::Item] {\n self\n }\n\n /// Remove the element at position `index`, replacing it with the last\n /// element.\n ///\n /// This does not preserve ordering, but is O(1).\n ///\n /// Panics if `index` is out of bounds.\n #[inline]\n pub fn swap_remove(&mut self, index: usize) -> A::Item {\n let len = self.len();\n self.swap(len - 1, index);\n self.pop()\n .unwrap_or_else(|| unsafe { unreachable_unchecked() })\n }\n\n /// Remove all elements from the vector.\n #[inline]\n pub fn clear(&mut self) {\n self.truncate(0);\n }\n\n /// Remove and return the element at position `index`, shifting all elements\n /// after it to the left.\n ///\n /// Panics if `index` is out of bounds.\n pub fn remove(&mut self, index: usize) -> A::Item {\n unsafe {\n let (ptr, len_ptr, _) = self.triple_mut();\n let len = *len_ptr;\n assert!(index < len);\n *len_ptr = len - 1;\n let ptr = ptr.as_ptr().add(index);\n let item = ptr::read(ptr);\n ptr::copy(ptr.add(1), ptr, len - index - 1);\n item\n }\n }\n\n /// Insert an element at position `index`, shifting all elements after it to\n /// the right.\n ///\n /// Panics if `index > len`.\n pub fn insert(&mut self, index: usize, element: A::Item) {\n unsafe {\n let (mut ptr, mut len_ptr, cap) = self.triple_mut();\n if *len_ptr == cap {\n self.reserve_one_unchecked();\n let (heap_ptr, heap_len_ptr) = self.data.heap_mut();\n ptr = heap_ptr;\n len_ptr = heap_len_ptr;\n }\n let mut ptr = ptr.as_ptr();\n let len = *len_ptr;\n if index > len {\n panic!(\"index exceeds length\");\n }\n // SAFETY: add is UB if index > len, but we panicked first\n ptr = ptr.add(index);\n if index < len {\n // Shift element to the right of `index`.\n ptr::copy(ptr, ptr.add(1), len - index);\n }\n *len_ptr = len + 1;\n ptr::write(ptr, element);\n }\n }\n\n /// Insert multiple elements at position `index`, shifting all following\n /// elements toward the back.\n pub fn insert_many>(&mut self, index: usize, iterable: I) {\n let mut iter = iterable.into_iter();\n if index == self.len() {\n return self.extend(iter);\n }\n\n let (lower_size_bound, _) = iter.size_hint();\n #[allow(deprecated)]\n {\n assert!(lower_size_bound <= core::isize::MAX as usize)\n } // Ensure offset is indexable\n assert!(index + lower_size_bound >= index); // Protect against overflow\n\n let mut num_added = 0;\n let old_len = self.len();\n assert!(index <= old_len);\n\n unsafe {\n // Reserve space for `lower_size_bound` elements.\n self.reserve(lower_size_bound);\n let start = self.as_mut_ptr();\n let ptr = start.add(index);\n\n // Move the trailing elements.\n ptr::copy(ptr, ptr.add(lower_size_bound), old_len - index);\n\n // In case the iterator panics, don't double-drop the items we just\n // copied above.\n self.set_len(0);\n let mut guard = DropOnPanic {\n start,\n skip: index..(index + lower_size_bound),\n len: old_len + lower_size_bound,\n };\n\n // The set_len above invalidates the previous pointers, so we must\n // re-create them.\n let start = self.as_mut_ptr();\n let ptr = start.add(index);\n\n while num_added < lower_size_bound {\n let element = match iter.next() {\n Some(x) => x,\n None => break,\n };\n let cur = ptr.add(num_added);\n ptr::write(cur, element);\n guard.skip.start += 1;\n num_added += 1;\n }\n\n if num_added < lower_size_bound {\n // Iterator provided fewer elements than the hint. Move the tail\n // backward.\n ptr::copy(\n ptr.add(lower_size_bound),\n ptr.add(num_added),\n old_len - index,\n );\n }\n // There are no more duplicate or uninitialized slots, so the guard\n // is not needed.\n self.set_len(old_len + num_added);\n mem::forget(guard);\n }\n\n // Insert any remaining elements one-by-one.\n for element in iter {\n self.insert(index + num_added, element);\n num_added += 1;\n }\n\n struct DropOnPanic {\n start: *mut T,\n skip: Range, // Space we copied-out-of, but haven't written-to yet.\n len: usize,\n }\n\n impl Drop for DropOnPanic {\n fn drop(&mut self) {\n for i in 0..self.len {\n if !self.skip.contains(&i) {\n unsafe {\n ptr::drop_in_place(self.start.add(i));\n }\n }\n }\n }\n }\n }\n\n /// Convert a `SmallVec` to a `Vec`, without reallocating if the `SmallVec`\n /// has already spilled onto the heap.\n pub fn into_vec(mut self) -> Vec {\n if self.spilled() {\n unsafe {\n let (ptr, &mut len) = self.data.heap_mut();\n let v = Vec::from_raw_parts(ptr.as_ptr(), len, self.capacity);\n mem::forget(self);\n v\n }\n } else {\n self.into_iter().collect()\n }\n }\n\n /// Converts a `SmallVec` into a `Box<[T]>` without reallocating if the\n /// `SmallVec` has already spilled onto the heap.\n ///\n /// Note that this will drop any excess capacity.\n pub fn into_boxed_slice(self) -> Box<[A::Item]> {\n self.into_vec().into_boxed_slice()\n }\n\n /// Convert the `SmallVec` into an `A` if possible. Otherwise return\n /// `Err(Self)`.\n ///\n /// This method returns `Err(Self)` if the `SmallVec` is too short (and the\n /// `A` contains uninitialized elements), or if the `SmallVec` is too\n /// long (and all the elements were spilled to the heap).\n pub fn into_inner(self) -> Result {\n if self.spilled() || self.len() != A::size() {\n // Note: A::size, not Self::inline_capacity\n Err(self)\n } else {\n unsafe {\n let data = ptr::read(&self.data);\n mem::forget(self);\n Ok(data.into_inline().assume_init())\n }\n }\n }\n\n /// Retains only the elements specified by the predicate.\n ///\n /// In other words, remove all elements `e` such that `f(&e)` returns\n /// `false`. This method operates in place and preserves the order of\n /// the retained elements.\n pub fn retain bool>(&mut self, mut f: F) {\n let mut del = 0;\n let len = self.len();\n for i in 0..len {\n if !f(&mut self[i]) {\n del += 1;\n } else if del > 0 {\n self.swap(i - del, i);\n }\n }\n self.truncate(len - del);\n }\n\n /// Retains only the elements specified by the predicate.\n ///\n /// This method is identical in behaviour to [`SmallVec::retain`]; it is\n /// included only to maintain api-compatibility with `std::Vec`, where\n /// the methods are separate for historical reasons.\n pub fn retain_mut bool>(&mut self, f: F) {\n self.retain(f)\n }\n\n /// Removes consecutive duplicate elements.\n pub fn dedup(&mut self)\n where\n A::Item: PartialEq,\n {\n self.dedup_by(|a, b| a == b);\n }\n\n /// Removes consecutive duplicate elements using the given equality\n /// relation.\n pub fn dedup_by(&mut self, mut same_bucket: F)\n where\n F: FnMut(&mut A::Item, &mut A::Item) -> bool,\n {\n // See the implementation of Vec::dedup_by in the\n // standard library for an explanation of this algorithm.\n let len = self.len();\n if len <= 1 {\n return;\n }\n\n let ptr = self.as_mut_ptr();\n let mut w: usize = 1;\n\n unsafe {\n for r in 1..len {\n let p_r = ptr.add(r);\n let p_wm1 = ptr.add(w - 1);\n if !same_bucket(&mut *p_r, &mut *p_wm1) {\n if r != w {\n let p_w = p_wm1.add(1);\n mem::swap(&mut *p_r, &mut *p_w);\n }\n w += 1;\n }\n }\n }\n\n self.truncate(w);\n }\n\n /// Removes consecutive elements that map to the same key.\n pub fn dedup_by_key(&mut self, mut key: F)\n where\n F: FnMut(&mut A::Item) -> K,\n K: PartialEq,\n {\n self.dedup_by(|a, b| key(a) == key(b));\n }\n\n /// Resizes the `SmallVec` in-place so that `len` is equal to `new_len`.\n ///\n /// If `new_len` is greater than `len`, the `SmallVec` is extended by the\n /// difference, with each additional slot filled with the result of\n /// calling the closure `f`. The return values from `f` will end up in\n /// the `SmallVec` in the order they have been generated.\n ///\n /// If `new_len` is less than `len`, the `SmallVec` is simply truncated.\n ///\n /// This method uses a closure to create new values on every push. If you'd\n /// rather `Clone` a given value, use `resize`. If you want to use the\n /// `Default` trait to generate values, you can pass\n /// `Default::default()` as the second argument.\n ///\n /// Added for `std::vec::Vec` compatibility (added in Rust 1.33.0)\n ///\n /// ```\n /// # use smallvec::{smallvec, SmallVec};\n /// let mut vec: SmallVec<[_; 4]> = smallvec![1, 2, 3];\n /// vec.resize_with(5, Default::default);\n /// assert_eq!(&*vec, &[1, 2, 3, 0, 0]);\n ///\n /// let mut vec: SmallVec<[_; 4]> = smallvec![];\n /// let mut p = 1;\n /// vec.resize_with(4, || {\n /// p *= 2;\n /// p\n /// });\n /// assert_eq!(&*vec, &[2, 4, 8, 16]);\n /// ```\n pub fn resize_with(&mut self, new_len: usize, f: F)\n where\n F: FnMut() -> A::Item,\n {\n let old_len = self.len();\n if old_len < new_len {\n let mut f = f;\n let additional = new_len - old_len;\n self.reserve(additional);\n for _ in 0..additional {\n self.push(f());\n }\n } else if old_len > new_len {\n self.truncate(new_len);\n }\n }\n\n /// Creates a `SmallVec` directly from the raw components of another\n /// `SmallVec`.\n ///\n /// # Safety\n ///\n /// This is highly unsafe, due to the number of invariants that aren't\n /// checked:\n ///\n /// * `ptr` needs to have been previously allocated via `SmallVec` for its\n /// spilled storage (at least, it's highly likely to be incorrect if it\n /// wasn't).\n /// * `ptr`'s `A::Item` type needs to be the same size and alignment that it\n /// was allocated with\n /// * `length` needs to be less than or equal to `capacity`.\n /// * `capacity` needs to be the capacity that the pointer was allocated\n /// with.\n ///\n /// Violating these may cause problems like corrupting the allocator's\n /// internal data structures.\n ///\n /// Additionally, `capacity` must be greater than the amount of inline\n /// storage `A` has; that is, the new `SmallVec` must need to spill over\n /// into heap allocated storage. This condition is asserted against.\n ///\n /// The ownership of `ptr` is effectively transferred to the\n /// `SmallVec` which may then deallocate, reallocate or change the\n /// contents of memory pointed to by the pointer at will. Ensure\n /// that nothing else uses the pointer after calling this\n /// function.\n ///\n /// # Examples\n ///\n /// ```\n /// # use smallvec::{smallvec, SmallVec};\n /// use std::mem;\n /// use std::ptr;\n ///\n /// fn main() {\n /// let mut v: SmallVec<[_; 1]> = smallvec![1, 2, 3];\n ///\n /// // Pull out the important parts of `v`.\n /// let p = v.as_mut_ptr();\n /// let len = v.len();\n /// let cap = v.capacity();\n /// let spilled = v.spilled();\n ///\n /// unsafe {\n /// // Forget all about `v`. The heap allocation that stored the\n /// // three values won't be deallocated.\n /// mem::forget(v);\n ///\n /// // Overwrite memory with [4, 5, 6].\n /// //\n /// // This is only safe if `spilled` is true! Otherwise, we are\n /// // writing into the old `SmallVec`'s inline storage on the\n /// // stack.\n /// assert!(spilled);\n /// for i in 0..len {\n /// ptr::write(p.add(i), 4 + i);\n /// }\n ///\n /// // Put everything back together into a SmallVec with a different\n /// // amount of inline storage, but which is still less than `cap`.\n /// let rebuilt = SmallVec::<[_; 2]>::from_raw_parts(p, len, cap);\n /// assert_eq!(&*rebuilt, &[4, 5, 6]);\n /// }\n /// }\n #[inline]\n pub unsafe fn from_raw_parts(ptr: *mut A::Item, length: usize, capacity: usize) -> SmallVec {\n // SAFETY: We require caller to provide same ptr as we alloc\n // and we never alloc null pointer.\n let ptr = unsafe {\n debug_assert!(!ptr.is_null(), \"Called `from_raw_parts` with null pointer.\");\n NonNull::new_unchecked(ptr)\n };\n assert!(capacity > Self::inline_capacity());\n SmallVec {\n capacity,\n data: SmallVecData::from_heap(ptr, length),\n }\n }\n\n /// Returns a raw pointer to the vector's buffer.\n pub fn as_ptr(&self) -> *const A::Item {\n // We shadow the slice method of the same name to avoid going through\n // `deref`, which creates an intermediate reference that may place\n // additional safety constraints on the contents of the slice.\n self.triple().0.as_ptr()\n }\n\n /// Returns a raw mutable pointer to the vector's buffer.\n pub fn as_mut_ptr(&mut self) -> *mut A::Item {\n // We shadow the slice method of the same name to avoid going through\n // `deref_mut`, which creates an intermediate reference that may place\n // additional safety constraints on the contents of the slice.\n self.triple_mut().0.as_ptr()\n }\n}\n\nimpl SmallVec\nwhere\n A::Item: Copy,\n{\n /// Copy the elements from a slice into a new `SmallVec`.\n ///\n /// For slices of `Copy` types, this is more efficient than\n /// `SmallVec::from(slice)`.\n pub fn from_slice(slice: &[A::Item]) -> Self {\n let len = slice.len();\n if len <= Self::inline_capacity() {\n SmallVec {\n capacity: len,\n data: SmallVecData::from_inline(unsafe {\n let mut data: MaybeUninit = MaybeUninit::uninit();\n ptr::copy_nonoverlapping(\n slice.as_ptr(),\n data.as_mut_ptr() as *mut A::Item,\n len,\n );\n data\n }),\n }\n } else {\n let mut b = slice.to_vec();\n let cap = b.capacity();\n let ptr = NonNull::new(b.as_mut_ptr()).expect(\"Vec always contain non null pointers.\");\n mem::forget(b);\n SmallVec {\n capacity: cap,\n data: SmallVecData::from_heap(ptr, len),\n }\n }\n }\n\n /// Copy elements from a slice into the vector at position `index`, shifting\n /// any following elements toward the back.\n ///\n /// For slices of `Copy` types, this is more efficient than `insert`.\n #[inline]\n pub fn insert_from_slice(&mut self, index: usize, slice: &[A::Item]) {\n self.reserve(slice.len());\n\n let len = self.len();\n assert!(index <= len);\n\n unsafe {\n let slice_ptr = slice.as_ptr();\n let ptr = self.as_mut_ptr().add(index);\n ptr::copy(ptr, ptr.add(slice.len()), len - index);\n ptr::copy_nonoverlapping(slice_ptr, ptr, slice.len());\n self.set_len(len + slice.len());\n }\n }\n\n /// Copy elements from a slice and append them to the vector.\n ///\n /// For slices of `Copy` types, this is more efficient than `extend`.\n #[inline]\n pub fn extend_from_slice(&mut self, slice: &[A::Item]) {\n let len = self.len();\n self.insert_from_slice(len, slice);\n }\n}\n\nimpl SmallVec\nwhere\n A::Item: Clone,\n{\n /// Resizes the vector so that its length is equal to `len`.\n ///\n /// If `len` is less than the current length, the vector simply truncated.\n ///\n /// If `len` is greater than the current length, `value` is appended to the\n /// vector until its length equals `len`.\n pub fn resize(&mut self, len: usize, value: A::Item) {\n let old_len = self.len();\n\n if len > old_len {\n self.extend(repeat(value).take(len - old_len));\n } else {\n self.truncate(len);\n }\n }\n\n /// Creates a `SmallVec` with `n` copies of `elem`.\n /// ```\n /// use smallvec::SmallVec;\n ///\n /// let v = SmallVec::<[char; 128]>::from_elem('d', 2);\n /// assert_eq!(v, SmallVec::from_buf(['d', 'd']));\n /// ```\n pub fn from_elem(elem: A::Item, n: usize) -> Self {\n if n > Self::inline_capacity() {\n vec![elem; n].into()\n } else {\n let mut v = SmallVec::::new();\n unsafe {\n let (ptr, len_ptr, _) = v.triple_mut();\n let ptr = ptr.as_ptr();\n let mut local_len = SetLenOnDrop::new(len_ptr);\n\n for i in 0..n {\n ::core::ptr::write(ptr.add(i), elem.clone());\n local_len.increment_len(1);\n }\n }\n v\n }\n }\n}\n\nimpl ops::Deref for SmallVec {\n type Target = [A::Item];\n #[inline]\n fn deref(&self) -> &[A::Item] {\n unsafe {\n let (ptr, len, _) = self.triple();\n slice::from_raw_parts(ptr.as_ptr(), len)\n }\n }\n}\n\nimpl ops::DerefMut for SmallVec {\n #[inline]\n fn deref_mut(&mut self) -> &mut [A::Item] {\n unsafe {\n let (ptr, &mut len, _) = self.triple_mut();\n slice::from_raw_parts_mut(ptr.as_ptr(), len)\n }\n }\n}\n\nimpl AsRef<[A::Item]> for SmallVec {\n #[inline]\n fn as_ref(&self) -> &[A::Item] {\n self\n }\n}\n\nimpl AsMut<[A::Item]> for SmallVec {\n #[inline]\n fn as_mut(&mut self) -> &mut [A::Item] {\n self\n }\n}\n\nimpl Borrow<[A::Item]> for SmallVec {\n #[inline]\n fn borrow(&self) -> &[A::Item] {\n self\n }\n}\n\nimpl BorrowMut<[A::Item]> for SmallVec {\n #[inline]\n fn borrow_mut(&mut self) -> &mut [A::Item] {\n self\n }\n}\n\n#[cfg(feature = \"write\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"write\")))]\nimpl> io::Write for SmallVec {\n #[inline]\n fn write(&mut self, buf: &[u8]) -> io::Result {\n self.extend_from_slice(buf);\n Ok(buf.len())\n }\n\n #[inline]\n fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {\n self.extend_from_slice(buf);\n Ok(())\n }\n\n #[inline]\n fn flush(&mut self) -> io::Result<()> {\n Ok(())\n }\n}\n\n#[cfg(feature = \"serde\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"serde\")))]\nimpl Serialize for SmallVec\nwhere\n A::Item: Serialize,\n{\n fn serialize(&self, serializer: S) -> Result {\n let mut state = serializer.serialize_seq(Some(self.len()))?;\n for item in self {\n state.serialize_element(&item)?;\n }\n state.end()\n }\n}\n\n#[cfg(feature = \"serde\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"serde\")))]\nimpl<'de, A: Array> Deserialize<'de> for SmallVec\nwhere\n A::Item: Deserialize<'de>,\n{\n fn deserialize>(deserializer: D) -> Result {\n deserializer.deserialize_seq(SmallVecVisitor {\n phantom: PhantomData,\n })\n }\n}\n\n#[cfg(feature = \"serde\")]\nstruct SmallVecVisitor {\n phantom: PhantomData,\n}\n\n#[cfg(feature = \"serde\")]\nimpl<'de, A: Array> Visitor<'de> for SmallVecVisitor\nwhere\n A::Item: Deserialize<'de>,\n{\n type Value = SmallVec;\n\n fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {\n formatter.write_str(\"a sequence\")\n }\n\n fn visit_seq(self, mut seq: B) -> Result\n where\n B: SeqAccess<'de>,\n {\n use serde::de::Error;\n let len = seq.size_hint().unwrap_or(0);\n let mut values = SmallVec::new();\n values.try_reserve(len).map_err(B::Error::custom)?;\n\n while let Some(value) = seq.next_element()? {\n values.push(value);\n }\n\n Ok(values)\n }\n}\n\n#[cfg(feature = \"malloc_size_of\")]\nimpl MallocShallowSizeOf for SmallVec {\n fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize {\n if self.spilled() {\n unsafe { ops.malloc_size_of(self.as_ptr()) }\n } else {\n 0\n }\n }\n}\n\n#[cfg(feature = \"malloc_size_of\")]\nimpl MallocSizeOf for SmallVec\nwhere\n A: Array,\n A::Item: MallocSizeOf,\n{\n fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize {\n let mut n = self.shallow_size_of(ops);\n for elem in self.iter() {\n n += elem.size_of(ops);\n }\n n\n }\n}\n\n#[cfg(feature = \"specialization\")]\ntrait SpecFrom {\n fn spec_from(slice: S) -> SmallVec;\n}\n\n#[cfg(feature = \"specialization\")]\nmod specialization;\n\n#[cfg(feature = \"arbitrary\")]\nmod arbitrary;\n\n#[cfg(feature = \"specialization\")]\nimpl<'a, A: Array> SpecFrom for SmallVec\nwhere\n A::Item: Copy,\n{\n #[inline]\n fn spec_from(slice: &'a [A::Item]) -> SmallVec {\n SmallVec::from_slice(slice)\n }\n}\n\nimpl<'a, A: Array> From<&'a [A::Item]> for SmallVec\nwhere\n A::Item: Clone,\n{\n #[cfg(not(feature = \"specialization\"))]\n #[inline]\n fn from(slice: &'a [A::Item]) -> SmallVec {\n slice.iter().cloned().collect()\n }\n\n #[cfg(feature = \"specialization\")]\n #[inline]\n fn from(slice: &'a [A::Item]) -> SmallVec {\n SmallVec::spec_from(slice)\n }\n}\n\nimpl From> for SmallVec {\n #[inline]\n fn from(vec: Vec) -> SmallVec {\n SmallVec::from_vec(vec)\n }\n}\n\nimpl From for SmallVec {\n #[inline]\n fn from(array: A) -> SmallVec {\n SmallVec::from_buf(array)\n }\n}\n\nimpl> ops::Index for SmallVec {\n type Output = I::Output;\n\n fn index(&self, index: I) -> &I::Output {\n &(**self)[index]\n }\n}\n\nimpl> ops::IndexMut for SmallVec {\n fn index_mut(&mut self, index: I) -> &mut I::Output {\n &mut (&mut **self)[index]\n }\n}\n\n#[allow(deprecated)]\nimpl ExtendFromSlice for SmallVec\nwhere\n A::Item: Copy,\n{\n fn extend_from_slice(&mut self, other: &[A::Item]) {\n SmallVec::extend_from_slice(self, other)\n }\n}\n\nimpl FromIterator for SmallVec {\n #[inline]\n fn from_iter>(iterable: I) -> SmallVec {\n let mut v = SmallVec::new();\n v.extend(iterable);\n v\n }\n}\n\nimpl Extend for SmallVec {\n fn extend>(&mut self, iterable: I) {\n let mut iter = iterable.into_iter();\n let (lower_size_bound, _) = iter.size_hint();\n self.reserve(lower_size_bound);\n\n unsafe {\n let (ptr, len_ptr, cap) = self.triple_mut();\n let ptr = ptr.as_ptr();\n let mut len = SetLenOnDrop::new(len_ptr);\n while len.get() < cap {\n if let Some(out) = iter.next() {\n ptr::write(ptr.add(len.get()), out);\n len.increment_len(1);\n } else {\n return;\n }\n }\n }\n\n for elem in iter {\n self.push(elem);\n }\n }\n}\n\nimpl fmt::Debug for SmallVec\nwhere\n A::Item: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_list().entries(self.iter()).finish()\n }\n}\n\nimpl Default for SmallVec {\n #[inline]\n fn default() -> SmallVec {\n SmallVec::new()\n }\n}\n\n#[cfg(feature = \"may_dangle\")]\nunsafe impl<#[may_dangle] A: Array> Drop for SmallVec {\n fn drop(&mut self) {\n unsafe {\n if self.spilled() {\n let (ptr, &mut len) = self.data.heap_mut();\n Vec::from_raw_parts(ptr.as_ptr(), len, self.capacity);\n } else {\n ptr::drop_in_place(&mut self[..]);\n }\n }\n }\n}\n\n#[cfg(not(feature = \"may_dangle\"))]\nimpl Drop for SmallVec {\n fn drop(&mut self) {\n unsafe {\n if self.spilled() {\n let (ptr, &mut len) = self.data.heap_mut();\n drop(Vec::from_raw_parts(ptr.as_ptr(), len, self.capacity));\n } else {\n ptr::drop_in_place(&mut self[..]);\n }\n }\n }\n}\n\nimpl Clone for SmallVec\nwhere\n A::Item: Clone,\n{\n #[inline]\n fn clone(&self) -> SmallVec {\n SmallVec::from(self.as_slice())\n }\n\n fn clone_from(&mut self, source: &Self) {\n // Inspired from `impl Clone for Vec`.\n\n // drop anything that will not be overwritten\n self.truncate(source.len());\n\n // self.len <= other.len due to the truncate above, so the\n // slices here are always in-bounds.\n let (init, tail) = source.split_at(self.len());\n\n // reuse the contained values' allocations/resources.\n self.clone_from_slice(init);\n self.extend(tail.iter().cloned());\n }\n}\n\nimpl PartialEq> for SmallVec\nwhere\n A::Item: PartialEq,\n{\n #[inline]\n fn eq(&self, other: &SmallVec) -> bool {\n self[..] == other[..]\n }\n}\n\nimpl Eq for SmallVec where A::Item: Eq {}\n\nimpl PartialOrd for SmallVec\nwhere\n A::Item: PartialOrd,\n{\n #[inline]\n fn partial_cmp(&self, other: &SmallVec) -> Option {\n PartialOrd::partial_cmp(&**self, &**other)\n }\n}\n\nimpl Ord for SmallVec\nwhere\n A::Item: Ord,\n{\n #[inline]\n fn cmp(&self, other: &SmallVec) -> cmp::Ordering {\n Ord::cmp(&**self, &**other)\n }\n}\n\nimpl Hash for SmallVec\nwhere\n A::Item: Hash,\n{\n fn hash(&self, state: &mut H) {\n (**self).hash(state)\n }\n}\n\nunsafe impl Send for SmallVec where A::Item: Send {}\n\n/// An iterator that consumes a `SmallVec` and yields its items by value.\n///\n/// Returned from [`SmallVec::into_iter`][1].\n///\n/// [1]: struct.SmallVec.html#method.into_iter\npub struct IntoIter {\n data: SmallVec,\n current: usize,\n end: usize,\n}\n\nimpl fmt::Debug for IntoIter\nwhere\n A::Item: fmt::Debug,\n{\n fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n f.debug_tuple(\"IntoIter\").field(&self.as_slice()).finish()\n }\n}\n\nimpl Clone for IntoIter\nwhere\n A::Item: Clone,\n{\n fn clone(&self) -> IntoIter {\n SmallVec::from(self.as_slice()).into_iter()\n }\n}\n\nimpl Drop for IntoIter {\n fn drop(&mut self) {\n for _ in self {}\n }\n}\n\nimpl Iterator for IntoIter {\n type Item = A::Item;\n\n #[inline]\n fn next(&mut self) -> Option {\n if self.current == self.end {\n None\n } else {\n unsafe {\n let current = self.current;\n self.current += 1;\n Some(ptr::read(self.data.as_ptr().add(current)))\n }\n }\n }\n\n #[inline]\n fn size_hint(&self) -> (usize, Option) {\n let size = self.end - self.current;\n (size, Some(size))\n }\n}\n\nimpl DoubleEndedIterator for IntoIter {\n #[inline]\n fn next_back(&mut self) -> Option {\n if self.current == self.end {\n None\n } else {\n unsafe {\n self.end -= 1;\n Some(ptr::read(self.data.as_ptr().add(self.end)))\n }\n }\n }\n}\n\nimpl ExactSizeIterator for IntoIter {}\nimpl FusedIterator for IntoIter {}\n\nimpl IntoIter {\n /// Returns the remaining items of this iterator as a slice.\n pub fn as_slice(&self) -> &[A::Item] {\n let len = self.end - self.current;\n unsafe { core::slice::from_raw_parts(self.data.as_ptr().add(self.current), len) }\n }\n\n /// Returns the remaining items of this iterator as a mutable slice.\n pub fn as_mut_slice(&mut self) -> &mut [A::Item] {\n let len = self.end - self.current;\n unsafe { core::slice::from_raw_parts_mut(self.data.as_mut_ptr().add(self.current), len) }\n }\n}\n\nimpl IntoIterator for SmallVec {\n type IntoIter = IntoIter;\n type Item = A::Item;\n fn into_iter(mut self) -> Self::IntoIter {\n unsafe {\n // Set SmallVec len to zero as `IntoIter` drop handles dropping of\n // the elements\n let len = self.len();\n self.set_len(0);\n IntoIter {\n data: self,\n current: 0,\n end: len,\n }\n }\n }\n}\n\nimpl<'a, A: Array> IntoIterator for &'a SmallVec {\n type IntoIter = slice::Iter<'a, A::Item>;\n type Item = &'a A::Item;\n fn into_iter(self) -> Self::IntoIter {\n self.iter()\n }\n}\n\nimpl<'a, A: Array> IntoIterator for &'a mut SmallVec {\n type IntoIter = slice::IterMut<'a, A::Item>;\n type Item = &'a mut A::Item;\n fn into_iter(self) -> Self::IntoIter {\n self.iter_mut()\n }\n}\n\n/// Types that can be used as the backing store for a [`SmallVec`].\npub unsafe trait Array {\n /// The type of the array's elements.\n type Item;\n /// Returns the number of items the array can hold.\n fn size() -> usize;\n}\n\n/// Set the length of the vec when the `SetLenOnDrop` value goes out of scope.\n///\n/// Copied from \nstruct SetLenOnDrop<'a> {\n len: &'a mut usize,\n local_len: usize,\n}\n\nimpl<'a> SetLenOnDrop<'a> {\n #[inline]\n fn new(len: &'a mut usize) -> Self {\n SetLenOnDrop {\n local_len: *len,\n len,\n }\n }\n\n #[inline]\n fn get(&self) -> usize {\n self.local_len\n }\n\n #[inline]\n fn increment_len(&mut self, increment: usize) {\n self.local_len += increment;\n }\n}\n\nimpl<'a> Drop for SetLenOnDrop<'a> {\n #[inline]\n fn drop(&mut self) {\n *self.len = self.local_len;\n }\n}\n\n#[cfg(feature = \"const_new\")]\nimpl SmallVec<[T; N]> {\n /// Construct an empty vector.\n ///\n /// This is a `const` version of [`SmallVec::new`] that is enabled by the\n /// feature `const_new`, with the limitation that it only works for arrays.\n #[cfg_attr(docsrs, doc(cfg(feature = \"const_new\")))]\n #[inline]\n pub const fn new_const() -> Self {\n SmallVec {\n capacity: 0,\n data: SmallVecData::from_const(MaybeUninit::uninit()),\n }\n }\n\n /// The array passed as an argument is moved to be an inline version of\n /// `SmallVec`.\n ///\n /// This is a `const` version of [`SmallVec::from_buf`] that is enabled by\n /// the feature `const_new`, with the limitation that it only works for\n /// arrays.\n #[cfg_attr(docsrs, doc(cfg(feature = \"const_new\")))]\n #[inline]\n pub const fn from_const(items: [T; N]) -> Self {\n SmallVec {\n capacity: N,\n data: SmallVecData::from_const(MaybeUninit::new(items)),\n }\n }\n\n /// Constructs a new `SmallVec` on the stack from an array without\n /// copying elements. Also sets the length. The user is responsible\n /// for ensuring that `len <= N`.\n ///\n /// This is a `const` version of [`SmallVec::from_buf_and_len_unchecked`]\n /// that is enabled by the feature `const_new`, with the limitation that it\n /// only works for arrays.\n #[cfg_attr(docsrs, doc(cfg(feature = \"const_new\")))]\n #[inline]\n pub const unsafe fn from_const_with_len_unchecked(items: [T; N], len: usize) -> Self {\n SmallVec {\n capacity: len,\n data: SmallVecData::from_const(MaybeUninit::new(items)),\n }\n }\n}\n\n#[cfg(feature = \"const_generics\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"const_generics\")))]\nunsafe impl Array for [T; N] {\n type Item = T;\n #[inline]\n fn size() -> usize {\n N\n }\n}\n\n#[cfg(not(feature = \"const_generics\"))]\nmacro_rules! impl_array(\n ($($size:expr),+) => {\n $(\n unsafe impl Array for [T; $size] {\n type Item = T;\n #[inline]\n fn size() -> usize { $size }\n }\n )+\n }\n);\n\n#[cfg(not(feature = \"const_generics\"))]\nimpl_array!(\n 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,\n 26, 27, 28, 29, 30, 31, 32, 36, 0x40, 0x60, 0x80, 0x100, 0x200, 0x400, 0x600, 0x800, 0x1000,\n 0x2000, 0x4000, 0x6000, 0x8000, 0x10000, 0x20000, 0x40000, 0x60000, 0x80000, 0x10_0000\n);\n\n/// Convenience trait for constructing a `SmallVec`\npub trait ToSmallVec {\n /// Construct a new `SmallVec` from a slice.\n fn to_smallvec(&self) -> SmallVec;\n}\n\nimpl ToSmallVec for [A::Item]\nwhere\n A::Item: Copy,\n{\n #[inline]\n fn to_smallvec(&self) -> SmallVec {\n SmallVec::from_slice(self)\n }\n}\n\n// Immutable counterpart for `NonNull`.\n#[repr(transparent)]\nstruct ConstNonNull(NonNull);\n\nimpl ConstNonNull {\n #[inline]\n fn new(ptr: *const T) -> Option {\n NonNull::new(ptr as *mut T).map(Self)\n }\n #[inline]\n fn as_ptr(self) -> *const T {\n self.0.as_ptr()\n }\n}\n\nimpl Clone for ConstNonNull {\n #[inline]\n fn clone(&self) -> Self {\n *self\n }\n}\n\nimpl Copy for ConstNonNull {}\n\n#[cfg(feature = \"impl_bincode\")]\nuse bincode::{\n de::{read::Reader, BorrowDecoder, Decode, Decoder},\n enc::{write::Writer, Encode, Encoder},\n error::{DecodeError, EncodeError},\n BorrowDecode,\n};\n\n#[cfg(feature = \"impl_bincode\")]\nimpl Decode for SmallVec\nwhere\n A: Array,\n A::Item: Decode,\n{\n fn decode>(decoder: &mut D) -> Result {\n use core::convert::TryInto;\n let len = u64::decode(decoder)?;\n let len = len\n .try_into()\n .map_err(|_| DecodeError::OutsideUsizeRange(len))?;\n decoder.claim_container_read::(len)?;\n\n let mut vec = SmallVec::with_capacity(len);\n if unty::type_equal::() {\n // Initialize the smallvec's buffer. Note that we need to do this\n // through the raw pointer as we cannot name the type\n // [u8; N] even though A::Item is u8.\n let ptr = vec.as_mut_ptr();\n // SAFETY: A::Item is u8 and the smallvec has been allocated with\n // enough capacity\n unsafe {\n core::ptr::write_bytes(ptr, 0, len);\n vec.set_len(len);\n }\n // Read the data into the smallvec's buffer.\n let slice = vec.as_mut_slice();\n // SAFETY: A::Item is u8\n let slice = unsafe { core::mem::transmute::<&mut [A::Item], &mut [u8]>(slice) };\n decoder.reader().read(slice)?;\n } else {\n for _ in 0..len {\n decoder.unclaim_bytes_read(core::mem::size_of::());\n vec.push(A::Item::decode(decoder)?);\n }\n }\n Ok(vec)\n }\n}\n\n#[cfg(feature = \"impl_bincode\")]\nimpl<'de, A, Context> BorrowDecode<'de, Context> for SmallVec\nwhere\n A: Array,\n A::Item: BorrowDecode<'de, Context>,\n{\n fn borrow_decode>(\n decoder: &mut D,\n ) -> Result {\n use core::convert::TryInto;\n let len = u64::decode(decoder)?;\n let len = len\n .try_into()\n .map_err(|_| DecodeError::OutsideUsizeRange(len))?;\n decoder.claim_container_read::(len)?;\n\n let mut vec = SmallVec::with_capacity(len);\n if unty::type_equal::() {\n // Initialize the smallvec's buffer. Note that we need to do this\n // through the raw pointer as we cannot name the type\n // [u8; N] even though A::Item is u8.\n let ptr = vec.as_mut_ptr();\n // SAFETY: A::Item is u8 and the smallvec has been allocated with\n // enough capacity\n unsafe {\n core::ptr::write_bytes(ptr, 0, len);\n vec.set_len(len);\n }\n // Read the data into the smallvec's buffer.\n let slice = vec.as_mut_slice();\n // SAFETY: A::Item is u8\n let slice = unsafe { core::mem::transmute::<&mut [A::Item], &mut [u8]>(slice) };\n decoder.reader().read(slice)?;\n } else {\n for _ in 0..len {\n decoder.unclaim_bytes_read(core::mem::size_of::());\n vec.push(A::Item::borrow_decode(decoder)?);\n }\n }\n Ok(vec)\n }\n}\n\n#[cfg(feature = \"impl_bincode\")]\nimpl Encode for SmallVec\nwhere\n A: Array,\n A::Item: Encode,\n{\n fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> {\n (self.len() as u64).encode(encoder)?;\n if unty::type_equal::() {\n // Safety: A::Item is u8\n let slice: &[u8] = unsafe { core::mem::transmute(self.as_slice()) };\n encoder.writer().write(slice)?;\n } else {\n for item in self.iter() {\n item.encode(encoder)?;\n }\n }\n Ok(())\n }\n}\n" }, { "package": "smallvec@1.16.1", "sourcePath": "src/specialization.rs", "sha256": "46433586203399251cba496d67b88d34e1be3c2b591986b77463513da1c66471", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Implementations that require `default fn`.\n\nuse super::{Array, SmallVec, SpecFrom};\n\nimpl<'a, A: Array> SpecFrom for SmallVec\nwhere\n A::Item: Clone,\n{\n #[inline]\n default fn spec_from(slice: &'a [A::Item]) -> SmallVec {\n slice.into_iter().cloned().collect()\n }\n}\n" }, { "package": "svgtypes@0.11.0", "sourcePath": "README.md", "sha256": "4c84903e4a242563e6f703ae6ee1fd06c0318fc1ee0edcff0a85a14641facd8c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "## svgtypes\n![Build Status](https://github.com/RazrFalcon/svgtypes/workflows/svgtypes/badge.svg)\n[![Crates.io](https://img.shields.io/crates/v/svgtypes.svg)](https://crates.io/crates/svgtypes)\n[![Documentation](https://docs.rs/svgtypes/badge.svg)](https://docs.rs/svgtypes)\n[![Rust 1.65+](https://img.shields.io/badge/rust-1.65+-orange.svg)](https://www.rust-lang.org)\n![](https://img.shields.io/badge/unsafe-forbidden-brightgreen.svg)\n\n*svgtypes* is a collection of parsers for [SVG](https://www.w3.org/TR/SVG2/) types.\n\n### Supported SVG types\n\n- [``](https://www.w3.org/TR/css-color-3/)\n- [``](https://www.w3.org/TR/SVG2/types.html#InterfaceSVGNumber)\n- [``](https://www.w3.org/TR/SVG2/types.html#InterfaceSVGLength)\n- [``](https://www.w3.org/TR/SVG2/types.html#InterfaceSVGAngle)\n- [``](https://www.w3.org/TR/SVG2/coords.html#ViewBoxAttribute)\n- [``](https://www.w3.org/TR/SVG2/paths.html#PathData)\n- [``](https://www.w3.org/TR/SVG11/types.html#DataTypeTransformList)\n- [``](https://www.w3.org/TR/SVG2/types.html#InterfaceSVGNumberList)\n- [``](https://www.w3.org/TR/SVG2/types.html#InterfaceSVGLengthList)\n- [``](https://www.w3.org/TR/SVG11/shapes.html#PointsBNF)\n- [``](https://www.w3.org/TR/filter-effects-1/#typedef-filter-value-list)\n- [``](https://www.w3.org/TR/SVG2/painting.html#SpecifyingPaint)\n- [``](https://www.w3.org/TR/SVG11/coords.html#PreserveAspectRatioAttribute)\n- [``](https://www.w3.org/TR/SVG11/filters.html#EnableBackgroundProperty)\n- [``](https://www.w3.org/TR/SVG11/types.html#DataTypeIRI)\n- [``](https://www.w3.org/TR/SVG11/types.html#DataTypeFuncIRI)\n- [`paint-order`](https://www.w3.org/TR/SVG2/painting.html#PaintOrder)\n\n### Features\n\n- Complete support of paths, so data like `M10-20A5.5.3-4 110-.1` will be parsed correctly.\n- Implicit path commands will be automatically converted into explicit one.\n- Some SVG2 data types support.\n- Pretty fast.\n\n### Limitations\n\n- Accepts only [normalized](https://www.w3.org/TR/REC-xml/#AVNormalize) values,\n e.g. an input text should not contain ` ` or `&data;`.\n- All keywords must be lowercase.\n Case-insensitive parsing is supported only for colors (requires allocation for named colors).\n- The `` followed by the `` is not supported. As the `` itself.\n- [System colors](https://www.w3.org/TR/css3-color/#css2-system), like `fill=\"AppWorkspace\"`,\n are not supported. They were deprecated anyway.\n\n### Safety\n\n- The library should not panic. Any panic considered as a critical bug and should be reported.\n- The library forbids unsafe code.\n\n### Alternatives\n\nNone.\n\n### License\n\nLicensed under either of\n\n- Apache License, Version 2.0\n ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n- MIT license\n ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "package": "svgtypes@0.14.0", "sourcePath": "README.md", "sha256": "6370a2a7fc140bbe84261103248bcf94a1daf4ddf71b475501c9c28016399a9d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "## svgtypes\n![Build Status](https://github.com/RazrFalcon/svgtypes/workflows/svgtypes/badge.svg)\n[![Crates.io](https://img.shields.io/crates/v/svgtypes.svg)](https://crates.io/crates/svgtypes)\n[![Documentation](https://docs.rs/svgtypes/badge.svg)](https://docs.rs/svgtypes)\n[![Rust 1.65+](https://img.shields.io/badge/rust-1.65+-orange.svg)](https://www.rust-lang.org)\n![](https://img.shields.io/badge/unsafe-forbidden-brightgreen.svg)\n\n*svgtypes* is a collection of parsers for [SVG](https://www.w3.org/TR/SVG2/) types.\n\n### Supported SVG types\n\n- [``](https://www.w3.org/TR/css-color-3/)\n- [``](https://www.w3.org/TR/SVG2/types.html#InterfaceSVGNumber)\n- [``](https://www.w3.org/TR/SVG2/types.html#InterfaceSVGLength)\n- [``](https://www.w3.org/TR/SVG2/types.html#InterfaceSVGAngle)\n- [``](https://www.w3.org/TR/SVG2/coords.html#ViewBoxAttribute)\n- [``](https://www.w3.org/TR/SVG2/paths.html#PathData)\n- [``](https://www.w3.org/TR/SVG11/types.html#DataTypeTransformList)\n- [`transform-origin`](https://drafts.csswg.org/css-transforms/#transform-origin-property)\n- [``](https://www.w3.org/TR/SVG2/types.html#InterfaceSVGNumberList)\n- [``](https://www.w3.org/TR/SVG2/types.html#InterfaceSVGLengthList)\n- [``](https://www.w3.org/TR/SVG11/shapes.html#PointsBNF)\n- [``](https://www.w3.org/TR/filter-effects-1/#typedef-filter-value-list)\n- [``](https://www.w3.org/TR/SVG2/painting.html#SpecifyingPaint)\n- [``](https://www.w3.org/TR/SVG11/coords.html#PreserveAspectRatioAttribute)\n- [``](https://www.w3.org/TR/SVG11/filters.html#EnableBackgroundProperty)\n- [``](https://www.w3.org/TR/SVG11/types.html#DataTypeIRI)\n- [``](https://www.w3.org/TR/SVG11/types.html#DataTypeFuncIRI)\n- [`paint-order`](https://www.w3.org/TR/SVG2/painting.html#PaintOrder)\n- [``](https://www.w3.org/TR/2018/REC-css-fonts-3-20180920/#font-family-prop)\n- [`font`](https://www.w3.org/TR/css-fonts-3/#font-prop)\n\n### Features\n\n- Complete support of paths, so data like `M10-20A5.5.3-4 110-.1` will be parsed correctly.\n- Implicit path commands will be automatically converted into explicit one.\n- Some SVG2 data types support.\n- Pretty fast.\n\n### Limitations\n\n- Accepts only [normalized](https://www.w3.org/TR/REC-xml/#AVNormalize) values,\n e.g. an input text should not contain ` ` or `&data;`.\n- All keywords must be lowercase.\n Case-insensitive parsing is supported only for colors (requires allocation for named colors).\n- The `` followed by the `` is not supported. As the `` itself.\n- [System colors](https://www.w3.org/TR/css3-color/#css2-system), like `fill=\"AppWorkspace\"`,\n are not supported. They were deprecated anyway.\n\n### Safety\n\n- The library should not panic. Any panic considered as a critical bug and should be reported.\n- The library forbids unsafe code.\n\n### Alternatives\n\nNone.\n\n### License\n\nLicensed under either of\n\n- Apache License, Version 2.0\n ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n- MIT license\n ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "package": "syn@2.0.119", "sourcePath": "README.md", "sha256": "4a050aa3ee831ad3d27ab15d9b612bb32712f86517fa2e6be0b1cac37137ebe5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Parser for Rust source code\n===========================\n\n[\"github\"](https://github.com/dtolnay/syn)\n[\"crates.io\"](https://crates.io/crates/syn)\n[\"docs.rs\"](https://docs.rs/syn)\n[\"build](https://github.com/dtolnay/syn/actions?query=branch%3Amaster)\n\nSyn is a parsing library for parsing a stream of Rust tokens into a syntax tree\nof Rust source code.\n\nCurrently this library is geared toward use in Rust procedural macros, but\ncontains some APIs that may be useful more generally.\n\n- **Data structures** — Syn provides a complete syntax tree that can represent\n any valid Rust source code. The syntax tree is rooted at [`syn::File`] which\n represents a full source file, but there are other entry points that may be\n useful to procedural macros including [`syn::Item`], [`syn::Expr`] and\n [`syn::Type`].\n\n- **Derives** — Of particular interest to derive macros is [`syn::DeriveInput`]\n which is any of the three legal input items to a derive macro. An example\n below shows using this type in a library that can derive implementations of a\n user-defined trait.\n\n- **Parsing** — Parsing in Syn is built around [parser functions] with the\n signature `fn(ParseStream) -> Result`. Every syntax tree node defined by\n Syn is individually parsable and may be used as a building block for custom\n syntaxes, or you may dream up your own brand new syntax without involving any\n of our syntax tree types.\n\n- **Location information** — Every token parsed by Syn is associated with a\n `Span` that tracks line and column information back to the source of that\n token. These spans allow a procedural macro to display detailed error messages\n pointing to all the right places in the user's code. There is an example of\n this below.\n\n- **Feature flags** — Functionality is aggressively feature gated so your\n procedural macros enable only what they need, and do not pay in compile time\n for all the rest.\n\n[`syn::File`]: https://docs.rs/syn/2.0/syn/struct.File.html\n[`syn::Item`]: https://docs.rs/syn/2.0/syn/enum.Item.html\n[`syn::Expr`]: https://docs.rs/syn/2.0/syn/enum.Expr.html\n[`syn::Type`]: https://docs.rs/syn/2.0/syn/enum.Type.html\n[`syn::DeriveInput`]: https://docs.rs/syn/2.0/syn/struct.DeriveInput.html\n[parser functions]: https://docs.rs/syn/2.0/syn/parse/index.html\n\n[*Release notes*](https://github.com/dtolnay/syn/releases)\n\n
\n\n## Resources\n\nThe best way to learn about procedural macros is by writing some. Consider\nworking through [this procedural macro workshop][workshop] to get familiar with\nthe different types of procedural macros. The workshop contains relevant links\ninto the Syn documentation as you work through each project.\n\n[workshop]: https://github.com/dtolnay/proc-macro-workshop\n\n
\n\n## Example of a derive macro\n\nThe canonical derive macro using Syn looks like this. We write an ordinary Rust\nfunction tagged with a `proc_macro_derive` attribute and the name of the trait\nwe are deriving. Any time that derive appears in the user's code, the Rust\ncompiler passes their data structure as tokens into our macro. We get to execute\narbitrary Rust code to figure out what to do with those tokens, then hand some\ntokens back to the compiler to compile into the user's crate.\n\n[`TokenStream`]: https://doc.rust-lang.org/proc_macro/struct.TokenStream.html\n\n```toml\n[dependencies]\nsyn = \"2.0\"\nquote = \"1.0\"\n\n[lib]\nproc-macro = true\n```\n\n```rust\nuse proc_macro::TokenStream;\nuse quote::quote;\nuse syn::{parse_macro_input, DeriveInput};\n\n#[proc_macro_derive(MyMacro)]\npub fn my_macro(input: TokenStream) -> TokenStream {\n // Parse the input tokens into a syntax tree\n let input = parse_macro_input!(input as DeriveInput);\n\n // Build the output, possibly using quasi-quotation\n let expanded = quote! {\n // ...\n };\n\n // Hand the output tokens back to the compiler\n TokenStream::from(expanded)\n}\n```\n\nThe [`heapsize`] example directory shows a complete working implementation of a\nderive macro. The example derives a `HeapSize` trait which computes an estimate\nof the amount of heap memory owned by a value.\n\n[`heapsize`]: examples/heapsize\n\n```rust\npub trait HeapSize {\n /// Total number of bytes of heap memory owned by `self`.\n fn heap_size_of_children(&self) -> usize;\n}\n```\n\nThe derive macro allows users to write `#[derive(HeapSize)]` on data structures\nin their program.\n\n```rust\n#[derive(HeapSize)]\nstruct Demo<'a, T: ?Sized> {\n a: Box,\n b: u8,\n c: &'a str,\n d: String,\n}\n```\n\n
\n\n## Spans and error reporting\n\nThe token-based procedural macro API provides great control over where the\ncompiler's error messages are displayed in user code. Consider the error the\nuser sees if one of their field types does not implement `HeapSize`.\n\n```rust\n#[derive(HeapSize)]\nstruct Broken {\n ok: String,\n bad: std::thread::Thread,\n}\n```\n\nBy tracking span information all the way through the expansion of a procedural\nmacro as shown in the `heapsize` example, token-based macros in Syn are able to\ntrigger errors that directly pinpoint the source of the problem.\n\n```console\nerror[E0277]: the trait bound `std::thread::Thread: HeapSize` is not satisfied\n --> src/main.rs:7:5\n |\n7 | bad: std::thread::Thread,\n | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `HeapSize` is not implemented for `std::thread::Thread`\n```\n\n
\n\n## Parsing a custom syntax\n\nThe [`lazy-static`] example directory shows the implementation of a\n`functionlike!(...)` procedural macro in which the input tokens are parsed using\nSyn's parsing API.\n\n[`lazy-static`]: examples/lazy-static\n\nThe example reimplements the popular `lazy_static` crate from crates.io as a\nprocedural macro.\n\n```rust\nlazy_static! {\n static ref USERNAME: Regex = Regex::new(\"^[a-z0-9_-]{3,16}$\").unwrap();\n}\n```\n\nThe implementation shows how to trigger custom warnings and error messages on\nthe macro input.\n\n```console\nwarning: come on, pick a more creative name\n --> src/main.rs:10:16\n |\n10 | static ref FOO: String = \"lazy_static\".to_owned();\n | ^^^\n```\n\n
\n\n## Testing\n\nWhen testing macros, we often care not just that the macro can be used\nsuccessfully but also that when the macro is provided with invalid input it\nproduces maximally helpful error messages. Consider using the [`trybuild`] crate\nto write tests for errors that are emitted by your macro or errors detected by\nthe Rust compiler in the expanded code following misuse of the macro. Such tests\nhelp avoid regressions from later refactors that mistakenly make an error no\nlonger trigger or be less helpful than it used to be.\n\n[`trybuild`]: https://github.com/dtolnay/trybuild\n\n
\n\n## Debugging\n\nWhen developing a procedural macro it can be helpful to look at what the\ngenerated code looks like. Use `cargo rustc -- -Zunstable-options\n-Zunpretty=expanded` or the [`cargo expand`] subcommand.\n\n[`cargo expand`]: https://github.com/dtolnay/cargo-expand\n\nTo show the expanded code for some crate that uses your procedural macro, run\n`cargo expand` from that crate. To show the expanded code for one of your own\ntest cases, run `cargo expand --test the_test_case` where the last argument is\nthe name of the test file without the `.rs` extension.\n\nThis write-up by Brandon W Maister discusses debugging in more detail:\n[Debugging Rust's new Custom Derive system][debugging].\n\n[debugging]: https://quodlibetor.github.io/posts/debugging-rusts-new-custom-derive-system/\n\n
\n\n## Optional features\n\nSyn puts a lot of functionality behind optional features in order to optimize\ncompile time for the most common use cases. The following features are\navailable.\n\n- **`derive`** *(enabled by default)* — Data structures for representing the\n possible input to a derive macro, including structs and enums and types.\n- **`full`** — Data structures for representing the syntax tree of all valid\n Rust source code, including items and expressions.\n- **`parsing`** *(enabled by default)* — Ability to parse input tokens into a\n syntax tree node of a chosen type.\n- **`printing`** *(enabled by default)* — Ability to print a syntax tree node as\n tokens of Rust source code.\n- **`visit`** — Trait for traversing a syntax tree.\n- **`visit-mut`** — Trait for traversing and mutating in place a syntax tree.\n- **`fold`** — Trait for transforming an owned syntax tree.\n- **`clone-impls`** *(enabled by default)* — Clone impls for all syntax tree\n types.\n- **`extra-traits`** — Debug, Eq, PartialEq, Hash impls for all syntax tree\n types.\n- **`proc-macro`** *(enabled by default)* — Runtime dependency on the dynamic\n library libproc_macro from rustc toolchain.\n\n
\n\n## Proc macro shim\n\nSyn operates on the token representation provided by the [proc-macro2] crate\nfrom crates.io rather than using the compiler's built in proc-macro crate\ndirectly. This enables code using Syn to execute outside of the context of a\nprocedural macro, such as in unit tests or build.rs, and we avoid needing\nincompatible ecosystems for proc macros vs non-macro use cases.\n\nIn general all of your code should be written against proc-macro2 rather than\nproc-macro. The one exception is in the signatures of procedural macro entry\npoints, which are required by the language to use `proc_macro::TokenStream`.\n\nThe proc-macro2 crate will automatically detect and use the compiler's data\nstructures when a procedural macro is active.\n\n[proc-macro2]: https://docs.rs/proc-macro2/1.0/proc_macro2/\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this crate by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "syn@3.0.6", "sourcePath": "README.md", "sha256": "a00f270c7f468863f28b9984547872efb57c7db1b9beb6e685852dc24de6846e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Parser for Rust source code\n===========================\n\n[\"github\"](https://github.com/dtolnay/syn)\n[\"crates.io\"](https://crates.io/crates/syn)\n[\"docs.rs\"](https://docs.rs/syn)\n[\"build](https://github.com/dtolnay/syn/actions?query=branch%3Amaster)\n\nSyn is a parsing library for parsing a stream of Rust tokens into a syntax tree\nof Rust source code.\n\nCurrently this library is geared toward use in Rust procedural macros, but\ncontains some APIs that may be useful more generally.\n\n- **Data structures** — Syn provides a syntax tree that can represent most\n stable Rust source code and some unstable syntax. The syntax tree is rooted at\n [`syn::File`] which represents a full source file, but there are other entry\n points that may be useful to procedural macros including [`syn::Item`],\n [`syn::Expr`] and [`syn::Type`].\n\n- **Derives** — Of particular interest to derive macros is [`syn::DeriveInput`]\n which is any of the three legal input items to a derive macro. An example\n below shows using this type in a library that can derive implementations of a\n user-defined trait.\n\n- **Parsing** — Parsing in Syn is built around [parser functions] with the\n signature `fn(ParseStream) -> Result`. Every syntax tree node defined by\n Syn is individually parsable and may be used as a building block for custom\n syntaxes, or you may dream up your own brand new syntax without involving any\n of our syntax tree types.\n\n- **Location information** — Every token parsed by Syn is associated with a\n `Span` that tracks line and column information back to the source of that\n token. These spans allow a procedural macro to display detailed error messages\n pointing to all the right places in the user's code. There is an example of\n this below.\n\n- **Feature flags** — Functionality is aggressively feature gated so your\n procedural macros enable only what they need, and do not pay in compile time\n for all the rest.\n\n[`syn::File`]: https://docs.rs/syn/3/syn/struct.File.html\n[`syn::Item`]: https://docs.rs/syn/3/syn/enum.Item.html\n[`syn::Expr`]: https://docs.rs/syn/3/syn/enum.Expr.html\n[`syn::Type`]: https://docs.rs/syn/3/syn/enum.Type.html\n[`syn::DeriveInput`]: https://docs.rs/syn/3/syn/struct.DeriveInput.html\n[parser functions]: https://docs.rs/syn/3/syn/parse/index.html\n\n[*Release notes*](https://github.com/dtolnay/syn/releases)\n\n
\n\n## Resources\n\nThe best way to learn about procedural macros is by writing some. Consider\nworking through [this procedural macro workshop][workshop] to get familiar with\nthe different types of procedural macros. The workshop contains relevant links\ninto the Syn documentation as you work through each project.\n\n[workshop]: https://github.com/dtolnay/proc-macro-workshop\n\n
\n\n## Example of a derive macro\n\nThe canonical derive macro using Syn looks like this. We write an ordinary Rust\nfunction tagged with a `proc_macro_derive` attribute and the name of the trait\nwe are deriving. Any time that derive appears in the user's code, the Rust\ncompiler passes their data structure as tokens into our macro. We get to execute\narbitrary Rust code to figure out what to do with those tokens, then hand some\ntokens back to the compiler to compile into the user's crate.\n\n[`TokenStream`]: https://doc.rust-lang.org/proc_macro/struct.TokenStream.html\n\n```toml\n# Cargo.toml\n[package]\n...\n\n[lib]\nproc-macro = true\n\n[dependencies]\nsyn = \"3\"\nquote = \"1\"\n```\n\n```rust\nuse proc_macro::TokenStream;\nuse quote::quote;\nuse syn::{parse_macro_input, DeriveInput};\n\n#[proc_macro_derive(MyMacro)]\npub fn my_macro(input: TokenStream) -> TokenStream {\n // Parse the input tokens into a syntax tree\n let input = parse_macro_input!(input as DeriveInput);\n\n // Build the output, possibly using quasi-quotation\n let expanded = quote! {\n // ...\n };\n\n // Hand the output tokens back to the compiler\n TokenStream::from(expanded)\n}\n```\n\nThe [`heapsize`] example directory shows a complete working implementation of a\nderive macro. The example derives a `HeapSize` trait which computes an estimate\nof the amount of heap memory owned by a value.\n\n[`heapsize`]: examples/heapsize\n\n```rust\npub trait HeapSize {\n /// Total number of bytes of heap memory owned by `self`.\n fn heap_size_of_children(&self) -> usize;\n}\n```\n\nThe derive macro allows users to write `#[derive(HeapSize)]` on data structures\nin their program.\n\n```rust\n#[derive(HeapSize)]\nstruct Demo<'a, T: ?Sized> {\n a: Box,\n b: u8,\n c: &'a str,\n d: String,\n}\n```\n\n
\n\n## Spans and error reporting\n\nThe token-based procedural macro API provides great control over where the\ncompiler's error messages are displayed in user code. Consider the error the\nuser sees if one of their field types does not implement `HeapSize`.\n\n```rust\n#[derive(HeapSize)]\nstruct Broken {\n ok: String,\n bad: std::thread::Thread,\n}\n```\n\nBy tracking span information all the way through the expansion of a procedural\nmacro as shown in the `heapsize` example, token-based macros in Syn are able to\ntrigger errors that directly pinpoint the source of the problem.\n\n```console\nerror[E0277]: the trait bound `Thread: HeapSize` is not satisfied\n --> src/main.rs:9:5\n |\n3 | #[derive(HeapSize)]\n | -------- required by a bound introduced by this call\n...\n9 | bad: std::thread::Thread,\n | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `HeapSize` is not implemented for `Thread`\n |\n = help: the following other types implement trait `HeapSize`:\n &'a T\n Box\n Demo<'a, T>\n String\n [T]\n u8\n```\n\n
\n\n## Parsing a custom syntax\n\nThe [`lazy-static`] example directory shows the implementation of a\n`functionlike!(...)` procedural macro in which the input tokens are parsed using\nSyn's parsing API.\n\n[`lazy-static`]: examples/lazy-static\n\nThe example reimplements the popular `lazy_static` crate from crates.io as a\nprocedural macro.\n\n```rust\nlazy_static! {\n static ref USERNAME: Regex = Regex::new(\"^[a-z0-9_-]{3,16}$\").unwrap();\n}\n```\n\nThe implementation shows how to trigger custom warnings and error messages on\nthe macro input.\n\n```console\nwarning: come on, pick a more creative name\n --> src/main.rs:10:16\n |\n10 | static ref FOO: String = \"lazy_static\".to_owned();\n | ^^^\n```\n\n
\n\n## Testing\n\nWhen testing macros, we often care not just that the macro can be used\nsuccessfully but also that when the macro is provided with invalid input it\nproduces maximally helpful error messages. Consider using the [`trybuild`] crate\nto write tests for errors that are emitted by your macro or errors detected by\nthe Rust compiler in the expanded code following misuse of the macro. Such tests\nhelp avoid regressions from later refactors that mistakenly make an error no\nlonger trigger or be less helpful than it used to be.\n\n[`trybuild`]: https://github.com/dtolnay/trybuild\n\n
\n\n## Debugging\n\nWhen developing a procedural macro it can be helpful to look at what the\ngenerated code looks like. Use `cargo rustc -- -Zunstable-options\n-Zunpretty=expanded` or the [`cargo expand`] subcommand.\n\n[`cargo expand`]: https://github.com/dtolnay/cargo-expand\n\nTo show the expanded code for some crate that uses your procedural macro, run\n`cargo expand` from that crate. To show the expanded code for one of your own\ntest cases, run `cargo expand --test the_test_case` where the last argument is\nthe name of the test file without the `.rs` extension.\n\nThis write-up by Brandon W Maister discusses debugging in more detail:\n[Debugging Rust's new Custom Derive system][debugging].\n\n[debugging]: https://quodlibetor.github.io/posts/debugging-rusts-new-custom-derive-system/\n\n
\n\n## Optional features\n\nSyn puts a lot of functionality behind optional features in order to optimize\ncompile time for the most common use cases. The following features are\navailable.\n\n- **`derive`** *(enabled by default)* — Data structures for representing the\n possible input to a derive macro, including structs and enums and types.\n- **`full`** — Data structures for representing the syntax tree of all valid\n Rust source code, including items and expressions.\n- **`parsing`** *(enabled by default)* — Ability to parse input tokens into a\n syntax tree node of a chosen type.\n- **`printing`** *(enabled by default)* — Ability to print a syntax tree node as\n tokens of Rust source code.\n- **`visit`** — Trait for traversing a syntax tree.\n- **`visit-mut`** — Trait for traversing and mutating in place a syntax tree.\n- **`fold`** — Trait for transforming an owned syntax tree.\n- **`clone-impls`** *(enabled by default)* — Clone impls for all syntax tree\n types.\n- **`extra-traits`** — Debug, Eq, PartialEq, Hash impls for all syntax tree\n types.\n- **`proc-macro`** *(enabled by default)* — Runtime dependency on the dynamic\n library libproc_macro from rustc toolchain.\n\n
\n\n## Proc macro shim\n\nSyn operates on the token representation provided by the [proc-macro2] crate\nfrom crates.io rather than using the compiler's built in proc-macro crate\ndirectly. This enables code using Syn to execute outside of the context of a\nprocedural macro, such as in unit tests or build.rs, and we avoid needing\nincompatible ecosystems for proc macros vs non-macro use cases.\n\nIn general all of your code should be written against proc-macro2 rather than\nproc-macro. The one exception is in the signatures of procedural macro entry\npoints, which are required by the language to use `proc_macro::TokenStream`.\n\nThe proc-macro2 crate will automatically detect and use the compiler's data\nstructures when a procedural macro is active.\n\n[proc-macro2]: https://docs.rs/proc-macro2/1.0/proc_macro2/\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this crate by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "tap@1.0.1", "sourcePath": "LICENSE.txt", "sha256": "8b4e95f5cf0dc40269c99f7b787203ffe04ded245ca2427422c196efa2b2f42a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "MIT License\n\nCopyright (c) 2017 Elliot Linder \n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "package": "thiserror@1.0.69", "sourcePath": "README.md", "sha256": "868031e233823e023fd80a53a590da851b7aea469fda30c51dc097925e309cd1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "derive(Error)\n=============\n\n[\"github\"](https://github.com/dtolnay/thiserror)\n[\"crates.io\"](https://crates.io/crates/thiserror)\n[\"docs.rs\"](https://docs.rs/thiserror)\n[\"build](https://github.com/dtolnay/thiserror/actions?query=branch%3Amaster)\n\nThis library provides a convenient derive macro for the standard library's\n[`std::error::Error`] trait.\n\n[`std::error::Error`]: https://doc.rust-lang.org/std/error/trait.Error.html\n\n```toml\n[dependencies]\nthiserror = \"1.0\"\n```\n\n*Compiler support: requires rustc 1.61+*\n\n
\n\n## Example\n\n```rust\nuse thiserror::Error;\n\n#[derive(Error, Debug)]\npub enum DataStoreError {\n #[error(\"data store disconnected\")]\n Disconnect(#[from] io::Error),\n #[error(\"the data for key `{0}` is not available\")]\n Redaction(String),\n #[error(\"invalid header (expected {expected:?}, found {found:?})\")]\n InvalidHeader {\n expected: String,\n found: String,\n },\n #[error(\"unknown data store error\")]\n Unknown,\n}\n```\n\n
\n\n## Details\n\n- Thiserror deliberately does not appear in your public API. You get the same\n thing as if you had written an implementation of `std::error::Error` by hand,\n and switching from handwritten impls to thiserror or vice versa is not a\n breaking change.\n\n- Errors may be enums, structs with named fields, tuple structs, or unit\n structs.\n\n- A `Display` impl is generated for your error if you provide `#[error(\"...\")]`\n messages on the struct or each variant of your enum, as shown above in the\n example.\n\n The messages support a shorthand for interpolating fields from the error.\n\n - `#[error(\"{var}\")]` ⟶ `write!(\"{}\", self.var)`\n - `#[error(\"{0}\")]` ⟶ `write!(\"{}\", self.0)`\n - `#[error(\"{var:?}\")]` ⟶ `write!(\"{:?}\", self.var)`\n - `#[error(\"{0:?}\")]` ⟶ `write!(\"{:?}\", self.0)`\n\n These shorthands can be used together with any additional format args, which\n may be arbitrary expressions. For example:\n\n ```rust\n #[derive(Error, Debug)]\n pub enum Error {\n #[error(\"invalid rdo_lookahead_frames {0} (expected < {})\", i32::MAX)]\n InvalidLookahead(u32),\n }\n ```\n\n If one of the additional expression arguments needs to refer to a field of the\n struct or enum, then refer to named fields as `.var` and tuple fields as `.0`.\n\n ```rust\n #[derive(Error, Debug)]\n pub enum Error {\n #[error(\"first letter must be lowercase but was {:?}\", first_char(.0))]\n WrongCase(String),\n #[error(\"invalid index {idx}, expected at least {} and at most {}\", .limits.lo, .limits.hi)]\n OutOfBounds { idx: usize, limits: Limits },\n }\n ```\n\n- A `From` impl is generated for each variant that contains a `#[from]`\n attribute.\n\n The variant using `#[from]` must not contain any other fields beyond the\n source error (and possibly a backtrace — see below). Usually `#[from]`\n fields are unnamed, but `#[from]` is allowed on a named field too.\n\n ```rust\n #[derive(Error, Debug)]\n pub enum MyError {\n Io(#[from] io::Error),\n Glob(#[from] globset::Error),\n }\n ```\n\n- The Error trait's `source()` method is implemented to return whichever field\n has a `#[source]` attribute or is named `source`, if any. This is for\n identifying the underlying lower level error that caused your error.\n\n The `#[from]` attribute always implies that the same field is `#[source]`, so\n you don't ever need to specify both attributes.\n\n Any error type that implements `std::error::Error` or dereferences to `dyn\n std::error::Error` will work as a source.\n\n ```rust\n #[derive(Error, Debug)]\n pub struct MyError {\n msg: String,\n #[source] // optional if field name is `source`\n source: anyhow::Error,\n }\n ```\n\n- The Error trait's `provide()` method is implemented to provide whichever field\n has a type named `Backtrace`, if any, as a `std::backtrace::Backtrace`. Using\n `Backtrace` in errors requires a nightly compiler with Rust version 1.73 or\n newer.\n\n ```rust\n use std::backtrace::Backtrace;\n\n #[derive(Error, Debug)]\n pub struct MyError {\n msg: String,\n backtrace: Backtrace, // automatically detected\n }\n ```\n\n- If a field is both a source (named `source`, or has `#[source]` or `#[from]`\n attribute) *and* is marked `#[backtrace]`, then the Error trait's `provide()`\n method is forwarded to the source's `provide` so that both layers of the error\n share the same backtrace. The `#[backtrace]` attribute requires a nightly\n compiler with Rust version 1.73 or newer.\n\n\n ```rust\n #[derive(Error, Debug)]\n pub enum MyError {\n Io {\n #[backtrace]\n source: io::Error,\n },\n }\n ```\n\n- For variants that use `#[from]` and also contain a `Backtrace` field, a\n backtrace is captured from within the `From` impl.\n\n ```rust\n #[derive(Error, Debug)]\n pub enum MyError {\n Io {\n #[from]\n source: io::Error,\n backtrace: Backtrace,\n },\n }\n ```\n\n- Errors may use `error(transparent)` to forward the source and Display methods\n straight through to an underlying error without adding an additional message.\n This would be appropriate for enums that need an \"anything else\" variant.\n\n ```rust\n #[derive(Error, Debug)]\n pub enum MyError {\n ...\n\n #[error(transparent)]\n Other(#[from] anyhow::Error), // source and Display delegate to anyhow::Error\n }\n ```\n\n Another use case is hiding implementation details of an error representation\n behind an opaque error type, so that the representation is able to evolve\n without breaking the crate's public API.\n\n ```rust\n // PublicError is public, but opaque and easy to keep compatible.\n #[derive(Error, Debug)]\n #[error(transparent)]\n pub struct PublicError(#[from] ErrorRepr);\n\n impl PublicError {\n // Accessors for anything we do want to expose publicly.\n }\n\n // Private and free to change across minor version of the crate.\n #[derive(Error, Debug)]\n enum ErrorRepr {\n ...\n }\n ```\n\n- See also the [`anyhow`] library for a convenient single error type to use in\n application code.\n\n [`anyhow`]: https://github.com/dtolnay/anyhow\n\n
\n\n## Comparison to anyhow\n\nUse thiserror if you care about designing your own dedicated error type(s) so\nthat the caller receives exactly the information that you choose in the event of\nfailure. This most often applies to library-like code. Use [Anyhow] if you don't\ncare what error type your functions return, you just want it to be easy. This is\ncommon in application-like code.\n\n[Anyhow]: https://github.com/dtolnay/anyhow\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this crate by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "thiserror@2.0.20", "sourcePath": "README.md", "sha256": "96da8e4001dbd51bdc135de6deb4c954889624beb9f0c177ed0fdcd7e9ff41a2", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "derive(Error)\n=============\n\n[\"github\"](https://github.com/dtolnay/thiserror)\n[\"crates.io\"](https://crates.io/crates/thiserror)\n[\"docs.rs\"](https://docs.rs/thiserror)\n[\"build](https://github.com/dtolnay/thiserror/actions?query=branch%3Amaster)\n\nThis library provides a convenient derive macro for the standard library's\n[`std::error::Error`] trait.\n\n[`std::error::Error`]: https://doc.rust-lang.org/std/error/trait.Error.html\n\n```toml\n[dependencies]\nthiserror = \"2\"\n```\n\n
\n\n## Example\n\n```rust\nuse thiserror::Error;\n\n#[derive(Error, Debug)]\npub enum DataStoreError {\n #[error(\"data store disconnected\")]\n Disconnect(#[from] io::Error),\n #[error(\"the data for key `{0}` is not available\")]\n Redaction(String),\n #[error(\"invalid header (expected {expected:?}, found {found:?})\")]\n InvalidHeader {\n expected: String,\n found: String,\n },\n #[error(\"unknown data store error\")]\n Unknown,\n}\n```\n\n
\n\n## Details\n\n- Thiserror deliberately does not appear in your public API. You get the same\n thing as if you had written an implementation of `std::error::Error` by hand,\n and switching from handwritten impls to thiserror or vice versa is not a\n breaking change.\n\n- Errors may be enums, structs with named fields, tuple structs, or unit\n structs.\n\n- A `Display` impl is generated for your error if you provide `#[error(\"...\")]`\n messages on the struct or each variant of your enum, as shown above in the\n example.\n\n The messages support a shorthand for interpolating fields from the error.\n\n - `#[error(\"{var}\")]` ⟶ `write!(\"{}\", self.var)`\n - `#[error(\"{0}\")]` ⟶ `write!(\"{}\", self.0)`\n - `#[error(\"{var:?}\")]` ⟶ `write!(\"{:?}\", self.var)`\n - `#[error(\"{0:?}\")]` ⟶ `write!(\"{:?}\", self.0)`\n\n These shorthands can be used together with any additional format args, which\n may be arbitrary expressions. For example:\n\n ```rust\n #[derive(Error, Debug)]\n pub enum Error {\n #[error(\"invalid rdo_lookahead_frames {0} (expected < {max})\", max = i32::MAX)]\n InvalidLookahead(u32),\n }\n ```\n\n If one of the additional expression arguments needs to refer to a field of the\n struct or enum, then refer to named fields as `.var` and tuple fields as `.0`.\n\n ```rust\n #[derive(Error, Debug)]\n pub enum Error {\n #[error(\"first letter must be lowercase but was {:?}\", first_char(.0))]\n WrongCase(String),\n #[error(\"invalid index {idx}, expected at least {} and at most {}\", .limits.lo, .limits.hi)]\n OutOfBounds { idx: usize, limits: Limits },\n }\n ```\n\n- A `From` impl is generated for each variant that contains a `#[from]`\n attribute.\n\n The variant using `#[from]` must not contain any other fields beyond the\n source error (and possibly a backtrace — see below). Usually `#[from]`\n fields are unnamed, but `#[from]` is allowed on a named field too.\n\n ```rust\n #[derive(Error, Debug)]\n pub enum MyError {\n Io(#[from] io::Error),\n Glob(#[from] globset::Error),\n }\n ```\n\n- The Error trait's `source()` method is implemented to return whichever field\n has a `#[source]` attribute or is named `source`, if any. This is for\n identifying the underlying lower level error that caused your error.\n\n The `#[from]` attribute always implies that the same field is `#[source]`, so\n you don't ever need to specify both attributes.\n\n Any error type that implements `std::error::Error` or dereferences to `dyn\n std::error::Error` will work as a source.\n\n ```rust\n #[derive(Error, Debug)]\n pub struct MyError {\n msg: String,\n #[source] // optional if field name is `source`\n source: anyhow::Error,\n }\n ```\n\n- The Error trait's `provide()` method is implemented to provide whichever field\n has a type named `Backtrace`, if any, as a `std::backtrace::Backtrace`. Using\n `Backtrace` in errors requires a nightly compiler with Rust version 1.73 or\n newer.\n\n ```rust\n use std::backtrace::Backtrace;\n\n #[derive(Error, Debug)]\n pub struct MyError {\n msg: String,\n backtrace: Backtrace, // automatically detected\n }\n ```\n\n- If a field is both a source (named `source`, or has `#[source]` or `#[from]`\n attribute) *and* is marked `#[backtrace]`, then the Error trait's `provide()`\n method is forwarded to the source's `provide` so that both layers of the error\n share the same backtrace. The `#[backtrace]` attribute requires a nightly\n compiler with Rust version 1.73 or newer.\n\n\n ```rust\n #[derive(Error, Debug)]\n pub enum MyError {\n Io {\n #[backtrace]\n source: io::Error,\n },\n }\n ```\n\n- For variants that use `#[from]` and also contain a `Backtrace` field, a\n backtrace is captured from within the `From` impl.\n\n ```rust\n #[derive(Error, Debug)]\n pub enum MyError {\n Io {\n #[from]\n source: io::Error,\n backtrace: Backtrace,\n },\n }\n ```\n\n- Errors may use `error(transparent)` to forward the source and Display methods\n straight through to an underlying error without adding an additional message.\n This would be appropriate for enums that need an \"anything else\" variant.\n\n ```rust\n #[derive(Error, Debug)]\n pub enum MyError {\n ...\n\n #[error(transparent)]\n Other(#[from] anyhow::Error), // source and Display delegate to anyhow::Error\n }\n ```\n\n Another use case is hiding implementation details of an error representation\n behind an opaque error type, so that the representation is able to evolve\n without breaking the crate's public API.\n\n ```rust\n // PublicError is public, but opaque and easy to keep compatible.\n #[derive(Error, Debug)]\n #[error(transparent)]\n pub struct PublicError(#[from] ErrorRepr);\n\n impl PublicError {\n // Accessors for anything we do want to expose publicly.\n }\n\n // Private and free to change across minor version of the crate.\n #[derive(Error, Debug)]\n enum ErrorRepr {\n ...\n }\n ```\n\n- See also the [`anyhow`] library for a convenient single error type to use in\n application code.\n\n [`anyhow`]: https://github.com/dtolnay/anyhow\n\n
\n\n## Comparison to anyhow\n\nUse thiserror if you care about designing your own dedicated error type(s) so\nthat the caller receives exactly the information that you choose in the event of\nfailure. This most often applies to library-like code. Use [Anyhow] if you don't\ncare what error type your functions return, you just want it to be easy. This is\ncommon in application-like code.\n\n[Anyhow]: https://github.com/dtolnay/anyhow\n\n
\n\n#### License\n\n\nLicensed under either of Apache License, Version\n2.0 or MIT license at your option.\n\n\n
\n\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this crate by you, as defined in the Apache-2.0 license, shall\nbe dual licensed as above, without any additional terms or conditions.\n\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/alpha_runs.rs", "sha256": "bf5a2505be77c70d82768f46c69775fc362c3c47d93f2dd9b3a65434647fb303", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse alloc::vec;\nuse alloc::vec::Vec;\nuse core::convert::TryFrom;\nuse core::num::NonZeroU16;\n\nuse crate::color::AlphaU8;\nuse crate::LengthU32;\n\npub type AlphaRun = Option;\n\n/// Sparse array of run-length-encoded alpha (supersampling coverage) values.\n///\n/// Sparseness allows us to independently compose several paths into the\n/// same AlphaRuns buffer.\npub struct AlphaRuns {\n pub runs: Vec,\n pub alpha: Vec,\n}\n\nimpl AlphaRuns {\n pub fn new(width: LengthU32) -> Self {\n let mut runs = AlphaRuns {\n runs: vec![None; (width.get() + 1) as usize],\n alpha: vec![0; (width.get() + 1) as usize],\n };\n runs.reset(width);\n runs\n }\n\n /// Returns 0-255 given 0-256.\n pub fn catch_overflow(alpha: u16) -> AlphaU8 {\n debug_assert!(alpha <= 256);\n (alpha - (alpha >> 8)) as u8\n }\n\n /// Returns true if the scanline contains only a single run, of alpha value 0.\n pub fn is_empty(&self) -> bool {\n debug_assert!(self.runs[0].is_some());\n match self.runs[0] {\n Some(run) => self.alpha[0] == 0 && self.runs[usize::from(run.get())].is_none(),\n None => true,\n }\n }\n\n /// Reinitialize for a new scanline.\n pub fn reset(&mut self, width: LengthU32) {\n let run = u16::try_from(width.get()).unwrap();\n self.runs[0] = NonZeroU16::new(run);\n self.runs[width.get() as usize] = None;\n self.alpha[0] = 0;\n }\n\n /// Insert into the buffer a run starting at (x-offset_x).\n ///\n /// if start_alpha > 0\n /// one pixel with value += start_alpha,\n /// max 255\n /// if middle_count > 0\n /// middle_count pixels with value += max_value\n /// if stop_alpha > 0\n /// one pixel with value += stop_alpha\n ///\n /// Returns the offset_x value that should be passed on the next call,\n /// assuming we're on the same scanline. If the caller is switching\n /// scanlines, then offset_x should be 0 when this is called.\n pub fn add(\n &mut self,\n x: u32,\n start_alpha: AlphaU8,\n mut middle_count: usize,\n stop_alpha: AlphaU8,\n max_value: u8,\n offset_x: usize,\n ) -> usize {\n let mut x = x as usize;\n\n let mut runs_offset = offset_x;\n let mut alpha_offset = offset_x;\n let mut last_alpha_offset = offset_x;\n x -= offset_x;\n\n if start_alpha != 0 {\n Self::break_run(\n &mut self.runs[runs_offset..],\n &mut self.alpha[alpha_offset..],\n x,\n 1,\n );\n // I should be able to just add alpha[x] + start_alpha.\n // However, if the trailing edge of the previous span and the leading\n // edge of the current span round to the same super-sampled x value,\n // I might overflow to 256 with this add, hence the funny subtract (crud).\n let tmp = u16::from(self.alpha[alpha_offset + x]) + u16::from(start_alpha);\n debug_assert!(tmp <= 256);\n // was (tmp >> 7), but that seems wrong if we're trying to catch 256\n self.alpha[alpha_offset + x] = (tmp - (tmp >> 8)) as u8;\n\n runs_offset += x + 1;\n alpha_offset += x + 1;\n x = 0;\n }\n\n if middle_count != 0 {\n Self::break_run(\n &mut self.runs[runs_offset..],\n &mut self.alpha[alpha_offset..],\n x,\n middle_count,\n );\n alpha_offset += x;\n runs_offset += x;\n x = 0;\n loop {\n let a = Self::catch_overflow(\n u16::from(self.alpha[alpha_offset]) + u16::from(max_value),\n );\n self.alpha[alpha_offset] = a;\n\n let n = usize::from(self.runs[runs_offset].unwrap().get());\n debug_assert!(n <= middle_count);\n alpha_offset += n;\n runs_offset += n;\n middle_count -= n;\n\n if middle_count == 0 {\n break;\n }\n }\n\n last_alpha_offset = alpha_offset;\n }\n\n if stop_alpha != 0 {\n Self::break_run(\n &mut self.runs[runs_offset..],\n &mut self.alpha[alpha_offset..],\n x,\n 1,\n );\n alpha_offset += x;\n self.alpha[alpha_offset] = (self.alpha[alpha_offset] + stop_alpha) as u8;\n last_alpha_offset = alpha_offset;\n }\n\n // new offset_x\n last_alpha_offset\n }\n\n /// Break the runs in the buffer at offsets x and x+count, properly\n /// updating the runs to the right and left.\n ///\n /// i.e. from the state AAAABBBB, run-length encoded as A4B4,\n /// break_run(..., 2, 5) would produce AAAABBBB rle as A2A2B3B1.\n /// Allows add() to sum another run to some of the new sub-runs.\n /// i.e. adding ..CCCCC. would produce AADDEEEB, rle as A2D2E3B1.\n fn break_run(runs: &mut [AlphaRun], alpha: &mut [u8], mut x: usize, count: usize) {\n debug_assert!(count > 0);\n\n let orig_x = x;\n let mut runs_offset = 0;\n let mut alpha_offset = 0;\n\n while x > 0 {\n let n = usize::from(runs[runs_offset].unwrap().get());\n debug_assert!(n > 0);\n\n if x < n {\n alpha[alpha_offset + x] = alpha[alpha_offset];\n runs[runs_offset + 0] = NonZeroU16::new(x as u16);\n runs[runs_offset + x] = NonZeroU16::new((n - x) as u16);\n break;\n }\n runs_offset += n;\n alpha_offset += n;\n x -= n;\n }\n\n runs_offset = orig_x;\n alpha_offset = orig_x;\n x = count;\n\n loop {\n let n = usize::from(runs[runs_offset].unwrap().get());\n debug_assert!(n > 0);\n\n if x < n {\n alpha[alpha_offset + x] = alpha[alpha_offset];\n runs[runs_offset + 0] = NonZeroU16::new(x as u16);\n runs[runs_offset + x] = NonZeroU16::new((n - x) as u16);\n break;\n }\n\n x -= n;\n if x == 0 {\n break;\n }\n\n runs_offset += n;\n alpha_offset += n;\n }\n }\n\n /// Cut (at offset x in the buffer) a run into two shorter runs with\n /// matching alpha values.\n ///\n /// Used by the RectClipBlitter to trim a RLE encoding to match the\n /// clipping rectangle.\n pub fn break_at(alpha: &mut [AlphaU8], runs: &mut [AlphaRun], mut x: i32) {\n let mut alpha_i = 0;\n let mut run_i = 0;\n while x > 0 {\n let n = runs[run_i].unwrap().get();\n let n_usize = usize::from(n);\n let n_i32 = i32::from(n);\n if x < n_i32 {\n alpha[alpha_i + x as usize] = alpha[alpha_i];\n runs[0] = NonZeroU16::new(x as u16);\n runs[x as usize] = NonZeroU16::new((n_i32 - x) as u16);\n break;\n }\n\n run_i += n_usize;\n alpha_i += n_usize;\n x -= n_i32;\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/blitter.rs", "sha256": "3262acaac9f2a78736295e937eee9542667b1de07b5a23db554830b192eb929e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse crate::geom::ScreenIntRect;\n\nuse crate::alpha_runs::AlphaRun;\nuse crate::color::AlphaU8;\nuse crate::LengthU32;\n\n/// Mask is used to describe alpha bitmaps.\npub struct Mask {\n pub image: [u8; 2],\n pub bounds: ScreenIntRect,\n pub row_bytes: u32,\n}\n\n/// Blitter is responsible for actually writing pixels into memory.\n///\n/// Besides efficiency, they handle clipping and antialiasing.\n/// An object that implements Blitter contains all the context needed to generate pixels\n/// for the destination and how src/generated pixels map to the destination.\n/// The coordinates passed to the `blit_*` calls are in destination pixel space.\npub trait Blitter {\n /// Blits a horizontal run of one or more pixels.\n fn blit_h(&mut self, _x: u32, _y: u32, _width: LengthU32) {\n unreachable!()\n }\n\n /// Blits a horizontal run of antialiased pixels.\n ///\n /// runs[] is a *sparse* zero-terminated run-length encoding of spans of constant alpha values.\n ///\n /// The runs[] and antialias[] work together to represent long runs of pixels with the same\n /// alphas. The runs[] contains the number of pixels with the same alpha, and antialias[]\n /// contain the coverage value for that number of pixels. The runs[] (and antialias[]) are\n /// encoded in a clever way. The runs array is zero terminated, and has enough entries for\n /// each pixel plus one, in most cases some of the entries will not contain valid data. An entry\n /// in the runs array contains the number of pixels (np) that have the same alpha value. The\n /// next np value is found np entries away. For example, if runs[0] = 7, then the next valid\n /// entry will by at runs[7]. The runs array and antialias[] are coupled by index. So, if the\n /// np entry is at runs[45] = 12 then the alpha value can be found at antialias[45] = 0x88.\n /// This would mean to use an alpha value of 0x88 for the next 12 pixels starting at pixel 45.\n fn blit_anti_h(\n &mut self,\n _x: u32,\n _y: u32,\n _antialias: &mut [AlphaU8],\n _runs: &mut [AlphaRun],\n ) {\n unreachable!()\n }\n\n /// Blits a vertical run of pixels with a constant alpha value.\n fn blit_v(&mut self, _x: u32, _y: u32, _height: LengthU32, _alpha: AlphaU8) {\n unreachable!()\n }\n\n fn blit_anti_h2(&mut self, _x: u32, _y: u32, _alpha0: AlphaU8, _alpha1: AlphaU8) {\n unreachable!()\n }\n\n fn blit_anti_v2(&mut self, _x: u32, _y: u32, _alpha0: AlphaU8, _alpha1: AlphaU8) {\n unreachable!()\n }\n\n /// Blits a solid rectangle one or more pixels wide.\n fn blit_rect(&mut self, _rect: &ScreenIntRect) {\n unreachable!()\n }\n\n /// Blits a pattern of pixels defined by a rectangle-clipped mask.\n fn blit_mask(&mut self, _mask: &Mask, _clip: &ScreenIntRect) {\n unreachable!()\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/color.rs", "sha256": "41a25664ba67f30823dc48781fa35fd344d7f3cfd0c02a83e736c3fe6c18bc7d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse tiny_skia_path::{NormalizedF32, Scalar};\n\n/// 8-bit type for an alpha value. 255 is 100% opaque, zero is 100% transparent.\npub type AlphaU8 = u8;\n\n/// Represents fully transparent AlphaU8 value.\npub const ALPHA_U8_TRANSPARENT: AlphaU8 = 0x00;\n\n/// Represents fully opaque AlphaU8 value.\npub const ALPHA_U8_OPAQUE: AlphaU8 = 0xFF;\n\n/// Represents fully transparent Alpha value.\npub const ALPHA_TRANSPARENT: NormalizedF32 = NormalizedF32::ZERO;\n\n/// Represents fully opaque Alpha value.\npub const ALPHA_OPAQUE: NormalizedF32 = NormalizedF32::ONE;\n\n/// A 32-bit RGBA color value.\n///\n/// Byteorder: RGBA (relevant for bytemuck casts)\n#[repr(transparent)]\n#[derive(Copy, Clone, PartialEq)]\npub struct ColorU8([u8; 4]);\n\nimpl ColorU8 {\n /// Creates a new color.\n pub const fn from_rgba(r: u8, g: u8, b: u8, a: u8) -> Self {\n ColorU8([r, g, b, a])\n }\n\n /// Returns color's red component.\n pub const fn red(self) -> u8 {\n self.0[0]\n }\n\n /// Returns color's green component.\n pub const fn green(self) -> u8 {\n self.0[1]\n }\n\n /// Returns color's blue component.\n pub const fn blue(self) -> u8 {\n self.0[2]\n }\n\n /// Returns color's alpha component.\n pub const fn alpha(self) -> u8 {\n self.0[3]\n }\n\n /// Check that color is opaque.\n ///\n /// Alpha == 255\n pub fn is_opaque(&self) -> bool {\n self.alpha() == ALPHA_U8_OPAQUE\n }\n\n /// Converts into a premultiplied color.\n pub fn premultiply(&self) -> PremultipliedColorU8 {\n let a = self.alpha();\n if a != ALPHA_U8_OPAQUE {\n PremultipliedColorU8::from_rgba_unchecked(\n premultiply_u8(self.red(), a),\n premultiply_u8(self.green(), a),\n premultiply_u8(self.blue(), a),\n a,\n )\n } else {\n PremultipliedColorU8::from_rgba_unchecked(self.red(), self.green(), self.blue(), a)\n }\n }\n}\n\nimpl core::fmt::Debug for ColorU8 {\n fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n f.debug_struct(\"ColorU8\")\n .field(\"r\", &self.red())\n .field(\"g\", &self.green())\n .field(\"b\", &self.blue())\n .field(\"a\", &self.alpha())\n .finish()\n }\n}\n\n/// A 32-bit premultiplied RGBA color value.\n///\n/// Byteorder: RGBA (relevant for bytemuck casts)\n#[repr(transparent)]\n#[derive(Copy, Clone, PartialEq)]\npub struct PremultipliedColorU8([u8; 4]);\n\n// Perfectly safe, since [u8; 4] is already Pod.\nunsafe impl bytemuck::Zeroable for PremultipliedColorU8 {}\nunsafe impl bytemuck::Pod for PremultipliedColorU8 {}\n\nimpl PremultipliedColorU8 {\n /// A transparent color.\n pub const TRANSPARENT: Self = PremultipliedColorU8::from_rgba_unchecked(0, 0, 0, 0);\n\n /// Creates a new premultiplied color.\n ///\n /// RGB components must be <= alpha.\n pub fn from_rgba(r: u8, g: u8, b: u8, a: u8) -> Option {\n if r <= a && g <= a && b <= a {\n Some(PremultipliedColorU8([r, g, b, a]))\n } else {\n None\n }\n }\n\n /// Creates a new color.\n pub(crate) const fn from_rgba_unchecked(r: u8, g: u8, b: u8, a: u8) -> Self {\n PremultipliedColorU8([r, g, b, a])\n }\n\n /// Returns color's red component.\n ///\n /// The value is <= alpha.\n pub const fn red(self) -> u8 {\n self.0[0]\n }\n\n /// Returns color's green component.\n ///\n /// The value is <= alpha.\n pub const fn green(self) -> u8 {\n self.0[1]\n }\n\n /// Returns color's blue component.\n ///\n /// The value is <= alpha.\n pub const fn blue(self) -> u8 {\n self.0[2]\n }\n\n /// Returns color's alpha component.\n pub const fn alpha(self) -> u8 {\n self.0[3]\n }\n\n /// Check that color is opaque.\n ///\n /// Alpha == 255\n pub fn is_opaque(&self) -> bool {\n self.alpha() == ALPHA_U8_OPAQUE\n }\n\n /// Returns a demultiplied color.\n pub fn demultiply(&self) -> ColorU8 {\n let alpha = self.alpha();\n if alpha == ALPHA_U8_OPAQUE {\n ColorU8(self.0)\n } else {\n let a = alpha as f64 / 255.0;\n ColorU8::from_rgba(\n (self.red() as f64 / a + 0.5) as u8,\n (self.green() as f64 / a + 0.5) as u8,\n (self.blue() as f64 / a + 0.5) as u8,\n alpha,\n )\n }\n }\n}\n\nimpl core::fmt::Debug for PremultipliedColorU8 {\n fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n f.debug_struct(\"PremultipliedColorU8\")\n .field(\"r\", &self.red())\n .field(\"g\", &self.green())\n .field(\"b\", &self.blue())\n .field(\"a\", &self.alpha())\n .finish()\n }\n}\n\n/// An RGBA color value, holding four floating point components.\n///\n/// # Guarantees\n///\n/// - All values are in 0..=1 range.\n#[derive(Copy, Clone, PartialEq, Debug)]\npub struct Color {\n r: NormalizedF32,\n g: NormalizedF32,\n b: NormalizedF32,\n a: NormalizedF32,\n}\n\nconst NV_ZERO: NormalizedF32 = NormalizedF32::ZERO;\nconst NV_ONE: NormalizedF32 = NormalizedF32::ONE;\n\nimpl Color {\n /// A transparent color.\n pub const TRANSPARENT: Color = Color {\n r: NV_ZERO,\n g: NV_ZERO,\n b: NV_ZERO,\n a: NV_ZERO,\n };\n /// A black color.\n pub const BLACK: Color = Color {\n r: NV_ZERO,\n g: NV_ZERO,\n b: NV_ZERO,\n a: NV_ONE,\n };\n /// A white color.\n pub const WHITE: Color = Color {\n r: NV_ONE,\n g: NV_ONE,\n b: NV_ONE,\n a: NV_ONE,\n };\n\n /// Creates a new color from 4 components.\n ///\n /// All values must be in 0..=1 range.\n pub fn from_rgba(r: f32, g: f32, b: f32, a: f32) -> Option {\n Some(Color {\n r: NormalizedF32::new(r)?,\n g: NormalizedF32::new(g)?,\n b: NormalizedF32::new(b)?,\n a: NormalizedF32::new(a)?,\n })\n }\n\n /// Creates a new color from 4 components.\n ///\n /// u8 will be divided by 255 to get the float component.\n pub fn from_rgba8(r: u8, g: u8, b: u8, a: u8) -> Self {\n Color {\n r: NormalizedF32::new_u8(r),\n g: NormalizedF32::new_u8(g),\n b: NormalizedF32::new_u8(b),\n a: NormalizedF32::new_u8(a),\n }\n }\n\n /// Returns color's red component.\n ///\n /// The value is guarantee to be in a 0..=1 range.\n pub fn red(&self) -> f32 {\n self.r.get()\n }\n\n /// Returns color's green component.\n ///\n /// The value is guarantee to be in a 0..=1 range.\n pub fn green(&self) -> f32 {\n self.g.get()\n }\n\n /// Returns color's blue component.\n ///\n /// The value is guarantee to be in a 0..=1 range.\n pub fn blue(&self) -> f32 {\n self.b.get()\n }\n\n /// Returns color's alpha component.\n ///\n /// The value is guarantee to be in a 0..=1 range.\n pub fn alpha(&self) -> f32 {\n self.a.get()\n }\n\n /// Sets the red component value.\n ///\n /// The new value will be clipped to the 0..=1 range.\n pub fn set_red(&mut self, c: f32) {\n self.r = NormalizedF32::new_clamped(c);\n }\n\n /// Sets the green component value.\n ///\n /// The new value will be clipped to the 0..=1 range.\n pub fn set_green(&mut self, c: f32) {\n self.g = NormalizedF32::new_clamped(c);\n }\n\n /// Sets the blue component value.\n ///\n /// The new value will be clipped to the 0..=1 range.\n pub fn set_blue(&mut self, c: f32) {\n self.b = NormalizedF32::new_clamped(c);\n }\n\n /// Sets the alpha component value.\n ///\n /// The new value will be clipped to the 0..=1 range.\n pub fn set_alpha(&mut self, c: f32) {\n self.a = NormalizedF32::new_clamped(c);\n }\n\n /// Shifts color's opacity.\n ///\n /// Essentially, multiplies color's alpha by opacity.\n ///\n /// `opacity` will be clamped to the 0..=1 range first.\n /// The final alpha will also be clamped.\n pub fn apply_opacity(&mut self, opacity: f32) {\n self.a = NormalizedF32::new_clamped(self.a.get() * opacity.bound(0.0, 1.0));\n }\n\n /// Check that color is opaque.\n ///\n /// Alpha == 1.0\n pub fn is_opaque(&self) -> bool {\n self.a == ALPHA_OPAQUE\n }\n\n /// Converts into a premultiplied color.\n pub fn premultiply(&self) -> PremultipliedColor {\n if self.is_opaque() {\n PremultipliedColor {\n r: self.r,\n g: self.g,\n b: self.b,\n a: self.a,\n }\n } else {\n PremultipliedColor {\n r: NormalizedF32::new_clamped(self.r.get() * self.a.get()),\n g: NormalizedF32::new_clamped(self.g.get() * self.a.get()),\n b: NormalizedF32::new_clamped(self.b.get() * self.a.get()),\n a: self.a,\n }\n }\n }\n\n /// Converts into `ColorU8`.\n pub fn to_color_u8(&self) -> ColorU8 {\n let c = color_f32_to_u8(self.r, self.g, self.b, self.a);\n ColorU8::from_rgba(c[0], c[1], c[2], c[3])\n }\n}\n\n/// A premultiplied RGBA color value, holding four floating point components.\n///\n/// # Guarantees\n///\n/// - All values are in 0..=1 range.\n/// - RGB components are <= A.\n#[derive(Copy, Clone, PartialEq, Debug)]\npub struct PremultipliedColor {\n r: NormalizedF32,\n g: NormalizedF32,\n b: NormalizedF32,\n a: NormalizedF32,\n}\n\nimpl PremultipliedColor {\n /// Returns color's red component.\n ///\n /// - The value is guarantee to be in a 0..=1 range.\n /// - The value is <= alpha.\n pub fn red(&self) -> f32 {\n self.r.get()\n }\n\n /// Returns color's green component.\n ///\n /// - The value is guarantee to be in a 0..=1 range.\n /// - The value is <= alpha.\n pub fn green(&self) -> f32 {\n self.g.get()\n }\n\n /// Returns color's blue component.\n ///\n /// - The value is guarantee to be in a 0..=1 range.\n /// - The value is <= alpha.\n pub fn blue(&self) -> f32 {\n self.b.get()\n }\n\n /// Returns color's alpha component.\n ///\n /// - The value is guarantee to be in a 0..=1 range.\n pub fn alpha(&self) -> f32 {\n self.a.get()\n }\n\n /// Returns a demultiplied color.\n pub fn demultiply(&self) -> Color {\n let a = self.a.get();\n if a == 0.0 {\n Color::TRANSPARENT\n } else {\n Color {\n r: NormalizedF32::new_clamped(self.r.get() / a),\n g: NormalizedF32::new_clamped(self.g.get() / a),\n b: NormalizedF32::new_clamped(self.b.get() / a),\n a: self.a,\n }\n }\n }\n\n /// Converts into `PremultipliedColorU8`.\n pub fn to_color_u8(&self) -> PremultipliedColorU8 {\n let c = color_f32_to_u8(self.r, self.g, self.b, self.a);\n PremultipliedColorU8::from_rgba_unchecked(c[0], c[1], c[2], c[3])\n }\n}\n\n/// Return a*b/255, rounding any fractional bits.\npub fn premultiply_u8(c: u8, a: u8) -> u8 {\n let prod = u32::from(c) * u32::from(a) + 128;\n ((prod + (prod >> 8)) >> 8) as u8\n}\n\nfn color_f32_to_u8(\n r: NormalizedF32,\n g: NormalizedF32,\n b: NormalizedF32,\n a: NormalizedF32,\n) -> [u8; 4] {\n [\n (r.get() * 255.0 + 0.5) as u8,\n (g.get() * 255.0 + 0.5) as u8,\n (b.get() * 255.0 + 0.5) as u8,\n (a.get() * 255.0 + 0.5) as u8,\n ]\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn premultiply_u8() {\n assert_eq!(\n ColorU8::from_rgba(10, 20, 30, 40).premultiply(),\n PremultipliedColorU8::from_rgba_unchecked(2, 3, 5, 40)\n );\n }\n\n #[test]\n fn premultiply_u8_opaque() {\n assert_eq!(\n ColorU8::from_rgba(10, 20, 30, 255).premultiply(),\n PremultipliedColorU8::from_rgba_unchecked(10, 20, 30, 255)\n );\n }\n\n #[test]\n fn demultiply_u8_1() {\n assert_eq!(\n PremultipliedColorU8::from_rgba_unchecked(2, 3, 5, 40).demultiply(),\n ColorU8::from_rgba(13, 19, 32, 40)\n );\n }\n\n #[test]\n fn demultiply_u8_2() {\n assert_eq!(\n PremultipliedColorU8::from_rgba_unchecked(10, 20, 30, 255).demultiply(),\n ColorU8::from_rgba(10, 20, 30, 255)\n );\n }\n\n #[test]\n fn demultiply_u8_3() {\n assert_eq!(\n PremultipliedColorU8::from_rgba_unchecked(153, 99, 54, 180).demultiply(),\n ColorU8::from_rgba(217, 140, 77, 180)\n );\n }\n\n #[test]\n fn bytemuck_casts_rgba() {\n let slice = &[\n PremultipliedColorU8::from_rgba_unchecked(0, 1, 2, 3),\n PremultipliedColorU8::from_rgba_unchecked(10, 11, 12, 13),\n ];\n let bytes: &[u8] = bytemuck::cast_slice(slice);\n assert_eq!(bytes, &[0, 1, 2, 3, 10, 11, 12, 13]);\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/edge.rs", "sha256": "6c0c824169937e9720ad824aa2ab68efc8c71c32e553b828fd809dd064e607e1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse crate::Point;\n\nuse crate::fixed_point::{fdot16, fdot6, FDot16, FDot6};\nuse crate::math::left_shift;\n\n/// We store 1< &LineEdge {\n match self {\n Edge::Line(line) => line,\n Edge::Quadratic(quad) => &quad.line,\n Edge::Cubic(cubic) => &cubic.line,\n }\n }\n\n pub fn as_line_mut(&mut self) -> &mut LineEdge {\n match self {\n Edge::Line(line) => line,\n Edge::Quadratic(quad) => &mut quad.line,\n Edge::Cubic(cubic) => &mut cubic.line,\n }\n }\n}\n\nimpl core::ops::Deref for Edge {\n type Target = LineEdge;\n\n fn deref(&self) -> &Self::Target {\n self.as_line()\n }\n}\n\nimpl core::ops::DerefMut for Edge {\n fn deref_mut(&mut self) -> &mut Self::Target {\n self.as_line_mut()\n }\n}\n\n#[derive(Clone, Default, Debug)]\npub struct LineEdge {\n // Imitate a linked list.\n pub prev: Option,\n pub next: Option,\n\n pub x: FDot16,\n pub dx: FDot16,\n pub first_y: i32,\n pub last_y: i32,\n pub winding: i8, // 1 or -1\n}\n\nimpl LineEdge {\n pub fn new(p0: Point, p1: Point, shift: i32) -> Option {\n let scale = (1 << (shift + 6)) as f32;\n let mut x0 = (p0.x * scale) as i32;\n let mut y0 = (p0.y * scale) as i32;\n let mut x1 = (p1.x * scale) as i32;\n let mut y1 = (p1.y * scale) as i32;\n\n let mut winding = 1;\n\n if y0 > y1 {\n core::mem::swap(&mut x0, &mut x1);\n core::mem::swap(&mut y0, &mut y1);\n winding = -1;\n }\n\n let top = fdot6::round(y0);\n let bottom = fdot6::round(y1);\n\n // are we a zero-height line?\n if top == bottom {\n return None;\n }\n\n let slope = fdot6::div(x1 - x0, y1 - y0);\n let dy = compute_dy(top, y0);\n\n Some(LineEdge {\n next: None,\n prev: None,\n x: fdot6::to_fdot16(x0 + fdot16::mul(slope, dy)),\n dx: slope,\n first_y: top,\n last_y: bottom - 1,\n winding,\n })\n }\n\n pub fn is_vertical(&self) -> bool {\n self.dx == 0\n }\n\n fn update(&mut self, mut x0: FDot16, mut y0: FDot16, mut x1: FDot16, mut y1: FDot16) -> bool {\n debug_assert!(self.winding == 1 || self.winding == -1);\n\n y0 >>= 10;\n y1 >>= 10;\n\n debug_assert!(y0 <= y1);\n\n let top = fdot6::round(y0);\n let bottom = fdot6::round(y1);\n\n // are we a zero-height line?\n if top == bottom {\n return false;\n }\n\n x0 >>= 10;\n x1 >>= 10;\n\n let slope = fdot6::div(x1 - x0, y1 - y0);\n let dy = compute_dy(top, y0);\n\n self.x = fdot6::to_fdot16(x0 + fdot16::mul(slope, dy));\n self.dx = slope;\n self.first_y = top;\n self.last_y = bottom - 1;\n\n true\n }\n}\n\n#[derive(Clone, Debug)]\npub struct QuadraticEdge {\n pub line: LineEdge,\n pub curve_count: i8,\n curve_shift: u8, // applied to all dx/ddx/dddx\n qx: FDot16,\n qy: FDot16,\n qdx: FDot16,\n qdy: FDot16,\n qddx: FDot16,\n qddy: FDot16,\n q_last_x: FDot16,\n q_last_y: FDot16,\n}\n\nimpl QuadraticEdge {\n pub fn new(points: &[Point], shift: i32) -> Option {\n let mut quad = Self::new2(points, shift)?;\n if quad.update() {\n Some(quad)\n } else {\n None\n }\n }\n\n fn new2(points: &[Point], mut shift: i32) -> Option {\n let scale = (1 << (shift + 6)) as f32;\n let mut x0 = (points[0].x * scale) as i32;\n let mut y0 = (points[0].y * scale) as i32;\n let x1 = (points[1].x * scale) as i32;\n let y1 = (points[1].y * scale) as i32;\n let mut x2 = (points[2].x * scale) as i32;\n let mut y2 = (points[2].y * scale) as i32;\n\n let mut winding = 1;\n if y0 > y2 {\n core::mem::swap(&mut x0, &mut x2);\n core::mem::swap(&mut y0, &mut y2);\n winding = -1;\n }\n debug_assert!(y0 <= y1 && y1 <= y2);\n\n let top = fdot6::round(y0);\n let bottom = fdot6::round(y2);\n\n // are we a zero-height quad (line)?\n if top == bottom {\n return None;\n }\n\n // compute number of steps needed (1 << shift)\n {\n let dx = (left_shift(x1, 1) - x0 - x2) >> 2;\n let dy = (left_shift(y1, 1) - y0 - y2) >> 2;\n // This is a little confusing:\n // before this line, shift is the scale up factor for AA;\n // after this line, shift is the fCurveShift.\n shift = diff_to_shift(dx, dy, shift);\n debug_assert!(shift >= 0);\n }\n\n // need at least 1 subdivision for our bias trick\n if shift == 0 {\n shift = 1;\n } else if shift > MAX_COEFF_SHIFT {\n shift = MAX_COEFF_SHIFT;\n }\n\n let curve_count = (1 << shift) as i8;\n\n // We want to reformulate into polynomial form, to make it clear how we\n // should forward-difference.\n //\n // p0 (1 - t)^2 + p1 t(1 - t) + p2 t^2 ==> At^2 + Bt + C\n //\n // A = p0 - 2p1 + p2\n // B = 2(p1 - p0)\n // C = p0\n //\n // Our caller must have constrained our inputs (p0..p2) to all fit into\n // 16.16. However, as seen above, we sometimes compute values that can be\n // larger (e.g. B = 2*(p1 - p0)). To guard against overflow, we will store\n // A and B at 1/2 of their actual value, and just apply a 2x scale during\n // application in updateQuadratic(). Hence we store (shift - 1) in\n // curve_shift.\n\n let curve_shift = (shift - 1) as u8;\n\n let mut a = fdot6_to_fixed_div2(x0 - x1 - x1 + x2); // 1/2 the real value\n let mut b = fdot6::to_fdot16(x1 - x0); // 1/2 the real value\n\n let qx = fdot6::to_fdot16(x0);\n let qdx = b + (a >> shift); // biased by shift\n let qddx = a >> (shift - 1); // biased by shift\n\n a = fdot6_to_fixed_div2(y0 - y1 - y1 + y2); // 1/2 the real value\n b = fdot6::to_fdot16(y1 - y0); // 1/2 the real value\n\n let qy = fdot6::to_fdot16(y0);\n let qdy = b + (a >> shift); // biased by shift\n let qddy = a >> (shift - 1); // biased by shift\n\n let q_last_x = fdot6::to_fdot16(x2);\n let q_last_y = fdot6::to_fdot16(y2);\n\n Some(QuadraticEdge {\n line: LineEdge {\n next: None,\n prev: None,\n x: 0,\n dx: 0,\n first_y: 0,\n last_y: 0,\n winding,\n },\n curve_count,\n curve_shift,\n qx,\n qy,\n qdx,\n qdy,\n qddx,\n qddy,\n q_last_x,\n q_last_y,\n })\n }\n\n pub fn update(&mut self) -> bool {\n let mut success;\n let mut count = self.curve_count;\n let mut oldx = self.qx;\n let mut oldy = self.qy;\n let mut dx = self.qdx;\n let mut dy = self.qdy;\n let mut newx;\n let mut newy;\n let shift = self.curve_shift;\n\n debug_assert!(count > 0);\n\n loop {\n count -= 1;\n if count > 0 {\n newx = oldx + (dx >> shift);\n dx += self.qddx;\n newy = oldy + (dy >> shift);\n dy += self.qddy;\n } else {\n // last segment\n newx = self.q_last_x;\n newy = self.q_last_y;\n }\n success = self.line.update(oldx, oldy, newx, newy);\n oldx = newx;\n oldy = newy;\n\n if count == 0 || success {\n break;\n }\n }\n\n self.qx = newx;\n self.qy = newy;\n self.qdx = dx;\n self.qdy = dy;\n self.curve_count = count as i8;\n\n success\n }\n}\n\n#[derive(Clone, Debug)]\npub struct CubicEdge {\n pub line: LineEdge,\n pub curve_count: i8,\n curve_shift: u8, // applied to all dx/ddx/dddx except for dshift exception\n dshift: u8, // applied to cdx and cdy\n cx: FDot16,\n cy: FDot16,\n cdx: FDot16,\n cdy: FDot16,\n cddx: FDot16,\n cddy: FDot16,\n cdddx: FDot16,\n cdddy: FDot16,\n c_last_x: FDot16,\n c_last_y: FDot16,\n}\n\nimpl CubicEdge {\n pub fn new(points: &[Point], shift: i32) -> Option {\n let mut cubic = Self::new2(points, shift, true)?;\n if cubic.update() {\n Some(cubic)\n } else {\n None\n }\n }\n\n fn new2(points: &[Point], mut shift: i32, sort_y: bool) -> Option {\n let scale = (1 << (shift + 6)) as f32;\n let mut x0 = (points[0].x * scale) as i32;\n let mut y0 = (points[0].y * scale) as i32;\n let mut x1 = (points[1].x * scale) as i32;\n let mut y1 = (points[1].y * scale) as i32;\n let mut x2 = (points[2].x * scale) as i32;\n let mut y2 = (points[2].y * scale) as i32;\n let mut x3 = (points[3].x * scale) as i32;\n let mut y3 = (points[3].y * scale) as i32;\n\n let mut winding = 1;\n if sort_y && y0 > y3 {\n core::mem::swap(&mut x0, &mut x3);\n core::mem::swap(&mut x1, &mut x2);\n core::mem::swap(&mut y0, &mut y3);\n core::mem::swap(&mut y1, &mut y2);\n winding = -1;\n }\n\n let top = fdot6::round(y0);\n let bot = fdot6::round(y3);\n\n // are we a zero-height cubic (line)?\n if sort_y && top == bot {\n return None;\n }\n\n // compute number of steps needed (1 << shift)\n {\n // Can't use (center of curve - center of baseline), since center-of-curve\n // need not be the max delta from the baseline (it could even be coincident)\n // so we try just looking at the two off-curve points\n let dx = cubic_delta_from_line(x0, x1, x2, x3);\n let dy = cubic_delta_from_line(y0, y1, y2, y3);\n // add 1 (by observation)\n shift = diff_to_shift(dx, dy, 2) + 1;\n }\n // need at least 1 subdivision for our bias trick\n debug_assert!(shift > 0);\n if shift > MAX_COEFF_SHIFT {\n shift = MAX_COEFF_SHIFT;\n }\n\n // Since our in coming data is initially shifted down by 10 (or 8 in\n // antialias). That means the most we can shift up is 8. However, we\n // compute coefficients with a 3*, so the safest upshift is really 6\n let mut up_shift = 6; // largest safe value\n let mut down_shift = shift + up_shift - 10;\n if down_shift < 0 {\n down_shift = 0;\n up_shift = 10 - shift;\n }\n\n let curve_count = left_shift(-1, shift) as i8;\n let curve_shift = shift as u8;\n let dshift = down_shift as u8;\n\n let mut b = fdot6_up_shift(3 * (x1 - x0), up_shift);\n let mut c = fdot6_up_shift(3 * (x0 - x1 - x1 + x2), up_shift);\n let mut d = fdot6_up_shift(x3 + 3 * (x1 - x2) - x0, up_shift);\n\n let cx = fdot6::to_fdot16(x0);\n let cdx = b + (c >> shift) + (d >> (2 * shift)); // biased by shift\n let cddx = 2 * c + ((3 * d) >> (shift - 1)); // biased by 2*shift\n let cdddx = (3 * d) >> (shift - 1); // biased by 2*shift\n\n b = fdot6_up_shift(3 * (y1 - y0), up_shift);\n c = fdot6_up_shift(3 * (y0 - y1 - y1 + y2), up_shift);\n d = fdot6_up_shift(y3 + 3 * (y1 - y2) - y0, up_shift);\n\n let cy = fdot6::to_fdot16(y0);\n let cdy = b + (c >> shift) + (d >> (2 * shift)); // biased by shift\n let cddy = 2 * c + ((3 * d) >> (shift - 1)); // biased by 2*shift\n let cdddy = (3 * d) >> (shift - 1); // biased by 2*shift\n\n let c_last_x = fdot6::to_fdot16(x3);\n let c_last_y = fdot6::to_fdot16(y3);\n\n Some(CubicEdge {\n line: LineEdge {\n next: None,\n prev: None,\n x: 0,\n dx: 0,\n first_y: 0,\n last_y: 0,\n winding,\n },\n curve_count,\n curve_shift,\n dshift,\n cx,\n cy,\n cdx,\n cdy,\n cddx,\n cddy,\n cdddx,\n cdddy,\n c_last_x,\n c_last_y,\n })\n }\n\n pub fn update(&mut self) -> bool {\n let mut success;\n let mut count = self.curve_count;\n let mut oldx = self.cx;\n let mut oldy = self.cy;\n let mut newx;\n let mut newy;\n let ddshift = self.curve_shift;\n let dshift = self.dshift;\n\n debug_assert!(count < 0);\n\n loop {\n count += 1;\n if count < 0 {\n newx = oldx + (self.cdx >> dshift);\n self.cdx += self.cddx >> ddshift;\n self.cddx += self.cdddx;\n\n newy = oldy + (self.cdy >> dshift);\n self.cdy += self.cddy >> ddshift;\n self.cddy += self.cdddy;\n } else {\n // last segment\n newx = self.c_last_x;\n newy = self.c_last_y;\n }\n\n // we want to say debug_assert(oldy <= newy), but our finite fixedpoint\n // doesn't always achieve that, so we have to explicitly pin it here.\n if newy < oldy {\n newy = oldy;\n }\n\n success = self.line.update(oldx, oldy, newx, newy);\n oldx = newx;\n oldy = newy;\n\n if count == 0 || success {\n break;\n }\n }\n\n self.cx = newx;\n self.cy = newy;\n self.curve_count = count;\n\n success\n }\n}\n\n// This correctly favors the lower-pixel when y0 is on a 1/2 pixel boundary\nfn compute_dy(top: FDot6, y0: FDot6) -> FDot6 {\n left_shift(top, 6) + 32 - y0\n}\n\nfn diff_to_shift(dx: FDot6, dy: FDot6, shift_aa: i32) -> i32 {\n // cheap calc of distance from center of p0-p2 to the center of the curve\n let mut dist = cheap_distance(dx, dy);\n\n // shift down dist (it is currently in dot6)\n // down by 3 should give us 1/8 pixel accuracy (assuming our dist is accurate...)\n // this is chosen by heuristic: make it as big as possible (to minimize segments)\n // ... but small enough so that our curves still look smooth\n // When shift > 0, we're using AA and everything is scaled up so we can\n // lower the accuracy.\n dist = (dist + (1 << 4)) >> (3 + shift_aa);\n\n // each subdivision (shift value) cuts this dist (error) by 1/4\n (32 - dist.leading_zeros() as i32) >> 1\n}\n\nfn cheap_distance(mut dx: FDot6, mut dy: FDot6) -> FDot6 {\n dx = dx.abs();\n dy = dy.abs();\n // return max + min/2\n if dx > dy {\n dx + (dy >> 1)\n } else {\n dy + (dx >> 1)\n }\n}\n\n// In LineEdge::new, QuadraticEdge::new, CubicEdge::new, the first thing we do is to convert\n// the points into FDot6. This is modulated by the shift parameter, which\n// will either be 0, or something like 2 for antialiasing.\n//\n// In the float case, we want to turn the float into .6 by saying pt * 64,\n// or pt * 256 for antialiasing. This is implemented as 1 << (shift + 6).\n//\n// In the fixed case, we want to turn the fixed into .6 by saying pt >> 10,\n// or pt >> 8 for antialiasing. This is implemented as pt >> (10 - shift).\nfn fdot6_to_fixed_div2(value: FDot6) -> FDot16 {\n // we want to return SkFDot6ToFixed(value >> 1), but we don't want to throw\n // away data in value, so just perform a modify up-shift\n left_shift(value, 16 - 6 - 1)\n}\n\nfn fdot6_up_shift(x: FDot6, up_shift: i32) -> i32 {\n debug_assert!((left_shift(x, up_shift) >> up_shift) == x);\n left_shift(x, up_shift)\n}\n\n// f(1/3) = (8a + 12b + 6c + d) / 27\n// f(2/3) = (a + 6b + 12c + 8d) / 27\n//\n// f(1/3)-b = (8a - 15b + 6c + d) / 27\n// f(2/3)-c = (a + 6b - 15c + 8d) / 27\n//\n// use 16/512 to approximate 1/27\nfn cubic_delta_from_line(a: FDot6, b: FDot6, c: FDot6, d: FDot6) -> FDot6 {\n // since our parameters may be negative, we don't use <<\n let one_third = ((a * 8 - b * 15 + 6 * c + d) * 19) >> 9;\n let two_third = ((a + 6 * b - c * 15 + d * 8) * 19) >> 9;\n\n one_third.abs().max(two_third.abs())\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/edge_builder.rs", "sha256": "82f050de9a216141ca6c1a4d190bc001ecaed90eb65998d05a69b2016bf3310d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2011 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse alloc::vec::Vec;\n\nuse tiny_skia_path::PathVerb;\n\nuse crate::{Path, Point};\n\nuse crate::edge::{CubicEdge, Edge, LineEdge, QuadraticEdge};\nuse crate::edge_clipper::EdgeClipperIter;\nuse crate::geom::ScreenIntRect;\nuse crate::path_geometry;\n\n#[derive(Copy, Clone, PartialEq, Debug)]\nenum Combine {\n No,\n Partial,\n Total,\n}\n\n#[derive(Copy, Clone, Debug)]\npub struct ShiftedIntRect {\n shifted: ScreenIntRect,\n shift: i32,\n}\n\nimpl ShiftedIntRect {\n pub fn new(rect: &ScreenIntRect, shift: i32) -> Option {\n let shifted = ScreenIntRect::from_xywh(\n rect.x() << shift,\n rect.y() << shift,\n rect.width() << shift,\n rect.height() << shift,\n )?;\n Some(ShiftedIntRect { shifted, shift })\n }\n\n pub fn shifted(&self) -> &ScreenIntRect {\n &self.shifted\n }\n\n pub fn recover(&self) -> ScreenIntRect {\n ScreenIntRect::from_xywh(\n self.shifted.x() >> self.shift,\n self.shifted.y() >> self.shift,\n self.shifted.width() >> self.shift,\n self.shifted.height() >> self.shift,\n )\n .unwrap() // cannot fail, because the original rect was valid\n }\n}\n\npub struct BasicEdgeBuilder {\n edges: Vec,\n clip_shift: i32,\n}\n\nimpl BasicEdgeBuilder {\n pub fn new(clip_shift: i32) -> Self {\n BasicEdgeBuilder {\n edges: Vec::with_capacity(64), // TODO: stack array + fallback\n clip_shift,\n }\n }\n\n // Skia returns a linked list here, but it's a nightmare to use in Rust,\n // so we're mimicking it with Vec.\n pub fn build_edges(\n path: &Path,\n clip: Option<&ShiftedIntRect>,\n clip_shift: i32,\n ) -> Option> {\n // If we're convex, then we need both edges, even if the right edge is past the clip.\n // let can_cull_to_the_right = !path.isConvex();\n let can_cull_to_the_right = false; // TODO: this\n\n let mut builder = BasicEdgeBuilder::new(clip_shift);\n if !builder.build(path, clip, can_cull_to_the_right) {\n log::warn!(\"infinite or NaN segments detected during edges building\");\n return None;\n }\n\n if builder.edges.len() < 2 {\n return None;\n }\n\n Some(builder.edges)\n }\n\n // TODO: build_poly\n pub fn build(\n &mut self,\n path: &Path,\n clip: Option<&ShiftedIntRect>,\n can_cull_to_the_right: bool,\n ) -> bool {\n if let Some(clip) = clip {\n let clip = clip.recover().to_rect();\n for edges in EdgeClipperIter::new(path, clip, can_cull_to_the_right) {\n for edge in edges {\n match edge {\n PathEdge::LineTo(p0, p1) => {\n if !p0.is_finite() || !p1.is_finite() {\n return false;\n }\n\n self.push_line(&[p0, p1])\n }\n PathEdge::QuadTo(p0, p1, p2) => {\n if !p0.is_finite() || !p1.is_finite() || !p2.is_finite() {\n return false;\n }\n\n self.push_quad(&[p0, p1, p2])\n }\n PathEdge::CubicTo(p0, p1, p2, p3) => {\n if !p0.is_finite()\n || !p1.is_finite()\n || !p2.is_finite()\n || !p3.is_finite()\n {\n return false;\n }\n\n self.push_cubic(&[p0, p1, p2, p3])\n }\n }\n }\n }\n } else {\n for edge in edge_iter(path) {\n match edge {\n PathEdge::LineTo(p0, p1) => {\n self.push_line(&[p0, p1]);\n }\n PathEdge::QuadTo(p0, p1, p2) => {\n let points = [p0, p1, p2];\n let mut mono_x = [Point::zero(); 5];\n let n = path_geometry::chop_quad_at_y_extrema(&points, &mut mono_x);\n for i in 0..=n {\n self.push_quad(&mono_x[i * 2..]);\n }\n }\n PathEdge::CubicTo(p0, p1, p2, p3) => {\n let points = [p0, p1, p2, p3];\n let mut mono_y = [Point::zero(); 10];\n let n = path_geometry::chop_cubic_at_y_extrema(&points, &mut mono_y);\n for i in 0..=n {\n self.push_cubic(&mono_y[i * 3..]);\n }\n }\n }\n }\n }\n\n true\n }\n\n fn push_line(&mut self, points: &[Point; 2]) {\n if let Some(edge) = LineEdge::new(points[0], points[1], self.clip_shift) {\n let combine = if edge.is_vertical() && !self.edges.is_empty() {\n if let Some(Edge::Line(last)) = self.edges.last_mut() {\n combine_vertical(&edge, last)\n } else {\n Combine::No\n }\n } else {\n Combine::No\n };\n\n match combine {\n Combine::Total => {\n self.edges.pop();\n }\n Combine::Partial => {}\n Combine::No => self.edges.push(Edge::Line(edge)),\n }\n }\n }\n\n fn push_quad(&mut self, points: &[Point]) {\n if let Some(edge) = QuadraticEdge::new(points, self.clip_shift) {\n self.edges.push(Edge::Quadratic(edge));\n }\n }\n\n fn push_cubic(&mut self, points: &[Point]) {\n if let Some(edge) = CubicEdge::new(points, self.clip_shift) {\n self.edges.push(Edge::Cubic(edge));\n }\n }\n}\n\nfn combine_vertical(edge: &LineEdge, last: &mut LineEdge) -> Combine {\n if last.dx != 0 || edge.x != last.x {\n return Combine::No;\n }\n\n if edge.winding == last.winding {\n return if edge.last_y + 1 == last.first_y {\n last.first_y = edge.first_y;\n Combine::Partial\n } else if edge.first_y == last.last_y + 1 {\n last.last_y = edge.last_y;\n Combine::Partial\n } else {\n Combine::No\n };\n }\n\n if edge.first_y == last.first_y {\n return if edge.last_y == last.last_y {\n Combine::Total\n } else if edge.last_y < last.last_y {\n last.first_y = edge.last_y + 1;\n Combine::Partial\n } else {\n last.first_y = last.last_y + 1;\n last.last_y = edge.last_y;\n last.winding = edge.winding;\n Combine::Partial\n };\n }\n\n if edge.last_y == last.last_y {\n if edge.first_y > last.first_y {\n last.last_y = edge.first_y - 1;\n } else {\n last.last_y = last.first_y - 1;\n last.first_y = edge.first_y;\n last.winding = edge.winding;\n }\n\n return Combine::Partial;\n }\n\n Combine::No\n}\n\npub fn edge_iter(path: &Path) -> PathEdgeIter {\n PathEdgeIter {\n path,\n verb_index: 0,\n points_index: 0,\n move_to: Point::zero(),\n needs_close_line: false,\n }\n}\n\n#[derive(Copy, Clone, PartialEq, Debug)]\npub enum PathEdge {\n LineTo(Point, Point),\n QuadTo(Point, Point, Point),\n CubicTo(Point, Point, Point, Point),\n}\n\n/// Lightweight variant of PathIter that only returns segments (e.g. lines/quads).\n///\n/// Does not return Move or Close. Always \"auto-closes\" each contour.\npub struct PathEdgeIter<'a> {\n path: &'a Path,\n verb_index: usize,\n points_index: usize,\n move_to: Point,\n needs_close_line: bool,\n}\n\nimpl<'a> PathEdgeIter<'a> {\n fn close_line(&mut self) -> Option {\n self.needs_close_line = false;\n\n let edge = PathEdge::LineTo(self.path.points()[self.points_index - 1], self.move_to);\n Some(edge)\n }\n}\n\nimpl<'a> Iterator for PathEdgeIter<'a> {\n type Item = PathEdge;\n\n fn next(&mut self) -> Option {\n if self.verb_index < self.path.verbs().len() {\n let verb = self.path.verbs()[self.verb_index];\n self.verb_index += 1;\n\n match verb {\n PathVerb::Move => {\n if self.needs_close_line {\n let res = self.close_line();\n self.move_to = self.path.points()[self.points_index];\n self.points_index += 1;\n return res;\n }\n\n self.move_to = self.path.points()[self.points_index];\n self.points_index += 1;\n self.next()\n }\n PathVerb::Close => {\n if self.needs_close_line {\n return self.close_line();\n }\n\n self.next()\n }\n _ => {\n // Actual edge.\n self.needs_close_line = true;\n\n let edge;\n match verb {\n PathVerb::Line => {\n edge = PathEdge::LineTo(\n self.path.points()[self.points_index - 1],\n self.path.points()[self.points_index + 0],\n );\n self.points_index += 1;\n }\n PathVerb::Quad => {\n edge = PathEdge::QuadTo(\n self.path.points()[self.points_index - 1],\n self.path.points()[self.points_index + 0],\n self.path.points()[self.points_index + 1],\n );\n self.points_index += 2;\n }\n PathVerb::Cubic => {\n edge = PathEdge::CubicTo(\n self.path.points()[self.points_index - 1],\n self.path.points()[self.points_index + 0],\n self.path.points()[self.points_index + 1],\n self.path.points()[self.points_index + 2],\n );\n self.points_index += 3;\n }\n _ => unreachable!(),\n };\n\n Some(edge)\n }\n }\n } else if self.needs_close_line {\n self.close_line()\n } else {\n None\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/edge_clipper.rs", "sha256": "ecc3c4ea2da87cc4dce22a8e61b828528bee875ec096ce53f2c580aa2d493dc2", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2009 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse arrayvec::ArrayVec;\n\nuse tiny_skia_path::{NormalizedF32Exclusive, SCALAR_MAX};\n\nuse crate::{Path, Point, Rect};\n\nuse crate::edge_builder::{edge_iter, PathEdge, PathEdgeIter};\nuse crate::line_clipper;\nuse crate::path_geometry;\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\n// This is a fail-safe `arr[n..n+3].try_into().unwrap()` alternative.\n// Everything is checked at compile-time so there is no bound checking and panics.\nmacro_rules! copy_3_points {\n ($arr:expr, $i:expr) => {\n [$arr[$i], $arr[$i + 1], $arr[$i + 2]]\n };\n}\n\nmacro_rules! copy_4_points {\n ($arr:expr, $i:expr) => {\n [$arr[$i], $arr[$i + 1], $arr[$i + 2], $arr[$i + 3]]\n };\n}\n\n/// Max curvature in X and Y split cubic into 9 pieces, * (line + cubic).\nconst MAX_VERBS: usize = 18;\n\npub type ClippedEdges = ArrayVec;\n\npub struct EdgeClipper {\n clip: Rect,\n can_cull_to_the_right: bool,\n edges: ClippedEdges,\n}\n\nimpl EdgeClipper {\n fn new(clip: Rect, can_cull_to_the_right: bool) -> Self {\n EdgeClipper {\n clip,\n can_cull_to_the_right,\n edges: ArrayVec::new(),\n }\n }\n\n fn clip_line(mut self, p0: Point, p1: Point) -> Option {\n let mut points = [Point::zero(); line_clipper::MAX_POINTS];\n let points = line_clipper::clip(\n &[p0, p1],\n &self.clip,\n self.can_cull_to_the_right,\n &mut points,\n );\n if !points.is_empty() {\n for i in 0..points.len() - 1 {\n self.push_line(points[i], points[i + 1]);\n }\n }\n\n if self.edges.is_empty() {\n None\n } else {\n Some(self.edges)\n }\n }\n\n fn push_line(&mut self, p0: Point, p1: Point) {\n self.edges.push(PathEdge::LineTo(p0, p1));\n }\n\n fn push_vline(&mut self, x: f32, mut y0: f32, mut y1: f32, reverse: bool) {\n if reverse {\n core::mem::swap(&mut y0, &mut y1);\n }\n\n self.edges.push(PathEdge::LineTo(\n Point::from_xy(x, y0),\n Point::from_xy(x, y1),\n ));\n }\n\n fn clip_quad(mut self, p0: Point, p1: Point, p2: Point) -> Option {\n let pts = [p0, p1, p2];\n let bounds = Rect::from_points(&pts)?;\n\n if !quick_reject(&bounds, &self.clip) {\n let mut mono_y = [Point::zero(); 5];\n let count_y = path_geometry::chop_quad_at_y_extrema(&pts, &mut mono_y);\n for y in 0..=count_y {\n let mut mono_x = [Point::zero(); 5];\n let y_points: [Point; 3] = copy_3_points!(mono_y, y * 2);\n let count_x = path_geometry::chop_quad_at_x_extrema(&y_points, &mut mono_x);\n for x in 0..=count_x {\n let x_points: [Point; 3] = copy_3_points!(mono_x, x * 2);\n self.clip_mono_quad(&x_points);\n }\n }\n }\n\n if self.edges.is_empty() {\n None\n } else {\n Some(self.edges)\n }\n }\n\n // src[] must be monotonic in X and Y\n fn clip_mono_quad(&mut self, src: &[Point; 3]) {\n let mut pts = [Point::zero(); 3];\n let mut reverse = sort_increasing_y(src, &mut pts);\n\n // are we completely above or below\n if pts[2].y <= self.clip.top() || pts[0].y >= self.clip.bottom() {\n return;\n }\n\n // Now chop so that pts is contained within clip in Y\n chop_quad_in_y(&self.clip, &mut pts);\n\n if pts[0].x > pts[2].x {\n pts.swap(0, 2);\n reverse = !reverse;\n }\n debug_assert!(pts[0].x <= pts[1].x);\n debug_assert!(pts[1].x <= pts[2].x);\n\n // Now chop in X has needed, and record the segments\n\n if pts[2].x <= self.clip.left() {\n // wholly to the left\n self.push_vline(self.clip.left(), pts[0].y, pts[2].y, reverse);\n return;\n }\n\n if pts[0].x >= self.clip.right() {\n // wholly to the right\n if !self.can_cull_to_the_right {\n self.push_vline(self.clip.right(), pts[0].y, pts[2].y, reverse);\n }\n\n return;\n }\n\n let mut t = NormalizedF32Exclusive::ANY;\n let mut tmp = [Point::zero(); 5];\n\n // are we partially to the left\n if pts[0].x < self.clip.left() {\n if chop_mono_quad_at_x(&pts, self.clip.left(), &mut t) {\n path_geometry::chop_quad_at(&pts, t, &mut tmp);\n self.push_vline(self.clip.left(), tmp[0].y, tmp[2].y, reverse);\n // clamp to clean up imprecise numerics in the chop\n tmp[2].x = self.clip.left();\n tmp[3].x = tmp[3].x.max(self.clip.left());\n\n pts[0] = tmp[2];\n pts[1] = tmp[3];\n } else {\n // if chopMonoQuadAtY failed, then we may have hit inexact numerics\n // so we just clamp against the left\n self.push_vline(self.clip.left(), pts[0].y, pts[2].y, reverse);\n return;\n }\n }\n\n // are we partially to the right\n if pts[2].x > self.clip.right() {\n if chop_mono_quad_at_x(&pts, self.clip.right(), &mut t) {\n path_geometry::chop_quad_at(&pts, t, &mut tmp);\n // clamp to clean up imprecise numerics in the chop\n tmp[1].x = tmp[1].x.min(self.clip.right());\n tmp[2].x = self.clip.right();\n\n self.push_quad(©_3_points!(tmp, 0), reverse);\n self.push_vline(self.clip.right(), tmp[2].y, tmp[4].y, reverse);\n } else {\n // if chopMonoQuadAtY failed, then we may have hit inexact numerics\n // so we just clamp against the right\n pts[1].x = pts[1].x.min(self.clip.right());\n pts[2].x = pts[2].x.min(self.clip.right());\n self.push_quad(&pts, reverse);\n }\n } else {\n // wholly inside the clip\n self.push_quad(&pts, reverse);\n }\n }\n\n fn push_quad(&mut self, pts: &[Point; 3], reverse: bool) {\n if reverse {\n self.edges.push(PathEdge::QuadTo(pts[2], pts[1], pts[0]));\n } else {\n self.edges.push(PathEdge::QuadTo(pts[0], pts[1], pts[2]));\n }\n }\n\n fn clip_cubic(mut self, p0: Point, p1: Point, p2: Point, p3: Point) -> Option {\n let pts = [p0, p1, p2, p3];\n let bounds = Rect::from_points(&pts)?;\n\n // check if we're clipped out vertically\n if bounds.bottom() > self.clip.top() && bounds.top() < self.clip.bottom() {\n if too_big_for_reliable_float_math(&bounds) {\n // can't safely clip the cubic, so we give up and draw a line (which we can safely clip)\n //\n // If we rewrote chopcubicat*extrema and chopmonocubic using doubles, we could very\n // likely always handle the cubic safely, but (it seems) at a big loss in speed, so\n // we'd only want to take that alternate impl if needed.\n return self.clip_line(p0, p3);\n } else {\n let mut mono_y = [Point::zero(); 10];\n let count_y = path_geometry::chop_cubic_at_y_extrema(&pts, &mut mono_y);\n for y in 0..=count_y {\n let mut mono_x = [Point::zero(); 10];\n let y_points: [Point; 4] = copy_4_points!(mono_y, y * 3);\n let count_x = path_geometry::chop_cubic_at_x_extrema(&y_points, &mut mono_x);\n for x in 0..=count_x {\n let x_points: [Point; 4] = copy_4_points!(mono_x, x * 3);\n self.clip_mono_cubic(&x_points);\n }\n }\n }\n }\n\n if self.edges.is_empty() {\n None\n } else {\n Some(self.edges)\n }\n }\n\n // src[] must be monotonic in X and Y\n fn clip_mono_cubic(&mut self, src: &[Point; 4]) {\n let mut pts = [Point::zero(); 4];\n let mut reverse = sort_increasing_y(src, &mut pts);\n\n // are we completely above or below\n if pts[3].y <= self.clip.top() || pts[0].y >= self.clip.bottom() {\n return;\n }\n\n // Now chop so that pts is contained within clip in Y\n chop_cubic_in_y(&self.clip, &mut pts);\n\n if pts[0].x > pts[3].x {\n pts.swap(0, 3);\n pts.swap(1, 2);\n reverse = !reverse;\n }\n\n // Now chop in X has needed, and record the segments\n\n if pts[3].x <= self.clip.left() {\n // wholly to the left\n self.push_vline(self.clip.left(), pts[0].y, pts[3].y, reverse);\n return;\n }\n\n if pts[0].x >= self.clip.right() {\n // wholly to the right\n if !self.can_cull_to_the_right {\n self.push_vline(self.clip.right(), pts[0].y, pts[3].y, reverse);\n }\n\n return;\n }\n\n // are we partially to the left\n if pts[0].x < self.clip.left() {\n let mut tmp = [Point::zero(); 7];\n chop_mono_cubic_at_x(&pts, self.clip.left(), &mut tmp);\n self.push_vline(self.clip.left(), tmp[0].y, tmp[3].y, reverse);\n\n // tmp[3, 4].fX should all be to the right of clip.left().\n // Since we can't trust the numerics of\n // the chopper, we force those conditions now\n tmp[3].x = self.clip.left();\n tmp[4].x = tmp[4].x.max(self.clip.left());\n\n pts[0] = tmp[3];\n pts[1] = tmp[4];\n pts[2] = tmp[5];\n }\n\n // are we partially to the right\n if pts[3].x > self.clip.right() {\n let mut tmp = [Point::zero(); 7];\n chop_mono_cubic_at_x(&pts, self.clip.right(), &mut tmp);\n tmp[3].x = self.clip.right();\n tmp[2].x = tmp[2].x.min(self.clip.right());\n\n self.push_cubic(©_4_points!(tmp, 0), reverse);\n self.push_vline(self.clip.right(), tmp[3].y, tmp[6].y, reverse);\n } else {\n // wholly inside the clip\n self.push_cubic(&pts, reverse);\n }\n }\n\n fn push_cubic(&mut self, pts: &[Point; 4], reverse: bool) {\n if reverse {\n self.edges\n .push(PathEdge::CubicTo(pts[3], pts[2], pts[1], pts[0]));\n } else {\n self.edges\n .push(PathEdge::CubicTo(pts[0], pts[1], pts[2], pts[3]));\n }\n }\n}\n\npub struct EdgeClipperIter<'a> {\n edge_iter: PathEdgeIter<'a>,\n clip: Rect,\n can_cull_to_the_right: bool,\n}\n\nimpl<'a> EdgeClipperIter<'a> {\n pub fn new(path: &'a Path, clip: Rect, can_cull_to_the_right: bool) -> Self {\n EdgeClipperIter {\n edge_iter: edge_iter(path),\n clip,\n can_cull_to_the_right,\n }\n }\n}\n\nimpl Iterator for EdgeClipperIter<'_> {\n type Item = ClippedEdges;\n\n fn next(&mut self) -> Option {\n for edge in &mut self.edge_iter {\n let clipper = EdgeClipper::new(self.clip, self.can_cull_to_the_right);\n\n match edge {\n PathEdge::LineTo(p0, p1) => {\n if let Some(edges) = clipper.clip_line(p0, p1) {\n return Some(edges);\n }\n }\n PathEdge::QuadTo(p0, p1, p2) => {\n if let Some(edges) = clipper.clip_quad(p0, p1, p2) {\n return Some(edges);\n }\n }\n PathEdge::CubicTo(p0, p1, p2, p3) => {\n if let Some(edges) = clipper.clip_cubic(p0, p1, p2, p3) {\n return Some(edges);\n }\n }\n }\n }\n\n None\n }\n}\n\nfn quick_reject(bounds: &Rect, clip: &Rect) -> bool {\n bounds.top() >= clip.bottom() || bounds.bottom() <= clip.top()\n}\n\n// src[] must be monotonic in Y. This routine copies src into dst, and sorts\n// it to be increasing in Y. If it had to reverse the order of the points,\n// it returns true, otherwise it returns false\nfn sort_increasing_y(src: &[Point], dst: &mut [Point]) -> bool {\n // We need the data to be monotonically increasing in Y.\n // Never fails, because src is always non-empty.\n if src[0].y > src.last().unwrap().y {\n for (i, p) in src.iter().rev().enumerate() {\n dst[i] = *p;\n }\n\n true\n } else {\n dst[0..src.len()].copy_from_slice(src);\n false\n }\n}\n\n/// Modifies pts[] in place so that it is clipped in Y to the clip rect.\nfn chop_quad_in_y(clip: &Rect, pts: &mut [Point; 3]) {\n let mut t = NormalizedF32Exclusive::ANY;\n let mut tmp = [Point::zero(); 5];\n\n // are we partially above\n if pts[0].y < clip.top() {\n if chop_mono_quad_at_y(pts, clip.top(), &mut t) {\n // take the 2nd chopped quad\n path_geometry::chop_quad_at(pts, t, &mut tmp);\n // clamp to clean up imprecise numerics in the chop\n tmp[2].y = clip.top();\n tmp[3].y = tmp[3].y.max(clip.top());\n\n pts[0] = tmp[2];\n pts[1] = tmp[3];\n } else {\n // if chop_mono_quad_at_y failed, then we may have hit inexact numerics\n // so we just clamp against the top\n for p in pts.iter_mut() {\n if p.y < clip.top() {\n p.y = clip.top();\n }\n }\n }\n }\n\n // are we partially below\n if pts[2].y > clip.bottom() {\n if chop_mono_quad_at_y(pts, clip.bottom(), &mut t) {\n path_geometry::chop_quad_at(pts, t, &mut tmp);\n // clamp to clean up imprecise numerics in the chop\n tmp[1].y = tmp[1].y.min(clip.bottom());\n tmp[2].y = clip.bottom();\n\n pts[1] = tmp[1];\n pts[2] = tmp[2];\n } else {\n // if chop_mono_quad_at_y failed, then we may have hit inexact numerics\n // so we just clamp against the bottom\n for p in pts.iter_mut() {\n if p.y > clip.bottom() {\n p.y = clip.bottom();\n }\n }\n }\n }\n}\n\nfn chop_mono_quad_at_x(pts: &[Point; 3], x: f32, t: &mut NormalizedF32Exclusive) -> bool {\n chop_mono_quad_at(pts[0].x, pts[1].x, pts[2].x, x, t)\n}\n\nfn chop_mono_quad_at_y(pts: &[Point; 3], y: f32, t: &mut NormalizedF32Exclusive) -> bool {\n chop_mono_quad_at(pts[0].y, pts[1].y, pts[2].y, y, t)\n}\n\nfn chop_mono_quad_at(\n c0: f32,\n c1: f32,\n c2: f32,\n target: f32,\n t: &mut NormalizedF32Exclusive,\n) -> bool {\n // Solve F(t) = y where F(t) := [0](1-t)^2 + 2[1]t(1-t) + [2]t^2\n // We solve for t, using quadratic equation, hence we have to rearrange\n // our coefficients to look like At^2 + Bt + C\n let a = c0 - c1 - c1 + c2;\n let b = 2.0 * (c1 - c0);\n let c = c0 - target;\n\n let mut roots = path_geometry::new_t_values();\n let count = path_geometry::find_unit_quad_roots(a, b, c, &mut roots);\n if count != 0 {\n *t = roots[0];\n true\n } else {\n false\n }\n}\n\nfn too_big_for_reliable_float_math(r: &Rect) -> bool {\n // limit set as the largest float value for which we can still reliably compute things like\n // - chopping at XY extrema\n // - chopping at Y or X values for clipping\n //\n // Current value chosen just by experiment. Larger (and still succeeds) is always better.\n\n let limit = (1 << 22) as f32;\n r.left() < -limit || r.top() < -limit || r.right() > limit || r.bottom() > limit\n}\n\n/// Modifies pts[] in place so that it is clipped in Y to the clip rect.\nfn chop_cubic_in_y(clip: &Rect, pts: &mut [Point; 4]) {\n // are we partially above\n if pts[0].y < clip.top() {\n let mut tmp = [Point::zero(); 7];\n chop_mono_cubic_at_y(pts, clip.top(), &mut tmp);\n\n // For a large range in the points, we can do a poor job of chopping, such that the t\n // we computed resulted in the lower cubic still being partly above the clip.\n //\n // If just the first or first 2 Y values are above the fTop, we can just smash them\n // down. If the first 3 Ys are above fTop, we can't smash all 3, as that can really\n // distort the cubic. In this case, we take the first output (tmp[3..6] and treat it as\n // a guess, and re-chop against fTop. Then we fall through to checking if we need to\n // smash the first 1 or 2 Y values.\n if tmp[3].y < clip.top() && tmp[4].y < clip.top() && tmp[5].y < clip.top() {\n let tmp2: [Point; 4] = copy_4_points!(tmp, 3);\n chop_mono_cubic_at_y(&tmp2, clip.top(), &mut tmp);\n }\n\n // tmp[3, 4].y should all be to the below clip.fTop.\n // Since we can't trust the numerics of the chopper, we force those conditions now\n tmp[3].y = clip.top();\n tmp[4].y = tmp[4].y.max(clip.top());\n\n pts[0] = tmp[3];\n pts[1] = tmp[4];\n pts[2] = tmp[5];\n }\n\n // are we partially below\n if pts[3].y > clip.bottom() {\n let mut tmp = [Point::zero(); 7];\n chop_mono_cubic_at_y(pts, clip.bottom(), &mut tmp);\n tmp[3].y = clip.bottom();\n tmp[2].y = tmp[2].y.min(clip.bottom());\n\n pts[1] = tmp[1];\n pts[2] = tmp[2];\n pts[3] = tmp[3];\n }\n}\n\nfn chop_mono_cubic_at_x(src: &[Point; 4], x: f32, dst: &mut [Point; 7]) {\n if path_geometry::chop_mono_cubic_at_x(src, x, dst) {\n return;\n }\n\n let src_values = [src[0].x, src[1].x, src[2].x, src[3].x];\n path_geometry::chop_cubic_at2(src, mono_cubic_closest_t(&src_values, x), dst);\n}\n\nfn chop_mono_cubic_at_y(src: &[Point; 4], y: f32, dst: &mut [Point; 7]) {\n if path_geometry::chop_mono_cubic_at_y(src, y, dst) {\n return;\n }\n\n let src_values = [src[0].y, src[1].y, src[2].y, src[3].y];\n path_geometry::chop_cubic_at2(src, mono_cubic_closest_t(&src_values, y), dst);\n}\n\nfn mono_cubic_closest_t(src: &[f32; 4], mut x: f32) -> NormalizedF32Exclusive {\n let mut t = 0.5;\n let mut last_t;\n let mut best_t = t;\n let mut step = 0.25;\n let d = src[0];\n let a = src[3] + 3.0 * (src[1] - src[2]) - d;\n let b = 3.0 * (src[2] - src[1] - src[1] + d);\n let c = 3.0 * (src[1] - d);\n x -= d;\n let mut closest = SCALAR_MAX;\n loop {\n let loc = ((a * t + b) * t + c) * t;\n let dist = (loc - x).abs();\n if closest > dist {\n closest = dist;\n best_t = t;\n }\n\n last_t = t;\n t += if loc < x { step } else { -step };\n step *= 0.5;\n\n if !(closest > 0.25 && last_t != t) {\n break;\n }\n }\n\n NormalizedF32Exclusive::new(best_t).unwrap()\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/fixed_point.rs", "sha256": "9bf2c97418e0640fbcadcdb34ca3bf2c4c6ca9aee746d1b4acbf4d4d8c9d712d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// Skia uses fixed points pretty chaotically, therefore we cannot use\n// strongly typed wrappers. Which is unfortunate.\n\nuse tiny_skia_path::SaturateCast;\n\nuse crate::math::{bound, left_shift, left_shift64};\n\n/// A 26.6 fixed point.\npub type FDot6 = i32;\n\n/// A 24.8 fixed point.\npub type FDot8 = i32;\n\n/// A 16.16 fixed point.\npub type FDot16 = i32;\n\npub mod fdot6 {\n use super::*;\n use core::convert::TryFrom;\n\n pub const ONE: FDot6 = 64;\n\n pub fn from_i32(n: i32) -> FDot6 {\n debug_assert!(n as i16 as i32 == n);\n n << 6\n }\n\n pub fn from_f32(n: f32) -> FDot6 {\n (n * 64.0) as i32\n }\n\n pub fn floor(n: FDot6) -> FDot6 {\n n >> 6\n }\n\n pub fn ceil(n: FDot6) -> FDot6 {\n (n + 63) >> 6\n }\n\n pub fn round(n: FDot6) -> FDot6 {\n (n + 32) >> 6\n }\n\n pub fn to_fdot16(n: FDot6) -> FDot16 {\n debug_assert!((left_shift(n, 10) >> 10) == n);\n left_shift(n, 10)\n }\n\n pub fn div(a: FDot6, b: FDot6) -> FDot16 {\n debug_assert_ne!(b, 0);\n\n if i16::try_from(a).is_ok() {\n left_shift(a, 16) / b\n } else {\n fdot16::div(a, b)\n }\n }\n\n pub fn can_convert_to_fdot16(n: FDot6) -> bool {\n let max_dot6 = core::i32::MAX >> (16 - 6);\n n.abs() <= max_dot6\n }\n\n pub fn small_scale(value: u8, dot6: FDot6) -> u8 {\n debug_assert!(dot6 as u32 <= 64);\n ((value as i32 * dot6) >> 6) as u8\n }\n}\n\npub mod fdot8 {\n use super::*;\n\n // Extracted from SkScan_Antihair.cpp\n\n pub fn from_fdot16(x: FDot16) -> FDot8 {\n (x + 0x80) >> 8\n }\n}\n\npub mod fdot16 {\n use super::*;\n\n pub const HALF: FDot16 = (1 << 16) / 2;\n pub const ONE: FDot16 = 1 << 16;\n\n // `from_f32` seems to lack a rounding step. For all fixed-point\n // values, this version is as accurate as possible for (fixed -> float -> fixed). Rounding reduces\n // accuracy if the intermediate floats are in the range that only holds integers (adding 0.5 to an\n // odd integer then snaps to nearest even). Using double for the rounding math gives maximum\n // accuracy for (float -> fixed -> float), but that's usually overkill.\n pub fn from_f32(x: f32) -> FDot16 {\n i32::saturate_from(x * ONE as f32)\n }\n\n pub fn floor_to_i32(x: FDot16) -> i32 {\n x >> 16\n }\n\n pub fn ceil_to_i32(x: FDot16) -> i32 {\n (x + ONE - 1) >> 16\n }\n\n pub fn round_to_i32(x: FDot16) -> i32 {\n (x + HALF) >> 16\n }\n\n // The divide may exceed 32 bits. Clamp to a signed 32 bit result.\n pub fn mul(a: FDot16, b: FDot16) -> FDot16 {\n ((i64::from(a) * i64::from(b)) >> 16) as FDot16\n }\n\n // The divide may exceed 32 bits. Clamp to a signed 32 bit result.\n pub fn div(numer: FDot6, denom: FDot6) -> FDot16 {\n let v = left_shift64(numer as i64, 16) / denom as i64;\n let n = bound(i32::MIN as i64, v, i32::MAX as i64);\n n as i32\n }\n\n pub fn fast_div(a: FDot6, b: FDot6) -> FDot16 {\n debug_assert!((left_shift(a, 16) >> 16) == a);\n debug_assert!(b != 0);\n left_shift(a, 16) / b\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/geom.rs", "sha256": "5bdfe8f1633e392f98c78a503f12bad13bb01afdfc4dcd116c13fecedc916f37", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse core::convert::TryFrom;\n\nuse tiny_skia_path::{IntRect, IntSize, Rect};\n\nuse crate::LengthU32;\n\n/// A screen `IntRect`.\n///\n/// # Guarantees\n///\n/// - X and Y are in 0..=i32::MAX range.\n/// - Width and height are in 1..=i32::MAX range.\n/// - x+width and y+height does not overflow.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, PartialEq, Debug)]\npub struct ScreenIntRect {\n x: u32,\n y: u32,\n width: LengthU32,\n height: LengthU32,\n}\n\nimpl ScreenIntRect {\n /// Creates a new `ScreenIntRect`.\n pub fn from_xywh(x: u32, y: u32, width: u32, height: u32) -> Option {\n i32::try_from(x).ok()?;\n i32::try_from(y).ok()?;\n i32::try_from(width).ok()?;\n i32::try_from(height).ok()?;\n\n x.checked_add(width)?;\n y.checked_add(height)?;\n\n let width = LengthU32::new(width)?;\n let height = LengthU32::new(height)?;\n\n Some(ScreenIntRect {\n x,\n y,\n width,\n height,\n })\n }\n\n /// Creates a new `ScreenIntRect`.\n pub const fn from_xywh_safe(x: u32, y: u32, width: LengthU32, height: LengthU32) -> Self {\n ScreenIntRect {\n x,\n y,\n width,\n height,\n }\n }\n\n /// Returns rect's X position.\n pub fn x(&self) -> u32 {\n self.x\n }\n\n /// Returns rect's Y position.\n pub fn y(&self) -> u32 {\n self.y\n }\n\n /// Returns rect's width.\n pub fn width(&self) -> u32 {\n self.width.get()\n }\n\n /// Returns rect's height.\n pub fn height(&self) -> u32 {\n self.height.get()\n }\n\n /// Returns rect's width.\n pub fn width_safe(&self) -> LengthU32 {\n self.width\n }\n\n /// Returns rect's left edge.\n pub fn left(&self) -> u32 {\n self.x\n }\n\n /// Returns rect's top edge.\n pub fn top(&self) -> u32 {\n self.y\n }\n\n /// Returns rect's right edge.\n ///\n /// The right edge is at least 1.\n pub fn right(&self) -> u32 {\n // No overflow is guaranteed by constructors.\n self.x + self.width.get()\n }\n\n /// Returns rect's bottom edge.\n ///\n /// The bottom edge is at least 1.\n pub fn bottom(&self) -> u32 {\n // No overflow is guaranteed by constructors.\n self.y + self.height.get()\n }\n\n /// Returns rect's size.\n pub fn size(&self) -> IntSize {\n IntSize::from_wh(self.width(), self.height()).unwrap()\n }\n\n /// Checks that the rect is completely includes `other` Rect.\n pub fn contains(&self, other: &Self) -> bool {\n self.x <= other.x\n && self.y <= other.y\n && self.right() >= other.right()\n && self.bottom() >= other.bottom()\n }\n\n /// Converts into a `IntRect`.\n pub fn to_int_rect(&self) -> IntRect {\n // Everything is already checked by constructors.\n IntRect::from_xywh(\n self.x as i32,\n self.y as i32,\n self.width.get(),\n self.height.get(),\n )\n .unwrap()\n }\n\n /// Converts into a `Rect`.\n pub fn to_rect(&self) -> Rect {\n // Can't fail, because `ScreenIntRect` is always valid.\n // And u32 always fits into f32.\n Rect::from_ltrb(\n self.x as f32,\n self.y as f32,\n self.x as f32 + self.width.get() as f32,\n self.y as f32 + self.height.get() as f32,\n )\n .unwrap()\n }\n}\n\n#[cfg(test)]\nmod screen_int_rect_tests {\n use super::*;\n\n #[test]\n fn tests() {\n assert_eq!(ScreenIntRect::from_xywh(0, 0, 0, 0), None);\n assert_eq!(ScreenIntRect::from_xywh(0, 0, 1, 0), None);\n assert_eq!(ScreenIntRect::from_xywh(0, 0, 0, 1), None);\n\n assert_eq!(\n ScreenIntRect::from_xywh(0, 0, core::u32::MAX, core::u32::MAX),\n None\n );\n assert_eq!(ScreenIntRect::from_xywh(0, 0, 1, core::u32::MAX), None);\n assert_eq!(ScreenIntRect::from_xywh(0, 0, core::u32::MAX, 1), None);\n\n assert_eq!(ScreenIntRect::from_xywh(core::u32::MAX, 0, 1, 1), None);\n assert_eq!(ScreenIntRect::from_xywh(0, core::u32::MAX, 1, 1), None);\n\n assert_eq!(\n ScreenIntRect::from_xywh(\n core::u32::MAX,\n core::u32::MAX,\n core::u32::MAX,\n core::u32::MAX\n ),\n None\n );\n\n let r = ScreenIntRect::from_xywh(1, 2, 3, 4).unwrap();\n assert_eq!(r.x(), 1);\n assert_eq!(r.y(), 2);\n assert_eq!(r.width(), 3);\n assert_eq!(r.height(), 4);\n assert_eq!(r.right(), 4);\n assert_eq!(r.bottom(), 6);\n }\n}\n\npub trait IntSizeExt {\n /// Converts the current size into a `IntRect` at a provided position.\n fn to_screen_int_rect(&self, x: u32, y: u32) -> ScreenIntRect;\n}\n\nimpl IntSizeExt for IntSize {\n fn to_screen_int_rect(&self, x: u32, y: u32) -> ScreenIntRect {\n ScreenIntRect::from_xywh(x, y, self.width(), self.height()).unwrap()\n }\n}\n\npub trait IntRectExt {\n /// Converts into `ScreenIntRect`.\n ///\n /// # Checks\n ///\n /// - x >= 0\n /// - y >= 0\n fn to_screen_int_rect(&self) -> Option;\n}\n\nimpl IntRectExt for IntRect {\n fn to_screen_int_rect(&self) -> Option {\n let x = u32::try_from(self.x()).ok()?;\n let y = u32::try_from(self.y()).ok()?;\n ScreenIntRect::from_xywh(x, y, self.width(), self.height())\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/line_clipper.rs", "sha256": "b17d6ce38666a08f24d365551d10632c17edee5838943a31506c77417762e798", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2011 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse crate::{Point, Rect};\n\nuse tiny_skia_path::Scalar;\n\npub const MAX_POINTS: usize = 4;\n\n/// Clip the line pts[0]...pts[1] against clip, ignoring segments that\n/// lie completely above or below the clip. For portions to the left or\n/// right, turn those into vertical line segments that are aligned to the\n/// edge of the clip.\n///\n/// Return the number of line segments that result, and store the end-points\n/// of those segments sequentially in lines as follows:\n///\n/// 1st segment: lines[0]..lines[1]\n/// 2nd segment: lines[1]..lines[2]\n/// 3rd segment: lines[2]..lines[3]\npub fn clip<'a>(\n src: &[Point; 2],\n clip: &Rect,\n can_cull_to_the_right: bool,\n points: &'a mut [Point; MAX_POINTS],\n) -> &'a [Point] {\n let (mut index0, mut index1) = if src[0].y < src[1].y { (0, 1) } else { (1, 0) };\n\n // Check if we're completely clipped out in Y (above or below)\n\n if src[index1].y <= clip.top() {\n // we're above the clip\n return &[];\n }\n\n if src[index0].y >= clip.bottom() {\n // we're below the clip\n return &[];\n }\n\n // Chop in Y to produce a single segment, stored in tmp[0..1]\n\n let mut tmp = *src;\n\n // now compute intersections\n if src[index0].y < clip.top() {\n tmp[index0] = Point::from_xy(sect_with_horizontal(src, clip.top()), clip.top());\n debug_assert!(is_between_unsorted(tmp[index0].x, src[0].x, src[1].x));\n }\n\n if tmp[index1].y > clip.bottom() {\n tmp[index1] = Point::from_xy(sect_with_horizontal(src, clip.bottom()), clip.bottom());\n debug_assert!(is_between_unsorted(tmp[index1].x, src[0].x, src[1].x));\n }\n\n // Chop it into 1..3 segments that are wholly within the clip in X.\n\n // temp storage for up to 3 segments\n let mut result_storage = [Point::zero(); MAX_POINTS];\n let mut line_count = 1;\n let mut reverse;\n\n if src[0].x < src[1].x {\n index0 = 0;\n index1 = 1;\n reverse = false;\n } else {\n index0 = 1;\n index1 = 0;\n reverse = true;\n }\n\n let result: &[Point] = if tmp[index1].x <= clip.left() {\n // wholly to the left\n tmp[0].x = clip.left();\n tmp[1].x = clip.left();\n reverse = false;\n &tmp\n } else if tmp[index0].x >= clip.right() {\n // wholly to the right\n if can_cull_to_the_right {\n return &[];\n }\n\n tmp[0].x = clip.right();\n tmp[1].x = clip.right();\n reverse = false;\n &tmp\n } else {\n let mut offset = 0;\n\n if tmp[index0].x < clip.left() {\n result_storage[offset] = Point::from_xy(clip.left(), tmp[index0].y);\n offset += 1;\n result_storage[offset] =\n Point::from_xy(clip.left(), sect_clamp_with_vertical(&tmp, clip.left()));\n debug_assert!(is_between_unsorted(\n result_storage[offset].y,\n tmp[0].y,\n tmp[1].y\n ));\n } else {\n result_storage[offset] = tmp[index0];\n }\n offset += 1;\n\n if tmp[index1].x > clip.right() {\n result_storage[offset] =\n Point::from_xy(clip.right(), sect_clamp_with_vertical(&tmp, clip.right()));\n debug_assert!(is_between_unsorted(\n result_storage[offset].y,\n tmp[0].y,\n tmp[1].y\n ));\n offset += 1;\n result_storage[offset] = Point::from_xy(clip.right(), tmp[index1].y);\n } else {\n result_storage[offset] = tmp[index1];\n }\n\n line_count = offset;\n &result_storage\n };\n\n // Now copy the results into the caller's lines[] parameter\n if reverse {\n // copy the pts in reverse order to maintain winding order\n for i in 0..=line_count {\n points[line_count - i] = result[i];\n }\n } else {\n let len = line_count + 1;\n points[0..len].copy_from_slice(&result[0..len]);\n }\n\n &points[0..line_count + 1]\n}\n\n/// Returns X coordinate of intersection with horizontal line at Y.\nfn sect_with_horizontal(src: &[Point; 2], y: f32) -> f32 {\n let dy = src[1].y - src[0].y;\n if dy.is_nearly_zero() {\n src[0].x.ave(src[1].x)\n } else {\n // need the extra precision so we don't compute a value that exceeds\n // our original limits\n let x0 = f64::from(src[0].x);\n let y0 = f64::from(src[0].y);\n let x1 = f64::from(src[1].x);\n let y1 = f64::from(src[1].y);\n let result = x0 + (f64::from(y) - y0) * (x1 - x0) / (y1 - y0);\n\n // The computed X value might still exceed [X0..X1] due to quantum flux\n // when the doubles were added and subtracted, so we have to pin the\n // answer :(\n pin_unsorted_f64(result, x0, x1) as f32\n }\n}\n\n/// Returns value between the two limits, where the limits are either ascending or descending.\nfn is_between_unsorted(value: f32, limit0: f32, limit1: f32) -> bool {\n if limit0 < limit1 {\n limit0 <= value && value <= limit1\n } else {\n limit1 <= value && value <= limit0\n }\n}\n\nfn sect_clamp_with_vertical(src: &[Point; 2], x: f32) -> f32 {\n let y = sect_with_vertical(src, x);\n // Our caller expects y to be between src[0].y and src[1].y (unsorted), but due to the\n // numerics of floats/doubles, we might have computed a value slightly outside of that,\n // so we have to manually clamp afterwards.\n // See skbug.com/7491\n pin_unsorted_f32(y, src[0].y, src[1].y)\n}\n\n/// Returns Y coordinate of intersection with vertical line at X.\nfn sect_with_vertical(src: &[Point; 2], x: f32) -> f32 {\n let dx = src[1].x - src[0].x;\n if dx.is_nearly_zero() {\n src[0].y.ave(src[1].y)\n } else {\n // need the extra precision so we don't compute a value that exceeds\n // our original limits\n let x0 = f64::from(src[0].x);\n let y0 = f64::from(src[0].y);\n let x1 = f64::from(src[1].x);\n let y1 = f64::from(src[1].y);\n let result = y0 + (f64::from(x) - x0) * (y1 - y0) / (x1 - x0);\n result as f32\n }\n}\n\nfn pin_unsorted_f32(value: f32, mut limit0: f32, mut limit1: f32) -> f32 {\n if limit1 < limit0 {\n core::mem::swap(&mut limit0, &mut limit1);\n }\n // now the limits are sorted\n debug_assert!(limit0 <= limit1);\n\n if value < limit0 {\n limit0\n } else if value > limit1 {\n limit1\n } else {\n value\n }\n}\n\nfn pin_unsorted_f64(value: f64, mut limit0: f64, mut limit1: f64) -> f64 {\n if limit1 < limit0 {\n core::mem::swap(&mut limit0, &mut limit1);\n }\n // now the limits are sorted\n debug_assert!(limit0 <= limit1);\n\n if value < limit0 {\n limit0\n } else if value > limit1 {\n limit1\n } else {\n value\n }\n}\n\n/// Intersect the line segment against the rect. If there is a non-empty\n/// resulting segment, return true and set dst[] to that segment. If not,\n/// return false and ignore dst[].\n///\n/// `clip` is specialized for scan-conversion, as it adds vertical\n/// segments on the sides to show where the line extended beyond the\n/// left or right sides. `intersect` does not.\npub fn intersect(src: &[Point; 2], clip: &Rect, dst: &mut [Point; 2]) -> bool {\n let bounds = Rect::from_ltrb(\n src[0].x.min(src[1].x),\n src[0].y.min(src[1].y),\n src[0].x.max(src[1].x),\n src[0].y.max(src[1].y),\n );\n\n if let Some(bounds) = bounds {\n if contains_no_empty_check(clip, &bounds) {\n dst.copy_from_slice(src);\n return true;\n }\n\n // check for no overlap, and only permit coincident edges if the line\n // and the edge are colinear\n if nested_lt(bounds.right(), clip.left(), bounds.width())\n || nested_lt(clip.right(), bounds.left(), bounds.width())\n || nested_lt(bounds.bottom(), clip.top(), bounds.height())\n || nested_lt(clip.bottom(), bounds.top(), bounds.height())\n {\n return false;\n }\n }\n\n let (index0, index1) = if src[0].y < src[1].y { (0, 1) } else { (1, 0) };\n\n let mut tmp = src.clone();\n\n // now compute Y intersections\n if tmp[index0].y < clip.top() {\n tmp[index0] = Point::from_xy(sect_with_horizontal(src, clip.top()), clip.top());\n }\n\n if tmp[index1].y > clip.bottom() {\n tmp[index1] = Point::from_xy(sect_with_horizontal(src, clip.bottom()), clip.bottom());\n }\n\n let (index0, index1) = if tmp[0].x < tmp[1].x { (0, 1) } else { (1, 0) };\n\n // check for quick-reject in X again, now that we may have been chopped\n if tmp[index1].x <= clip.left() || tmp[index0].x >= clip.right() {\n // usually we will return false, but we don't if the line is vertical and coincident\n // with the clip.\n if tmp[0].x != tmp[1].x || tmp[0].x < clip.left() || tmp[0].x > clip.right() {\n return false;\n }\n }\n\n if tmp[index0].x < clip.left() {\n tmp[index0] = Point::from_xy(clip.left(), sect_with_vertical(src, clip.left()));\n }\n\n if tmp[index1].x > clip.right() {\n tmp[index1] = Point::from_xy(clip.right(), sect_with_vertical(src, clip.right()));\n }\n\n dst.copy_from_slice(&tmp);\n true\n}\n\nfn nested_lt(a: f32, b: f32, dim: f32) -> bool {\n a <= b && (a < b || dim > 0.0)\n}\n\n// returns true if outer contains inner, even if inner is empty.\nfn contains_no_empty_check(outer: &Rect, inner: &Rect) -> bool {\n outer.left() <= inner.left()\n && outer.top() <= inner.top()\n && outer.right() >= inner.right()\n && outer.bottom() >= inner.bottom()\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/mask.rs", "sha256": "c0acccd67eb17082128647d81871aece69d80a1533835ac36a0e3d05cd5a56ce", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\nuse alloc::vec;\nuse alloc::vec::Vec;\n\nuse tiny_skia_path::{IntRect, IntSize, Path, Scalar, Transform};\n\nuse crate::geom::IntSizeExt;\nuse crate::painter::DrawTiler;\nuse crate::pipeline::RasterPipelineBlitter;\nuse crate::pixmap::SubPixmapMut;\nuse crate::scan;\nuse crate::{FillRule, PixmapRef};\n\n/// A mask type.\n#[derive(Clone, Copy, PartialEq, Debug)]\npub enum MaskType {\n /// Transfers only the Alpha channel from `Pixmap` to `Mask`.\n Alpha,\n /// Transfers RGB channels as luminance from `Pixmap` to `Mask`.\n ///\n /// Formula: `Y = 0.2126 * R + 0.7152 * G + 0.0722 * B`\n Luminance,\n}\n\n/// A mask.\n///\n/// During drawing over `Pixmap`, mask's black (0) \"pixels\" would block rendering\n/// and white (255) will allow it.\n/// Anything in between is used for gradual masking and anti-aliasing.\n///\n/// Unlike Skia, we're using just a simple 8bit alpha mask.\n/// It's way slower, but easier to implement.\n#[derive(Clone, PartialEq)]\npub struct Mask {\n data: Vec,\n size: IntSize,\n}\n\nimpl Mask {\n /// Creates a new mask by taking ownership over a mask buffer.\n ///\n /// The size needs to match the data provided.\n pub fn new(width: u32, height: u32) -> Option {\n let size = IntSize::from_wh(width, height)?;\n Some(Mask {\n data: vec![0; width as usize * height as usize],\n size,\n })\n }\n\n /// Creates a new mask from a `PixmapRef`.\n pub fn from_pixmap(pixmap: PixmapRef, mask_type: MaskType) -> Self {\n let data_len = pixmap.width() as usize * pixmap.height() as usize;\n let mut mask = Mask {\n data: vec![0; data_len],\n size: pixmap.size(),\n };\n\n // TODO: optimize\n match mask_type {\n MaskType::Alpha => {\n for (p, a) in pixmap.pixels().iter().zip(mask.data.as_mut_slice()) {\n *a = p.alpha();\n }\n }\n MaskType::Luminance => {\n for (p, ma) in pixmap.pixels().iter().zip(mask.data.as_mut_slice()) {\n // Normalize.\n let mut r = f32::from(p.red()) / 255.0;\n let mut g = f32::from(p.green()) / 255.0;\n let mut b = f32::from(p.blue()) / 255.0;\n let a = f32::from(p.alpha()) / 255.0;\n\n // Demultiply.\n if p.alpha() != 0 {\n r /= a;\n g /= a;\n b /= a;\n }\n\n let luma = r * 0.2126 + g * 0.7152 + b * 0.0722;\n *ma = ((luma * a) * 255.0).clamp(0.0, 255.0).ceil() as u8;\n }\n }\n }\n\n mask\n }\n\n /// Creates a new mask by taking ownership over a mask buffer.\n ///\n /// The size needs to match the data provided.\n pub fn from_vec(data: Vec, size: IntSize) -> Option {\n let data_len = size.width() as usize * size.height() as usize;\n if data.len() != data_len {\n return None;\n }\n\n Some(Mask { data, size })\n }\n\n /// Returns mask's width.\n #[inline]\n pub fn width(&self) -> u32 {\n self.size.width()\n }\n\n /// Returns mask's height.\n #[inline]\n pub fn height(&self) -> u32 {\n self.size.height()\n }\n\n /// Returns mask's size.\n #[allow(dead_code)]\n pub(crate) fn size(&self) -> IntSize {\n self.size\n }\n\n /// Returns the internal data.\n pub fn data(&self) -> &[u8] {\n self.data.as_slice()\n }\n\n /// Returns the mutable internal data.\n pub fn data_mut(&mut self) -> &mut [u8] {\n self.data.as_mut_slice()\n }\n\n pub(crate) fn as_submask<'a>(&'a self) -> SubMaskRef<'a> {\n SubMaskRef {\n size: self.size,\n real_width: self.size.width(),\n data: &self.data,\n }\n }\n\n pub(crate) fn submask<'a>(&'a self, rect: IntRect) -> Option> {\n let rect = self.size.to_int_rect(0, 0).intersect(&rect)?;\n let row_bytes = self.width() as usize;\n let offset = rect.top() as usize * row_bytes + rect.left() as usize;\n\n Some(SubMaskRef {\n size: rect.size(),\n real_width: self.size.width(),\n data: &self.data[offset..],\n })\n }\n\n pub(crate) fn as_subpixmap<'a>(&'a mut self) -> SubPixmapMut<'a> {\n SubPixmapMut {\n size: self.size,\n real_width: self.size.width() as usize,\n data: &mut self.data,\n }\n }\n\n pub(crate) fn subpixmap<'a>(&'a mut self, rect: IntRect) -> Option> {\n let rect = self.size.to_int_rect(0, 0).intersect(&rect)?;\n let row_bytes = self.width() as usize;\n let offset = rect.top() as usize * row_bytes + rect.left() as usize;\n\n Some(SubPixmapMut {\n size: rect.size(),\n real_width: self.size.width() as usize,\n data: &mut self.data[offset..],\n })\n }\n\n /// Loads a PNG file into a `Mask`.\n ///\n /// Only grayscale images are supported.\n #[cfg(feature = \"png-format\")]\n pub fn decode_png(data: &[u8]) -> Result {\n fn make_custom_png_error(msg: &str) -> png::DecodingError {\n std::io::Error::new(std::io::ErrorKind::Other, msg).into()\n }\n\n let mut decoder = png::Decoder::new(data);\n decoder.set_transformations(png::Transformations::normalize_to_color8());\n let mut reader = decoder.read_info()?;\n let mut img_data = vec![0; reader.output_buffer_size()];\n let info = reader.next_frame(&mut img_data)?;\n\n if info.bit_depth != png::BitDepth::Eight {\n return Err(make_custom_png_error(\"unsupported bit depth\"));\n }\n\n if info.color_type != png::ColorType::Grayscale {\n return Err(make_custom_png_error(\"only grayscale masks are supported\"));\n }\n\n let size = IntSize::from_wh(info.width, info.height)\n .ok_or_else(|| make_custom_png_error(\"invalid image size\"))?;\n\n Mask::from_vec(img_data, size)\n .ok_or_else(|| make_custom_png_error(\"failed to create a mask\"))\n }\n\n /// Loads a PNG file into a `Mask`.\n ///\n /// Only grayscale images are supported.\n #[cfg(feature = \"png-format\")]\n pub fn load_png>(path: P) -> Result {\n // `png::Decoder` is generic over input, which means that it will instance\n // two copies: one for `&[]` and one for `File`. Which will simply bloat the code.\n // Therefore we're using only one type for input.\n let data = std::fs::read(path)?;\n Self::decode_png(&data)\n }\n\n /// Encodes mask into a PNG data.\n #[cfg(feature = \"png-format\")]\n pub fn encode_png(&self) -> Result, png::EncodingError> {\n let mut data = Vec::new();\n {\n let mut encoder = png::Encoder::new(&mut data, self.width(), self.height());\n encoder.set_color(png::ColorType::Grayscale);\n encoder.set_depth(png::BitDepth::Eight);\n let mut writer = encoder.write_header()?;\n writer.write_image_data(&self.data)?;\n }\n\n Ok(data)\n }\n\n /// Saves mask as a PNG file.\n #[cfg(feature = \"png-format\")]\n pub fn save_png>(&self, path: P) -> Result<(), png::EncodingError> {\n let data = self.encode_png()?;\n std::fs::write(path, data)?;\n Ok(())\n }\n\n // Almost a direct copy of PixmapMut::fill_path\n /// Draws a filled path onto the mask.\n ///\n /// In terms of RGB (no alpha) image, draws a white path on top of black mask.\n ///\n /// Doesn't reset the existing mask content and draws the path on top of existing data.\n ///\n /// If the above behavior is undesired, [`clear()`] should be called first.\n ///\n /// This method is intended to be used for simple cases. For more complex masks\n /// prefer [`from_pixmap()`].\n pub fn fill_path(\n &mut self,\n path: &Path,\n fill_rule: FillRule,\n anti_alias: bool,\n transform: Transform,\n ) {\n if transform.is_identity() {\n // This is sort of similar to SkDraw::drawPath\n\n // Skip empty paths and horizontal/vertical lines.\n let path_bounds = path.bounds();\n if path_bounds.width().is_nearly_zero() || path_bounds.height().is_nearly_zero() {\n log::warn!(\"empty paths and horizontal/vertical lines cannot be filled\");\n return;\n }\n\n if crate::painter::is_too_big_for_math(path) {\n log::warn!(\"path coordinates are too big\");\n return;\n }\n\n // TODO: ignore paths outside the pixmap\n\n if let Some(tiler) = DrawTiler::new(self.width(), self.height()) {\n let mut path = path.clone(); // TODO: avoid cloning\n\n for tile in tiler {\n let ts = Transform::from_translate(-(tile.x() as f32), -(tile.y() as f32));\n path = match path.transform(ts) {\n Some(v) => v,\n None => {\n log::warn!(\"path transformation failed\");\n return;\n }\n };\n\n let clip_rect = tile.size().to_screen_int_rect(0, 0);\n let mut subpix = match self.subpixmap(tile.to_int_rect()) {\n Some(v) => v,\n None => continue, // technically unreachable\n };\n\n let mut blitter = match RasterPipelineBlitter::new_mask(&mut subpix) {\n Some(v) => v,\n None => continue, // nothing to do, all good\n };\n\n // We're ignoring \"errors\" here, because `fill_path` will return `None`\n // when rendering a tile that doesn't have a path on it.\n // Which is not an error in this case.\n if anti_alias {\n scan::path_aa::fill_path(&path, fill_rule, &clip_rect, &mut blitter);\n } else {\n scan::path::fill_path(&path, fill_rule, &clip_rect, &mut blitter);\n }\n\n let ts = Transform::from_translate(tile.x() as f32, tile.y() as f32);\n path = match path.transform(ts) {\n Some(v) => v,\n None => return, // technically unreachable\n };\n }\n } else {\n let clip_rect = self.size().to_screen_int_rect(0, 0);\n let mut subpix = self.as_subpixmap();\n let mut blitter = match RasterPipelineBlitter::new_mask(&mut subpix) {\n Some(v) => v,\n None => return, // nothing to do, all good\n };\n\n if anti_alias {\n scan::path_aa::fill_path(path, fill_rule, &clip_rect, &mut blitter);\n } else {\n scan::path::fill_path(path, fill_rule, &clip_rect, &mut blitter);\n }\n }\n } else {\n let path = match path.clone().transform(transform) {\n Some(v) => v,\n None => {\n log::warn!(\"path transformation failed\");\n return;\n }\n };\n\n self.fill_path(&path, fill_rule, anti_alias, Transform::identity());\n }\n }\n\n /// Intersects the provided path with the current clipping path.\n ///\n /// A temporary mask with the same size as the current one will be created.\n pub fn intersect_path(\n &mut self,\n path: &Path,\n fill_rule: FillRule,\n anti_alias: bool,\n transform: Transform,\n ) {\n let mut submask = Mask::new(self.width(), self.height()).unwrap();\n submask.fill_path(path, fill_rule, anti_alias, transform);\n\n for (a, b) in self.data.iter_mut().zip(submask.data.iter()) {\n *a = crate::color::premultiply_u8(*a, *b);\n }\n }\n\n /// Inverts the mask.\n pub fn invert(&mut self) {\n self.data.iter_mut().for_each(|a| *a = 255 - *a);\n }\n\n /// Clears the mask.\n ///\n /// Zero-fills the internal data buffer.\n pub fn clear(&mut self) {\n self.data.fill(0);\n }\n}\n\nimpl core::fmt::Debug for Mask {\n fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n f.debug_struct(\"Mask\")\n .field(\"data\", &\"...\")\n .field(\"width\", &self.size.width())\n .field(\"height\", &self.size.height())\n .finish()\n }\n}\n\n#[derive(Clone, Copy)]\npub struct SubMaskRef<'a> {\n pub data: &'a [u8],\n pub size: IntSize,\n pub real_width: u32,\n}\n\nimpl<'a> SubMaskRef<'a> {\n pub(crate) fn mask_ctx(&self) -> crate::pipeline::MaskCtx<'a> {\n crate::pipeline::MaskCtx {\n data: &self.data,\n real_width: self.real_width,\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/math.rs", "sha256": "59ff8f3201e0ff5c26b8057dc7105e59be7bff7a5120bb4d7b3aa20f000f5342", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse crate::LengthU32;\n\n// Perfectly safe.\npub const LENGTH_U32_ONE: LengthU32 = unsafe { LengthU32::new_unchecked(1) };\n\npub fn left_shift(value: i32, shift: i32) -> i32 {\n ((value as u32) << shift) as i32\n}\n\npub fn left_shift64(value: i64, shift: i32) -> i64 {\n ((value as u64) << shift) as i64\n}\n\npub fn bound(min: T, value: T, max: T) -> T {\n max.min(value).max(min)\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/painter.rs", "sha256": "76d57ca174e7619a4ac8b7170b3d640a4c85d92eafe3fe12879dffcbe34599eb", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse crate::*;\n\nuse tiny_skia_path::{PathStroker, Scalar, SCALAR_MAX};\n\nuse crate::geom::ScreenIntRect;\nuse crate::mask::SubMaskRef;\nuse crate::pipeline::{RasterPipelineBlitter, RasterPipelineBuilder};\nuse crate::pixmap::SubPixmapMut;\nuse crate::scan;\n\nuse crate::geom::IntSizeExt;\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\n/// A path filling rule.\n#[derive(Copy, Clone, PartialEq, Debug)]\npub enum FillRule {\n /// Specifies that \"inside\" is computed by a non-zero sum of signed edge crossings.\n Winding,\n /// Specifies that \"inside\" is computed by an odd number of edge crossings.\n EvenOdd,\n}\n\nimpl Default for FillRule {\n fn default() -> Self {\n FillRule::Winding\n }\n}\n\n/// Controls how a shape should be painted.\n#[derive(Clone, PartialEq, Debug)]\npub struct Paint<'a> {\n /// A paint shader.\n ///\n /// Default: black color\n pub shader: Shader<'a>,\n\n /// Paint blending mode.\n ///\n /// Default: SourceOver\n pub blend_mode: BlendMode,\n\n /// Enables anti-aliased painting.\n ///\n /// Default: true\n pub anti_alias: bool,\n\n /// Forces the high quality/precision rendering pipeline.\n ///\n /// `tiny-skia`, just like Skia, has two rendering pipelines:\n /// one uses `f32` and another one uses `u16`. `u16` one is usually way faster,\n /// but less precise. Which can lead to slight differences.\n ///\n /// By default, `tiny-skia` will choose the pipeline automatically,\n /// depending on a blending mode and other parameters.\n /// But you can force the high quality one using this flag.\n ///\n /// This feature is especially useful during testing.\n ///\n /// Unlike high quality pipeline, the low quality one doesn't support all\n /// rendering stages, therefore we cannot force it like hq one.\n ///\n /// Default: false\n pub force_hq_pipeline: bool,\n}\n\nimpl Default for Paint<'_> {\n fn default() -> Self {\n Paint {\n shader: Shader::SolidColor(Color::BLACK),\n blend_mode: BlendMode::default(),\n anti_alias: true,\n force_hq_pipeline: false,\n }\n }\n}\n\nimpl<'a> Paint<'a> {\n /// Sets a paint source to a solid color.\n pub fn set_color(&mut self, color: Color) {\n self.shader = Shader::SolidColor(color);\n }\n\n /// Sets a paint source to a solid color.\n ///\n /// `self.shader = Shader::SolidColor(Color::from_rgba8(50, 127, 150, 200));` shorthand.\n pub fn set_color_rgba8(&mut self, r: u8, g: u8, b: u8, a: u8) {\n self.set_color(Color::from_rgba8(r, g, b, a))\n }\n\n /// Checks that the paint source is a solid color.\n pub fn is_solid_color(&self) -> bool {\n matches!(self.shader, Shader::SolidColor(_))\n }\n}\n\nimpl Pixmap {\n /// Draws a filled rectangle onto the pixmap.\n ///\n /// See [`PixmapMut::fill_rect`](struct.PixmapMut.html#method.fill_rect) for details.\n pub fn fill_rect(\n &mut self,\n rect: Rect,\n paint: &Paint,\n transform: Transform,\n mask: Option<&Mask>,\n ) {\n self.as_mut().fill_rect(rect, paint, transform, mask);\n }\n\n /// Draws a filled path onto the pixmap.\n ///\n /// See [`PixmapMut::fill_path`](struct.PixmapMut.html#method.fill_path) for details.\n pub fn fill_path(\n &mut self,\n path: &Path,\n paint: &Paint,\n fill_rule: FillRule,\n transform: Transform,\n mask: Option<&Mask>,\n ) {\n self.as_mut()\n .fill_path(path, paint, fill_rule, transform, mask);\n }\n\n /// Strokes a path.\n ///\n /// See [`PixmapMut::stroke_path`](struct.PixmapMut.html#method.stroke_path) for details.\n pub fn stroke_path(\n &mut self,\n path: &Path,\n paint: &Paint,\n stroke: &Stroke,\n transform: Transform,\n mask: Option<&Mask>,\n ) {\n self.as_mut()\n .stroke_path(path, paint, stroke, transform, mask);\n }\n\n /// Draws a `Pixmap` on top of the current `Pixmap`.\n ///\n /// See [`PixmapMut::draw_pixmap`](struct.PixmapMut.html#method.draw_pixmap) for details.\n pub fn draw_pixmap(\n &mut self,\n x: i32,\n y: i32,\n pixmap: PixmapRef,\n paint: &PixmapPaint,\n transform: Transform,\n mask: Option<&Mask>,\n ) {\n self.as_mut()\n .draw_pixmap(x, y, pixmap, paint, transform, mask);\n }\n\n /// Applies a masks.\n ///\n /// See [`PixmapMut::apply_mask`](struct.PixmapMut.html#method.apply_mask) for details.\n pub fn apply_mask(&mut self, mask: &Mask) {\n self.as_mut().apply_mask(mask);\n }\n}\n\nimpl PixmapMut<'_> {\n // TODO: accept NonZeroRect?\n /// Draws a filled rectangle onto the pixmap.\n ///\n /// This function is usually slower than filling a rectangular path,\n /// but it produces better results. Mainly it doesn't suffer from weird\n /// clipping of horizontal/vertical edges.\n ///\n /// Used mainly to render a pixmap onto a pixmap.\n ///\n /// Returns `None` when there is nothing to fill or in case of a numeric overflow.\n pub fn fill_rect(\n &mut self,\n rect: Rect,\n paint: &Paint,\n transform: Transform,\n mask: Option<&Mask>,\n ) {\n // TODO: we probably can use tiler for rect too\n if transform.is_identity() && !DrawTiler::required(self.width(), self.height()) {\n // TODO: ignore rects outside the pixmap\n\n let clip = self.size().to_screen_int_rect(0, 0);\n\n let mask = mask.map(|mask| mask.as_submask());\n let mut subpix = self.as_subpixmap();\n let mut blitter = match RasterPipelineBlitter::new(paint, mask, &mut subpix) {\n Some(v) => v,\n None => return, // nothing to do, all good\n };\n\n if paint.anti_alias {\n scan::fill_rect_aa(&rect, &clip, &mut blitter);\n } else {\n scan::fill_rect(&rect, &clip, &mut blitter);\n }\n } else {\n let path = PathBuilder::from_rect(rect);\n self.fill_path(&path, paint, FillRule::Winding, transform, mask);\n }\n }\n\n /// Draws a filled path onto the pixmap.\n pub fn fill_path(\n &mut self,\n path: &Path,\n paint: &Paint,\n fill_rule: FillRule,\n transform: Transform,\n mask: Option<&Mask>,\n ) {\n if transform.is_identity() {\n // This is sort of similar to SkDraw::drawPath\n\n // Skip empty paths and horizontal/vertical lines.\n let path_bounds = path.bounds();\n if path_bounds.width().is_nearly_zero() || path_bounds.height().is_nearly_zero() {\n log::warn!(\"empty paths and horizontal/vertical lines cannot be filled\");\n return;\n }\n\n if is_too_big_for_math(path) {\n log::warn!(\"path coordinates are too big\");\n return;\n }\n\n // TODO: ignore paths outside the pixmap\n\n if let Some(tiler) = DrawTiler::new(self.width(), self.height()) {\n let mut path = path.clone(); // TODO: avoid cloning\n let mut paint = paint.clone();\n\n for tile in tiler {\n let ts = Transform::from_translate(-(tile.x() as f32), -(tile.y() as f32));\n path = match path.transform(ts) {\n Some(v) => v,\n None => {\n log::warn!(\"path transformation failed\");\n return;\n }\n };\n paint.shader.transform(ts);\n\n let clip_rect = tile.size().to_screen_int_rect(0, 0);\n let mut subpix = match self.subpixmap(tile.to_int_rect()) {\n Some(v) => v,\n None => continue, // technically unreachable\n };\n\n let submask = mask.and_then(|mask| mask.submask(tile.to_int_rect()));\n let mut blitter = match RasterPipelineBlitter::new(&paint, submask, &mut subpix)\n {\n Some(v) => v,\n None => continue, // nothing to do, all good\n };\n\n // We're ignoring \"errors\" here, because `fill_path` will return `None`\n // when rendering a tile that doesn't have a path on it.\n // Which is not an error in this case.\n if paint.anti_alias {\n scan::path_aa::fill_path(&path, fill_rule, &clip_rect, &mut blitter);\n } else {\n scan::path::fill_path(&path, fill_rule, &clip_rect, &mut blitter);\n }\n\n let ts = Transform::from_translate(tile.x() as f32, tile.y() as f32);\n path = match path.transform(ts) {\n Some(v) => v,\n None => return, // technically unreachable\n };\n paint.shader.transform(ts);\n }\n } else {\n let clip_rect = self.size().to_screen_int_rect(0, 0);\n let submask = mask.map(|mask| mask.as_submask());\n let mut subpix = self.as_subpixmap();\n let mut blitter = match RasterPipelineBlitter::new(paint, submask, &mut subpix) {\n Some(v) => v,\n None => return, // nothing to do, all good\n };\n\n if paint.anti_alias {\n scan::path_aa::fill_path(path, fill_rule, &clip_rect, &mut blitter);\n } else {\n scan::path::fill_path(path, fill_rule, &clip_rect, &mut blitter);\n }\n }\n } else {\n let path = match path.clone().transform(transform) {\n Some(v) => v,\n None => {\n log::warn!(\"path transformation failed\");\n return;\n }\n };\n\n let mut paint = paint.clone();\n paint.shader.transform(transform);\n\n self.fill_path(&path, &paint, fill_rule, Transform::identity(), mask)\n }\n }\n\n /// Strokes a path.\n ///\n /// Stroking is implemented using two separate algorithms:\n ///\n /// 1. If a stroke width is wider than 1px (after applying the transformation),\n /// a path will be converted into a stroked path and then filled using `fill_path`.\n /// Which means that we have to allocate a separate `Path`, that can be 2-3x larger\n /// then the original path.\n /// 2. If a stroke width is thinner than 1px (after applying the transformation),\n /// we will use hairline stroking, which doesn't involve a separate path allocation.\n ///\n /// Also, if a `stroke` has a dash array, then path will be converted into\n /// a dashed path first and then stroked. Which means a yet another allocation.\n pub fn stroke_path(\n &mut self,\n path: &Path,\n paint: &Paint,\n stroke: &Stroke,\n transform: Transform,\n mask: Option<&Mask>,\n ) {\n if stroke.width < 0.0 {\n log::warn!(\"negative stroke width isn't allowed\");\n return;\n }\n\n let res_scale = PathStroker::compute_resolution_scale(&transform);\n\n let dash_path;\n let path = if let Some(ref dash) = stroke.dash {\n dash_path = match path.dash(dash, res_scale) {\n Some(v) => v,\n None => {\n log::warn!(\"path dashing failed\");\n return;\n }\n };\n &dash_path\n } else {\n path\n };\n\n if let Some(coverage) = treat_as_hairline(paint, stroke, transform) {\n let mut paint = paint.clone();\n if coverage == 1.0 {\n // No changes to the `paint`.\n } else if paint.blend_mode.should_pre_scale_coverage() {\n // This is the old technique, which we preserve for now so\n // we don't change previous results (testing)\n // the new way seems fine, its just (a tiny bit) different.\n let scale = (coverage * 256.0) as i32;\n let new_alpha = (255 * scale) >> 8;\n paint.shader.apply_opacity(new_alpha as f32 / 255.0);\n }\n\n if let Some(tiler) = DrawTiler::new(self.width(), self.height()) {\n let mut path = path.clone(); // TODO: avoid cloning\n let mut paint = paint.clone();\n\n if !transform.is_identity() {\n paint.shader.transform(transform);\n path = match path.transform(transform) {\n Some(v) => v,\n None => {\n log::warn!(\"path transformation failed\");\n return;\n }\n };\n }\n\n for tile in tiler {\n let ts = Transform::from_translate(-(tile.x() as f32), -(tile.y() as f32));\n path = match path.transform(ts) {\n Some(v) => v,\n None => {\n log::warn!(\"path transformation failed\");\n return;\n }\n };\n paint.shader.transform(ts);\n\n let mut subpix = match self.subpixmap(tile.to_int_rect()) {\n Some(v) => v,\n None => continue, // technically unreachable\n };\n let submask = mask.and_then(|mask| mask.submask(tile.to_int_rect()));\n\n // We're ignoring \"errors\" here, because `stroke_hairline` will return `None`\n // when rendering a tile that doesn't have a path on it.\n // Which is not an error in this case.\n Self::stroke_hairline(&path, &paint, stroke.line_cap, submask, &mut subpix);\n\n let ts = Transform::from_translate(tile.x() as f32, tile.y() as f32);\n path = match path.transform(ts) {\n Some(v) => v,\n None => return,\n };\n paint.shader.transform(ts);\n }\n } else {\n let subpix = &mut self.as_subpixmap();\n let submask = mask.map(|mask| mask.as_submask());\n if !transform.is_identity() {\n paint.shader.transform(transform);\n\n // TODO: avoid clone\n let path = match path.clone().transform(transform) {\n Some(v) => v,\n None => {\n log::warn!(\"path transformation failed\");\n return;\n }\n };\n\n Self::stroke_hairline(&path, &paint, stroke.line_cap, submask, subpix);\n } else {\n Self::stroke_hairline(path, &paint, stroke.line_cap, submask, subpix);\n }\n }\n } else {\n let path = match path.stroke(stroke, res_scale) {\n Some(v) => v,\n None => {\n log::warn!(\"path stroking failed\");\n return;\n }\n };\n\n self.fill_path(&path, paint, FillRule::Winding, transform, mask);\n }\n }\n\n /// A stroking for paths with subpixel/hairline width.\n fn stroke_hairline(\n path: &Path,\n paint: &Paint,\n line_cap: LineCap,\n mask: Option,\n pixmap: &mut SubPixmapMut,\n ) {\n let clip = pixmap.size.to_screen_int_rect(0, 0);\n let mut blitter = match RasterPipelineBlitter::new(paint, mask, pixmap) {\n Some(v) => v,\n None => return, // nothing to do, all good\n };\n if paint.anti_alias {\n scan::hairline_aa::stroke_path(path, line_cap, &clip, &mut blitter);\n } else {\n scan::hairline::stroke_path(path, line_cap, &clip, &mut blitter);\n }\n }\n\n /// Draws a `Pixmap` on top of the current `Pixmap`.\n ///\n /// The same as filling a rectangle with a `pixmap` pattern.\n pub fn draw_pixmap(\n &mut self,\n x: i32,\n y: i32,\n pixmap: PixmapRef,\n paint: &PixmapPaint,\n transform: Transform,\n mask: Option<&Mask>,\n ) {\n let rect = pixmap.size().to_int_rect(x, y).to_rect();\n\n // TODO: SkSpriteBlitter\n // TODO: partially clipped\n // TODO: clipped out\n\n // Translate pattern as well as bounds.\n let patt_transform = Transform::from_translate(x as f32, y as f32);\n\n let paint = Paint {\n shader: Pattern::new(\n pixmap,\n SpreadMode::Pad, // Pad, otherwise we will get weird borders overlap.\n paint.quality,\n paint.opacity,\n patt_transform,\n ),\n blend_mode: paint.blend_mode,\n anti_alias: false, // Skia doesn't use it too.\n force_hq_pipeline: false, // Pattern will use hq anyway.\n };\n\n self.fill_rect(rect, &paint, transform, mask);\n }\n\n /// Applies a masks.\n ///\n /// When a `Mask` is passed to drawing methods, it will be used to mask-out\n /// content we're about to draw.\n /// This method masks-out an already drawn content.\n /// It's not as fast, but can be useful when a mask is not available during drawing.\n ///\n /// This method is similar to filling the whole pixmap with an another,\n /// mask-like pixmap using the `DestinationOut` blend mode.\n ///\n /// `Mask` must have the same size as `Pixmap`. No transform or offset are allowed.\n pub fn apply_mask(&mut self, mask: &Mask) {\n if self.size() != mask.size() {\n log::warn!(\"Pixmap and Mask are expected to have the same size\");\n return;\n }\n\n // Just a dummy.\n let pixmap_src = PixmapRef::from_bytes(&[0, 0, 0, 0], 1, 1).unwrap();\n\n let mut p = RasterPipelineBuilder::new();\n p.push(pipeline::Stage::LoadMaskU8);\n p.push(pipeline::Stage::LoadDestination);\n p.push(pipeline::Stage::DestinationIn);\n p.push(pipeline::Stage::Store);\n let mut p = p.compile();\n let rect = self.size().to_screen_int_rect(0, 0);\n p.run(\n &rect,\n pipeline::AAMaskCtx::default(),\n mask.as_submask().mask_ctx(),\n pixmap_src,\n &mut self.as_subpixmap(),\n );\n }\n}\n\nfn treat_as_hairline(paint: &Paint, stroke: &Stroke, mut ts: Transform) -> Option {\n fn fast_len(p: Point) -> f32 {\n let mut x = p.x.abs();\n let mut y = p.y.abs();\n if x < y {\n core::mem::swap(&mut x, &mut y);\n }\n\n x + y.half()\n }\n\n debug_assert!(stroke.width >= 0.0);\n\n if stroke.width == 0.0 {\n return Some(1.0);\n }\n\n if !paint.anti_alias {\n return None;\n }\n\n // We don't care about translate.\n ts.tx = 0.0;\n ts.ty = 0.0;\n\n // We need to try to fake a thick-stroke with a modulated hairline.\n let mut points = [\n Point::from_xy(stroke.width, 0.0),\n Point::from_xy(0.0, stroke.width),\n ];\n ts.map_points(&mut points);\n\n let len0 = fast_len(points[0]);\n let len1 = fast_len(points[1]);\n\n if len0 <= 1.0 && len1 <= 1.0 {\n return Some(len0.ave(len1));\n }\n\n None\n}\n\n/// Sometimes in the drawing pipeline, we have to perform math on path coordinates, even after\n/// the path is in device-coordinates. Tessellation and clipping are two examples. Usually this\n/// is pretty modest, but it can involve subtracting/adding coordinates, or multiplying by\n/// small constants (e.g. 2,3,4). To try to preflight issues where these optionations could turn\n/// finite path values into infinities (or NaNs), we allow the upper drawing code to reject\n/// the path if its bounds (in device coordinates) is too close to max float.\npub(crate) fn is_too_big_for_math(path: &Path) -> bool {\n // This value is just a guess. smaller is safer, but we don't want to reject largish paths\n // that we don't have to.\n const SCALE_DOWN_TO_ALLOW_FOR_SMALL_MULTIPLIES: f32 = 0.25;\n const MAX: f32 = SCALAR_MAX * SCALE_DOWN_TO_ALLOW_FOR_SMALL_MULTIPLIES;\n\n let b = path.bounds();\n\n // use ! expression so we return true if bounds contains NaN\n !(b.left() >= -MAX && b.top() >= -MAX && b.right() <= MAX && b.bottom() <= MAX)\n}\n\n/// Splits the target pixmap into a list of tiles.\n///\n/// Skia/tiny-skia uses a lot of fixed-point math during path rendering.\n/// Probably more for precision than performance.\n/// And our fixed-point types are limited by 8192 and 32768.\n/// Which means that we cannot render a path larger than 8192 onto a pixmap.\n/// When pixmap is smaller than 8192, the path will be automatically clipped anyway,\n/// but for large pixmaps we have to render in tiles.\npub(crate) struct DrawTiler {\n image_width: u32,\n image_height: u32,\n x_offset: u32,\n y_offset: u32,\n finished: bool,\n}\n\nimpl DrawTiler {\n // 8K is 1 too big, since 8K << supersample == 32768 which is too big for Fixed.\n const MAX_DIMENSIONS: u32 = 8192 - 1;\n\n fn required(image_width: u32, image_height: u32) -> bool {\n image_width > Self::MAX_DIMENSIONS || image_height > Self::MAX_DIMENSIONS\n }\n\n pub(crate) fn new(image_width: u32, image_height: u32) -> Option {\n if Self::required(image_width, image_height) {\n Some(DrawTiler {\n image_width,\n image_height,\n x_offset: 0,\n y_offset: 0,\n finished: false,\n })\n } else {\n None\n }\n }\n}\n\nimpl Iterator for DrawTiler {\n type Item = ScreenIntRect;\n\n fn next(&mut self) -> Option {\n if self.finished {\n return None;\n }\n\n // TODO: iterate only over tiles that actually affected by the shape\n\n if self.x_offset < self.image_width && self.y_offset < self.image_height {\n let h = if self.y_offset < self.image_height {\n (self.image_height - self.y_offset).min(Self::MAX_DIMENSIONS)\n } else {\n self.image_height\n };\n\n let r = ScreenIntRect::from_xywh(\n self.x_offset,\n self.y_offset,\n (self.image_width - self.x_offset).min(Self::MAX_DIMENSIONS),\n h,\n );\n\n self.x_offset += Self::MAX_DIMENSIONS;\n if self.x_offset >= self.image_width {\n self.x_offset = 0;\n self.y_offset += Self::MAX_DIMENSIONS;\n }\n\n return r;\n }\n\n None\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n const MAX_DIM: u32 = DrawTiler::MAX_DIMENSIONS;\n\n #[test]\n fn skip() {\n assert!(DrawTiler::new(100, 500).is_none());\n }\n\n #[test]\n fn horizontal() {\n let mut iter = DrawTiler::new(10000, 500).unwrap();\n assert_eq!(iter.next(), ScreenIntRect::from_xywh(0, 0, MAX_DIM, 500));\n assert_eq!(\n iter.next(),\n ScreenIntRect::from_xywh(MAX_DIM, 0, 10000 - MAX_DIM, 500)\n );\n assert_eq!(iter.next(), None);\n }\n\n #[test]\n fn vertical() {\n let mut iter = DrawTiler::new(500, 10000).unwrap();\n assert_eq!(iter.next(), ScreenIntRect::from_xywh(0, 0, 500, MAX_DIM));\n assert_eq!(\n iter.next(),\n ScreenIntRect::from_xywh(0, MAX_DIM, 500, 10000 - MAX_DIM)\n );\n assert_eq!(iter.next(), None);\n }\n\n #[test]\n fn rect() {\n let mut iter = DrawTiler::new(10000, 10000).unwrap();\n // Row 1\n assert_eq!(\n iter.next(),\n ScreenIntRect::from_xywh(0, 0, MAX_DIM, MAX_DIM)\n );\n assert_eq!(\n iter.next(),\n ScreenIntRect::from_xywh(MAX_DIM, 0, 10000 - MAX_DIM, MAX_DIM)\n );\n // Row 2\n assert_eq!(\n iter.next(),\n ScreenIntRect::from_xywh(0, MAX_DIM, MAX_DIM, 10000 - MAX_DIM)\n );\n assert_eq!(\n iter.next(),\n ScreenIntRect::from_xywh(MAX_DIM, MAX_DIM, 10000 - MAX_DIM, 10000 - MAX_DIM)\n );\n assert_eq!(iter.next(), None);\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/path64/cubic64.rs", "sha256": "1f03bfd3a26c465a8a861aac7ce7d9b6d77ad73c9db07f27fe13500b02e83d60", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2012 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse super::point64::{Point64, SearchAxis};\nuse super::quad64;\nuse super::Scalar64;\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\npub const POINT_COUNT: usize = 4;\nconst PI: f64 = 3.141592653589793;\n\npub struct Cubic64Pair {\n pub points: [Point64; 7],\n}\n\npub struct Cubic64 {\n pub points: [Point64; POINT_COUNT],\n}\n\nimpl Cubic64 {\n pub fn new(points: [Point64; POINT_COUNT]) -> Self {\n Cubic64 { points }\n }\n\n pub fn as_f64_slice(&self) -> [f64; POINT_COUNT * 2] {\n [\n self.points[0].x,\n self.points[0].y,\n self.points[1].x,\n self.points[1].y,\n self.points[2].x,\n self.points[2].y,\n self.points[3].x,\n self.points[3].y,\n ]\n }\n\n pub fn point_at_t(&self, t: f64) -> Point64 {\n if t == 0.0 {\n return self.points[0];\n }\n\n if t == 1.0 {\n return self.points[3];\n }\n\n let one_t = 1.0 - t;\n let one_t2 = one_t * one_t;\n let a = one_t2 * one_t;\n let b = 3.0 * one_t2 * t;\n let t2 = t * t;\n let c = 3.0 * one_t * t2;\n let d = t2 * t;\n Point64::from_xy(\n a * self.points[0].x\n + b * self.points[1].x\n + c * self.points[2].x\n + d * self.points[3].x,\n a * self.points[0].y\n + b * self.points[1].y\n + c * self.points[2].y\n + d * self.points[3].y,\n )\n }\n\n pub fn search_roots(\n &self,\n mut extrema: usize,\n axis_intercept: f64,\n x_axis: SearchAxis,\n extreme_ts: &mut [f64; 6],\n valid_roots: &mut [f64],\n ) -> usize {\n extrema += self.find_inflections(&mut extreme_ts[extrema..]);\n extreme_ts[extrema] = 0.0;\n extrema += 1;\n extreme_ts[extrema] = 1.0;\n debug_assert!(extrema < 6);\n extreme_ts[0..extrema].sort_by(cmp_f64);\n let mut valid_count = 0;\n let mut index = 0;\n while index < extrema {\n let min = extreme_ts[index];\n index += 1;\n let max = extreme_ts[index];\n if min == max {\n continue;\n }\n\n let new_t = self.binary_search(min, max, axis_intercept, x_axis);\n if new_t >= 0.0 {\n if valid_count >= 3 {\n return 0;\n }\n\n valid_roots[valid_count] = new_t;\n valid_count += 1;\n }\n }\n\n valid_count\n }\n\n fn find_inflections(&self, t_values: &mut [f64]) -> usize {\n let ax = self.points[1].x - self.points[0].x;\n let ay = self.points[1].y - self.points[0].y;\n let bx = self.points[2].x - 2.0 * self.points[1].x + self.points[0].x;\n let by = self.points[2].y - 2.0 * self.points[1].y + self.points[0].y;\n let cx = self.points[3].x + 3.0 * (self.points[1].x - self.points[2].x) - self.points[0].x;\n let cy = self.points[3].y + 3.0 * (self.points[1].y - self.points[2].y) - self.points[0].y;\n quad64::roots_valid_t(\n bx * cy - by * cx,\n ax * cy - ay * cx,\n ax * by - ay * bx,\n t_values,\n )\n }\n\n // give up when changing t no longer moves point\n // also, copy point rather than recompute it when it does change\n fn binary_search(&self, min: f64, max: f64, axis_intercept: f64, x_axis: SearchAxis) -> f64 {\n let mut t = (min + max) / 2.0;\n let mut step = (t - min) / 2.0;\n let mut cubic_at_t = self.point_at_t(t);\n let mut calc_pos = cubic_at_t.axis_coord(x_axis);\n let mut calc_dist = calc_pos - axis_intercept;\n loop {\n let prior_t = min.max(t - step);\n let less_pt = self.point_at_t(prior_t);\n if less_pt.x.approximately_equal_half(cubic_at_t.x)\n && less_pt.y.approximately_equal_half(cubic_at_t.y)\n {\n return -1.0; // binary search found no point at this axis intercept\n }\n\n let less_dist = less_pt.axis_coord(x_axis) - axis_intercept;\n let last_step = step;\n step /= 2.0;\n let ok = if calc_dist > 0.0 {\n calc_dist > less_dist\n } else {\n calc_dist < less_dist\n };\n if ok {\n t = prior_t;\n } else {\n let next_t = t + last_step;\n if next_t > max {\n return -1.0;\n }\n\n let more_pt = self.point_at_t(next_t);\n if more_pt.x.approximately_equal_half(cubic_at_t.x)\n && more_pt.y.approximately_equal_half(cubic_at_t.y)\n {\n return -1.0; // binary search found no point at this axis intercept\n }\n\n let more_dist = more_pt.axis_coord(x_axis) - axis_intercept;\n let ok = if calc_dist > 0.0 {\n calc_dist <= more_dist\n } else {\n calc_dist >= more_dist\n };\n if ok {\n continue;\n }\n\n t = next_t;\n }\n\n let test_at_t = self.point_at_t(t);\n cubic_at_t = test_at_t;\n calc_pos = cubic_at_t.axis_coord(x_axis);\n calc_dist = calc_pos - axis_intercept;\n\n if calc_pos.approximately_equal(axis_intercept) {\n break;\n }\n }\n\n t\n }\n\n pub fn chop_at(&self, t: f64) -> Cubic64Pair {\n let mut dst = [Point64::zero(); 7];\n if t == 0.5 {\n dst[0] = self.points[0];\n dst[1].x = (self.points[0].x + self.points[1].x) / 2.0;\n dst[1].y = (self.points[0].y + self.points[1].y) / 2.0;\n dst[2].x = (self.points[0].x + 2.0 * self.points[1].x + self.points[2].x) / 4.0;\n dst[2].y = (self.points[0].y + 2.0 * self.points[1].y + self.points[2].y) / 4.0;\n dst[3].x =\n (self.points[0].x + 3.0 * (self.points[1].x + self.points[2].x) + self.points[3].x)\n / 8.0;\n dst[3].y =\n (self.points[0].y + 3.0 * (self.points[1].y + self.points[2].y) + self.points[3].y)\n / 8.0;\n dst[4].x = (self.points[1].x + 2.0 * self.points[2].x + self.points[3].x) / 4.0;\n dst[4].y = (self.points[1].y + 2.0 * self.points[2].y + self.points[3].y) / 4.0;\n dst[5].x = (self.points[2].x + self.points[3].x) / 2.0;\n dst[5].y = (self.points[2].y + self.points[3].y) / 2.0;\n dst[6] = self.points[3];\n\n Cubic64Pair { points: dst }\n } else {\n interp_cubic_coords_x(&self.points, t, &mut dst);\n interp_cubic_coords_y(&self.points, t, &mut dst);\n Cubic64Pair { points: dst }\n }\n }\n}\n\npub fn coefficients(src: &[f64]) -> (f64, f64, f64, f64) {\n let mut a = src[6]; // d\n let mut b = src[4] * 3.0; // 3*c\n let mut c = src[2] * 3.0; // 3*b\n let d = src[0]; // a\n a -= d - c + b; // A = -a + 3*b - 3*c + d\n b += 3.0 * d - 2.0 * c; // B = 3*a - 6*b + 3*c\n c -= 3.0 * d; // C = -3*a + 3*b\n\n (a, b, c, d)\n}\n\n// from SkGeometry.cpp (and Numeric Solutions, 5.6)\npub fn roots_valid_t(a: f64, b: f64, c: f64, d: f64, t: &mut [f64; 3]) -> usize {\n let mut s = [0.0; 3];\n let real_roots = roots_real(a, b, c, d, &mut s);\n let mut found_roots = quad64::push_valid_ts(&s, real_roots, t);\n 'outer: for index in 0..real_roots {\n let t_value = s[index];\n if !t_value.approximately_one_or_less() && t_value.between(1.0, 1.00005) {\n for idx2 in 0..found_roots {\n if t[idx2].approximately_equal(1.0) {\n continue 'outer;\n }\n }\n\n debug_assert!(found_roots < 3);\n t[found_roots] = 1.0;\n found_roots += 1;\n } else if !t_value.approximately_zero_or_more() && t_value.between(-0.00005, 0.0) {\n for idx2 in 0..found_roots {\n if t[idx2].approximately_equal(0.0) {\n continue 'outer;\n }\n }\n\n debug_assert!(found_roots < 3);\n t[found_roots] = 0.0;\n found_roots += 1;\n }\n }\n\n found_roots\n}\n\nfn roots_real(a: f64, b: f64, c: f64, d: f64, s: &mut [f64; 3]) -> usize {\n if a.approximately_zero()\n && a.approximately_zero_when_compared_to(b)\n && a.approximately_zero_when_compared_to(c)\n && a.approximately_zero_when_compared_to(d)\n {\n // we're just a quadratic\n return quad64::roots_real(b, c, d, s);\n }\n\n if d.approximately_zero_when_compared_to(a)\n && d.approximately_zero_when_compared_to(b)\n && d.approximately_zero_when_compared_to(c)\n {\n // 0 is one root\n let mut num = quad64::roots_real(a, b, c, s);\n for i in 0..num {\n if s[i].approximately_zero() {\n return num;\n }\n }\n\n s[num] = 0.0;\n num += 1;\n\n return num;\n }\n\n if (a + b + c + d).approximately_zero() {\n // 1 is one root\n let mut num = quad64::roots_real(a, a + b, -d, s);\n for i in 0..num {\n if s[i].almost_dequal_ulps(1.0) {\n return num;\n }\n }\n s[num] = 1.0;\n num += 1;\n return num;\n }\n\n let (a, b, c) = {\n let inv_a = 1.0 / a;\n let a = b * inv_a;\n let b = c * inv_a;\n let c = d * inv_a;\n (a, b, c)\n };\n\n let a2 = a * a;\n let q = (a2 - b * 3.0) / 9.0;\n let r = (2.0 * a2 * a - 9.0 * a * b + 27.0 * c) / 54.0;\n let r2 = r * r;\n let q3 = q * q * q;\n let r2_minus_q3 = r2 - q3;\n let adiv3 = a / 3.0;\n let mut offset = 0;\n if r2_minus_q3 < 0.0 {\n // we have 3 real roots\n\n // the divide/root can, due to finite precisions, be slightly outside of -1...1\n let theta = (r / q3.sqrt()).bound(-1.0, 1.0).acos();\n let neg2_root_q = -2.0 * q.sqrt();\n\n let mut rr = neg2_root_q * (theta / 3.0).cos() - adiv3;\n s[offset] = rr;\n offset += 1;\n\n rr = neg2_root_q * ((theta + 2.0 * PI) / 3.0).cos() - adiv3;\n if !s[0].almost_dequal_ulps(rr) {\n s[offset] = rr;\n offset += 1;\n }\n\n rr = neg2_root_q * ((theta - 2.0 * PI) / 3.0).cos() - adiv3;\n if !s[0].almost_dequal_ulps(rr) && (offset == 1 || !s[1].almost_dequal_ulps(rr)) {\n s[offset] = rr;\n offset += 1;\n }\n } else {\n // we have 1 real root\n let sqrt_r2_minus_q3 = r2_minus_q3.sqrt();\n let mut a = r.abs() + sqrt_r2_minus_q3;\n a = super::cube_root(a);\n if r > 0.0 {\n a = -a;\n }\n\n if a != 0.0 {\n a += q / a;\n }\n\n let mut r2 = a - adiv3;\n s[offset] = r2;\n offset += 1;\n if r2.almost_dequal_ulps(q3) {\n r2 = -a / 2.0 - adiv3;\n if !s[0].almost_dequal_ulps(r2) {\n s[offset] = r2;\n offset += 1;\n }\n }\n }\n\n offset\n}\n\n// Cubic64'(t) = At^2 + Bt + C, where\n// A = 3(-a + 3(b - c) + d)\n// B = 6(a - 2b + c)\n// C = 3(b - a)\n// Solve for t, keeping only those that fit between 0 < t < 1\npub fn find_extrema(src: &[f64], t_values: &mut [f64]) -> usize {\n // we divide A,B,C by 3 to simplify\n let a = src[0];\n let b = src[2];\n let c = src[4];\n let d = src[6];\n let a2 = d - a + 3.0 * (b - c);\n let b2 = 2.0 * (a - b - b + c);\n let c2 = b - a;\n\n quad64::roots_valid_t(a2, b2, c2, t_values)\n}\n\n// Skia doesn't seems to care about NaN/inf during sorting, so we don't too.\nfn cmp_f64(a: &f64, b: &f64) -> core::cmp::Ordering {\n if a < b {\n core::cmp::Ordering::Less\n } else if a > b {\n core::cmp::Ordering::Greater\n } else {\n core::cmp::Ordering::Equal\n }\n}\n\n// classic one t subdivision\nfn interp_cubic_coords_x(src: &[Point64; 4], t: f64, dst: &mut [Point64; 7]) {\n use super::interp;\n\n let ab = interp(src[0].x, src[1].x, t);\n let bc = interp(src[1].x, src[2].x, t);\n let cd = interp(src[2].x, src[3].x, t);\n let abc = interp(ab, bc, t);\n let bcd = interp(bc, cd, t);\n let abcd = interp(abc, bcd, t);\n\n dst[0].x = src[0].x;\n dst[1].x = ab;\n dst[2].x = abc;\n dst[3].x = abcd;\n dst[4].x = bcd;\n dst[5].x = cd;\n dst[6].x = src[3].x;\n}\n\nfn interp_cubic_coords_y(src: &[Point64; 4], t: f64, dst: &mut [Point64; 7]) {\n use super::interp;\n\n let ab = interp(src[0].y, src[1].y, t);\n let bc = interp(src[1].y, src[2].y, t);\n let cd = interp(src[2].y, src[3].y, t);\n let abc = interp(ab, bc, t);\n let bcd = interp(bc, cd, t);\n let abcd = interp(abc, bcd, t);\n\n dst[0].y = src[0].y;\n dst[1].y = ab;\n dst[2].y = abc;\n dst[3].y = abcd;\n dst[4].y = bcd;\n dst[5].y = cd;\n dst[6].y = src[3].y;\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/path64/line_cubic_intersections.rs", "sha256": "67164e6971ae50b553b7553b3a37e389966d25590b637a3b5b1b3aecf77fd5df", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2012 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n/*\nFind the intersection of a line and cubic by solving for valid t values.\n\nAnalogous to line-quadratic intersection, solve line-cubic intersection by\nrepresenting the cubic as:\n x = a(1-t)^3 + 2b(1-t)^2t + c(1-t)t^2 + dt^3\n y = e(1-t)^3 + 2f(1-t)^2t + g(1-t)t^2 + ht^3\nand the line as:\n y = i*x + j (if the line is more horizontal)\nor:\n x = i*y + j (if the line is more vertical)\n\nThen using Mathematica, solve for the values of t where the cubic intersects the\nline:\n\n (in) Resultant[\n a*(1 - t)^3 + 3*b*(1 - t)^2*t + 3*c*(1 - t)*t^2 + d*t^3 - x,\n e*(1 - t)^3 + 3*f*(1 - t)^2*t + 3*g*(1 - t)*t^2 + h*t^3 - i*x - j, x]\n (out) -e + j +\n 3 e t - 3 f t -\n 3 e t^2 + 6 f t^2 - 3 g t^2 +\n e t^3 - 3 f t^3 + 3 g t^3 - h t^3 +\n i ( a -\n 3 a t + 3 b t +\n 3 a t^2 - 6 b t^2 + 3 c t^2 -\n a t^3 + 3 b t^3 - 3 c t^3 + d t^3 )\n\nif i goes to infinity, we can rewrite the line in terms of x. Mathematica:\n\n (in) Resultant[\n a*(1 - t)^3 + 3*b*(1 - t)^2*t + 3*c*(1 - t)*t^2 + d*t^3 - i*y - j,\n e*(1 - t)^3 + 3*f*(1 - t)^2*t + 3*g*(1 - t)*t^2 + h*t^3 - y, y]\n (out) a - j -\n 3 a t + 3 b t +\n 3 a t^2 - 6 b t^2 + 3 c t^2 -\n a t^3 + 3 b t^3 - 3 c t^3 + d t^3 -\n i ( e -\n 3 e t + 3 f t +\n 3 e t^2 - 6 f t^2 + 3 g t^2 -\n e t^3 + 3 f t^3 - 3 g t^3 + h t^3 )\n\nSolving this with Mathematica produces an expression with hundreds of terms;\ninstead, use Numeric Solutions recipe to solve the cubic.\n\nThe near-horizontal case, in terms of: Ax^3 + Bx^2 + Cx + D == 0\n A = (-(-e + 3*f - 3*g + h) + i*(-a + 3*b - 3*c + d) )\n B = 3*(-( e - 2*f + g ) + i*( a - 2*b + c ) )\n C = 3*(-(-e + f ) + i*(-a + b ) )\n D = (-( e ) + i*( a ) + j )\n\nThe near-vertical case, in terms of: Ax^3 + Bx^2 + Cx + D == 0\n A = ( (-a + 3*b - 3*c + d) - i*(-e + 3*f - 3*g + h) )\n B = 3*( ( a - 2*b + c ) - i*( e - 2*f + g ) )\n C = 3*( (-a + b ) - i*(-e + f ) )\n D = ( ( a ) - i*( e ) - j )\n\nFor horizontal lines:\n(in) Resultant[\n a*(1 - t)^3 + 3*b*(1 - t)^2*t + 3*c*(1 - t)*t^2 + d*t^3 - j,\n e*(1 - t)^3 + 3*f*(1 - t)^2*t + 3*g*(1 - t)*t^2 + h*t^3 - y, y]\n(out) e - j -\n 3 e t + 3 f t +\n 3 e t^2 - 6 f t^2 + 3 g t^2 -\n e t^3 + 3 f t^3 - 3 g t^3 + h t^3\n*/\n\nuse super::cubic64::{self, Cubic64};\nuse super::point64::SearchAxis;\nuse super::Scalar64;\n\npub fn horizontal_intersect(cubic: &Cubic64, axis_intercept: f64, roots: &mut [f64; 3]) -> usize {\n let (a, b, c, mut d) = cubic64::coefficients(&cubic.as_f64_slice()[1..]);\n d -= axis_intercept;\n let mut count = cubic64::roots_valid_t(a, b, c, d, roots);\n let mut index = 0;\n while index < count {\n let calc_pt = cubic.point_at_t(roots[index]);\n if !calc_pt.y.approximately_equal(axis_intercept) {\n let mut extreme_ts = [0.0; 6];\n let extrema = cubic64::find_extrema(&cubic.as_f64_slice()[1..], &mut extreme_ts);\n count = cubic.search_roots(\n extrema,\n axis_intercept,\n SearchAxis::Y,\n &mut extreme_ts,\n roots,\n );\n break;\n }\n\n index += 1;\n }\n\n count\n}\n\npub fn vertical_intersect(cubic: &Cubic64, axis_intercept: f64, roots: &mut [f64; 3]) -> usize {\n let (a, b, c, mut d) = cubic64::coefficients(&cubic.as_f64_slice());\n d -= axis_intercept;\n let mut count = cubic64::roots_valid_t(a, b, c, d, roots);\n let mut index = 0;\n while index < count {\n let calc_pt = cubic.point_at_t(roots[index]);\n if !calc_pt.x.approximately_equal(axis_intercept) {\n let mut extreme_ts = [0.0; 6];\n let extrema = cubic64::find_extrema(&cubic.as_f64_slice(), &mut extreme_ts);\n count = cubic.search_roots(\n extrema,\n axis_intercept,\n SearchAxis::X,\n &mut extreme_ts,\n roots,\n );\n break;\n }\n\n index += 1;\n }\n\n count\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/path64/mod.rs", "sha256": "b1a931a1e0cfcb2d42e50ae535f4e6991a9db4ff41eb28cdb1f45f14cf0b2e95", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2012 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse tiny_skia_path::{Scalar, SCALAR_MAX};\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\n// Must be first, because of macro scope rules.\n#[macro_use]\npub mod point64;\n\npub mod cubic64;\npub mod line_cubic_intersections;\nmod quad64;\n\n// The code below is from SkPathOpsTypes.\n\nconst DBL_EPSILON_ERR: f64 = f64::EPSILON * 4.0;\nconst FLT_EPSILON_HALF: f64 = (f32::EPSILON / 2.0) as f64;\nconst FLT_EPSILON_CUBED: f64 = (f32::EPSILON * f32::EPSILON * f32::EPSILON) as f64;\nconst FLT_EPSILON_INVERSE: f64 = 1.0 / f32::EPSILON as f64;\n\npub trait Scalar64 {\n fn bound(self, min: Self, max: Self) -> Self;\n fn between(self, a: f64, b: f64) -> bool;\n fn precisely_zero(self) -> bool;\n fn approximately_zero_or_more(self) -> bool;\n fn approximately_one_or_less(self) -> bool;\n fn approximately_zero(self) -> bool;\n fn approximately_zero_inverse(self) -> bool;\n fn approximately_zero_cubed(self) -> bool;\n fn approximately_zero_half(self) -> bool;\n fn approximately_zero_when_compared_to(self, other: Self) -> bool;\n fn approximately_equal(self, other: Self) -> bool;\n fn approximately_equal_half(self, other: Self) -> bool;\n fn almost_dequal_ulps(self, other: Self) -> bool;\n}\n\nimpl Scalar64 for f64 {\n // Works just like SkTPin, returning `max` for NaN/inf\n fn bound(self, min: Self, max: Self) -> Self {\n max.min(self).max(min)\n }\n\n /// Returns true if (a <= self <= b) || (a >= self >= b).\n fn between(self, a: f64, b: f64) -> bool {\n debug_assert!(\n ((a <= self && self <= b) || (a >= self && self >= b))\n == ((a - self) * (b - self) <= 0.0)\n || (a.precisely_zero() && self.precisely_zero() && b.precisely_zero())\n );\n\n (a - self) * (b - self) <= 0.0\n }\n\n fn precisely_zero(self) -> bool {\n self.abs() < DBL_EPSILON_ERR\n }\n\n fn approximately_zero_or_more(self) -> bool {\n self > -f64::EPSILON\n }\n\n fn approximately_one_or_less(self) -> bool {\n self < 1.0 + f64::EPSILON\n }\n\n fn approximately_zero(self) -> bool {\n self.abs() < f64::EPSILON\n }\n\n fn approximately_zero_inverse(self) -> bool {\n self.abs() > FLT_EPSILON_INVERSE\n }\n\n fn approximately_zero_cubed(self) -> bool {\n self.abs() < FLT_EPSILON_CUBED\n }\n\n fn approximately_zero_half(self) -> bool {\n self < FLT_EPSILON_HALF\n }\n\n fn approximately_zero_when_compared_to(self, other: Self) -> bool {\n self == 0.0 || self.abs() < (other * (f32::EPSILON as f64)).abs()\n }\n\n // Use this for comparing Ts in the range of 0 to 1. For general numbers (larger and smaller) use\n // AlmostEqualUlps instead.\n fn approximately_equal(self, other: Self) -> bool {\n (self - other).approximately_zero()\n }\n\n fn approximately_equal_half(self, other: Self) -> bool {\n (self - other).approximately_zero_half()\n }\n\n fn almost_dequal_ulps(self, other: Self) -> bool {\n if self.abs() < SCALAR_MAX as f64 && other.abs() < SCALAR_MAX as f64 {\n (self as f32).almost_dequal_ulps(other as f32)\n } else {\n (self - other).abs() / self.abs().max(other.abs()) < (f32::EPSILON * 16.0) as f64\n }\n }\n}\n\npub fn cube_root(x: f64) -> f64 {\n if x.approximately_zero_cubed() {\n return 0.0;\n }\n\n let result = halley_cbrt3d(x.abs());\n if x < 0.0 {\n -result\n } else {\n result\n }\n}\n\n// cube root approximation using 3 iterations of Halley's method (double)\nfn halley_cbrt3d(d: f64) -> f64 {\n let mut a = cbrt_5d(d);\n a = cbrta_halleyd(a, d);\n a = cbrta_halleyd(a, d);\n cbrta_halleyd(a, d)\n}\n\n// cube root approximation using bit hack for 64-bit float\n// adapted from Kahan's cbrt\nfn cbrt_5d(d: f64) -> f64 {\n let b1 = 715094163;\n let mut t: f64 = 0.0;\n let pt: &mut [u32; 2] = bytemuck::cast_mut(&mut t);\n let px: [u32; 2] = bytemuck::cast(d);\n pt[1] = px[1] / 3 + b1;\n t\n}\n\n// iterative cube root approximation using Halley's method (double)\nfn cbrta_halleyd(a: f64, r: f64) -> f64 {\n let a3 = a * a * a;\n a * (a3 + r + r) / (a3 + a3 + r)\n}\n\nfn interp(a: f64, b: f64, t: f64) -> f64 {\n a + (b - a) * t\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/path64/point64.rs", "sha256": "1f2c35539aad92ed3a3335d5bf03fedff2ac125ee104f732588694412835fd63", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2012 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse crate::Point;\n\n#[derive(Copy, Clone, PartialEq, Debug)]\npub enum SearchAxis {\n X,\n Y,\n}\n\n#[repr(C)]\n#[derive(Copy, Clone, PartialEq, Default, Debug)]\npub struct Point64 {\n pub x: f64,\n pub y: f64,\n}\n\nimpl Point64 {\n pub fn from_xy(x: f64, y: f64) -> Self {\n Point64 { x, y }\n }\n\n pub fn from_point(p: Point) -> Self {\n Point64 {\n x: f64::from(p.x),\n y: f64::from(p.y),\n }\n }\n\n pub fn zero() -> Self {\n Point64 { x: 0.0, y: 0.0 }\n }\n\n pub fn to_point(&self) -> Point {\n Point::from_xy(self.x as f32, self.y as f32)\n }\n\n pub fn axis_coord(&self, axis: SearchAxis) -> f64 {\n match axis {\n SearchAxis::X => self.x,\n SearchAxis::Y => self.y,\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/path64/quad64.rs", "sha256": "05b3263b9287c5c506442f3e1d4dfb17bb3f729d8c5074ce4c6387ee489ffc96", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2012 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse super::Scalar64;\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\npub fn push_valid_ts(s: &[f64], real_roots: usize, t: &mut [f64]) -> usize {\n let mut found_roots = 0;\n 'outer: for index in 0..real_roots {\n let mut t_value = s[index];\n if t_value.approximately_zero_or_more() && t_value.approximately_one_or_less() {\n t_value = t_value.bound(0.0, 1.0);\n\n for idx2 in 0..found_roots {\n if t[idx2].approximately_equal(t_value) {\n continue 'outer;\n }\n }\n\n t[found_roots] = t_value;\n found_roots += 1;\n }\n }\n\n found_roots\n}\n\n// note: caller expects multiple results to be sorted smaller first\n// note: http://en.wikipedia.org/wiki/Loss_of_significance has an interesting\n// analysis of the quadratic equation, suggesting why the following looks at\n// the sign of B -- and further suggesting that the greatest loss of precision\n// is in b squared less two a c\npub fn roots_valid_t(a: f64, b: f64, c: f64, t: &mut [f64]) -> usize {\n let mut s = [0.0; 3];\n let real_roots = roots_real(a, b, c, &mut s);\n push_valid_ts(&s, real_roots, t)\n}\n\n// Numeric Solutions (5.6) suggests to solve the quadratic by computing\n// Q = -1/2(B + sgn(B)Sqrt(B^2 - 4 A C))\n// and using the roots\n// t1 = Q / A\n// t2 = C / Q\n//\n// this does not discard real roots <= 0 or >= 1\npub fn roots_real(a: f64, b: f64, c: f64, s: &mut [f64; 3]) -> usize {\n if a == 0.0 {\n return handle_zero(b, c, s);\n }\n\n let p = b / (2.0 * a);\n let q = c / a;\n if a.approximately_zero() && (p.approximately_zero_inverse() || q.approximately_zero_inverse())\n {\n return handle_zero(b, c, s);\n }\n\n // normal form: x^2 + px + q = 0\n let p2 = p * p;\n if !p2.almost_dequal_ulps(q) && p2 < q {\n return 0;\n }\n\n let mut sqrt_d = 0.0;\n if p2 > q {\n sqrt_d = (p2 - q).sqrt();\n }\n\n s[0] = sqrt_d - p;\n s[1] = -sqrt_d - p;\n 1 + usize::from(!s[0].almost_dequal_ulps(s[1]))\n}\n\nfn handle_zero(b: f64, c: f64, s: &mut [f64; 3]) -> usize {\n if b.approximately_zero() {\n s[0] = 0.0;\n (c == 0.0) as usize\n } else {\n s[0] = -c / b;\n 1\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/path_geometry.rs", "sha256": "7b3a7070bafb899cea0195b72dba9a88556633e17b208769342214d97daa9bd4", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse tiny_skia_path::{NormalizedF32, NormalizedF32Exclusive, Point};\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\npub use tiny_skia_path::path_geometry::{\n chop_cubic_at2, chop_quad_at, find_cubic_max_curvature, find_unit_quad_roots, new_t_values,\n CubicCoeff, QuadCoeff,\n};\n\nuse tiny_skia_path::path_geometry::valid_unit_divide;\n\n// TODO: return custom type\n/// Returns 0 for 1 quad, and 1 for two quads, either way the answer is stored in dst[].\n///\n/// Guarantees that the 1/2 quads will be monotonic.\npub fn chop_quad_at_x_extrema(src: &[Point; 3], dst: &mut [Point; 5]) -> usize {\n let a = src[0].x;\n let mut b = src[1].x;\n let c = src[2].x;\n\n if is_not_monotonic(a, b, c) {\n if let Some(t_value) = valid_unit_divide(a - b, a - b - b + c) {\n chop_quad_at(src, t_value, dst);\n\n // flatten double quad extrema\n dst[1].x = dst[2].x;\n dst[3].x = dst[2].x;\n\n return 1;\n }\n\n // if we get here, we need to force dst to be monotonic, even though\n // we couldn't compute a unit_divide value (probably underflow).\n b = if (a - b).abs() < (b - c).abs() { a } else { c };\n }\n\n dst[0] = Point::from_xy(a, src[0].y);\n dst[1] = Point::from_xy(b, src[1].y);\n dst[2] = Point::from_xy(c, src[2].y);\n 0\n}\n\n/// Returns 0 for 1 quad, and 1 for two quads, either way the answer is stored in dst[].\n///\n/// Guarantees that the 1/2 quads will be monotonic.\npub fn chop_quad_at_y_extrema(src: &[Point; 3], dst: &mut [Point; 5]) -> usize {\n let a = src[0].y;\n let mut b = src[1].y;\n let c = src[2].y;\n\n if is_not_monotonic(a, b, c) {\n if let Some(t_value) = valid_unit_divide(a - b, a - b - b + c) {\n chop_quad_at(src, t_value, dst);\n\n // flatten double quad extrema\n dst[1].y = dst[2].y;\n dst[3].y = dst[2].y;\n\n return 1;\n }\n\n // if we get here, we need to force dst to be monotonic, even though\n // we couldn't compute a unit_divide value (probably underflow).\n b = if (a - b).abs() < (b - c).abs() { a } else { c };\n }\n\n dst[0] = Point::from_xy(src[0].x, a);\n dst[1] = Point::from_xy(src[1].x, b);\n dst[2] = Point::from_xy(src[2].x, c);\n 0\n}\n\nfn is_not_monotonic(a: f32, b: f32, c: f32) -> bool {\n let ab = a - b;\n let mut bc = b - c;\n if ab < 0.0 {\n bc = -bc;\n }\n\n ab == 0.0 || bc < 0.0\n}\n\npub fn chop_cubic_at_x_extrema(src: &[Point; 4], dst: &mut [Point; 10]) -> usize {\n let mut t_values = new_t_values();\n let t_values = find_cubic_extrema(src[0].x, src[1].x, src[2].x, src[3].x, &mut t_values);\n\n chop_cubic_at(src, t_values, dst);\n if !t_values.is_empty() {\n // we do some cleanup to ensure our X extrema are flat\n dst[2].x = dst[3].x;\n dst[4].x = dst[3].x;\n if t_values.len() == 2 {\n dst[5].x = dst[6].x;\n dst[7].x = dst[6].x;\n }\n }\n\n t_values.len()\n}\n\n/// Given 4 points on a cubic bezier, chop it into 1, 2, 3 beziers such that\n/// the resulting beziers are monotonic in Y.\n///\n/// This is called by the scan converter.\n///\n/// Depending on what is returned, dst[] is treated as follows:\n///\n/// - 0: dst[0..3] is the original cubic\n/// - 1: dst[0..3] and dst[3..6] are the two new cubics\n/// - 2: dst[0..3], dst[3..6], dst[6..9] are the three new cubics\npub fn chop_cubic_at_y_extrema(src: &[Point; 4], dst: &mut [Point; 10]) -> usize {\n let mut t_values = new_t_values();\n let t_values = find_cubic_extrema(src[0].y, src[1].y, src[2].y, src[3].y, &mut t_values);\n\n chop_cubic_at(src, t_values, dst);\n if !t_values.is_empty() {\n // we do some cleanup to ensure our Y extrema are flat\n dst[2].y = dst[3].y;\n dst[4].y = dst[3].y;\n if t_values.len() == 2 {\n dst[5].y = dst[6].y;\n dst[7].y = dst[6].y;\n }\n }\n\n t_values.len()\n}\n\n// Cubic'(t) = At^2 + Bt + C, where\n// A = 3(-a + 3(b - c) + d)\n// B = 6(a - 2b + c)\n// C = 3(b - a)\n// Solve for t, keeping only those that fit between 0 < t < 1\nfn find_cubic_extrema(\n a: f32,\n b: f32,\n c: f32,\n d: f32,\n t_values: &mut [NormalizedF32Exclusive; 3],\n) -> &[NormalizedF32Exclusive] {\n // we divide A,B,C by 3 to simplify\n let na = d - a + 3.0 * (b - c);\n let nb = 2.0 * (a - b - b + c);\n let nc = b - a;\n\n let roots = find_unit_quad_roots(na, nb, nc, t_values);\n &t_values[0..roots]\n}\n\n// http://code.google.com/p/skia/issues/detail?id=32\n//\n// This test code would fail when we didn't check the return result of\n// valid_unit_divide in SkChopCubicAt(... NormalizedF32Exclusives[], int roots). The reason is\n// that after the first chop, the parameters to valid_unit_divide are equal\n// (thanks to finite float precision and rounding in the subtracts). Thus\n// even though the 2nd NormalizedF32Exclusive looks < 1.0, after we renormalize it, we end\n// up with 1.0, hence the need to check and just return the last cubic as\n// a degenerate clump of 4 points in the same place.\npub fn chop_cubic_at(src: &[Point; 4], t_values: &[NormalizedF32Exclusive], dst: &mut [Point]) {\n if t_values.is_empty() {\n // nothing to chop\n dst[0] = src[0];\n dst[1] = src[1];\n dst[2] = src[2];\n dst[3] = src[3];\n } else {\n let mut t = t_values[0];\n let mut tmp = [Point::zero(); 4];\n\n // Reduce the `src` lifetime, so we can use `src = &tmp` later.\n let mut src = src;\n\n let mut dst_offset = 0;\n for i in 0..t_values.len() {\n chop_cubic_at2(src, t, &mut dst[dst_offset..]);\n if i == t_values.len() - 1 {\n break;\n }\n\n dst_offset += 3;\n // have src point to the remaining cubic (after the chop)\n tmp[0] = dst[dst_offset + 0];\n tmp[1] = dst[dst_offset + 1];\n tmp[2] = dst[dst_offset + 2];\n tmp[3] = dst[dst_offset + 3];\n src = &tmp;\n\n // watch out in case the renormalized t isn't in range\n let n = valid_unit_divide(\n t_values[i + 1].get() - t_values[i].get(),\n 1.0 - t_values[i].get(),\n );\n\n match n {\n Some(n) => t = n,\n None => {\n // if we can't, just create a degenerate cubic\n dst[dst_offset + 4] = src[3];\n dst[dst_offset + 5] = src[3];\n dst[dst_offset + 6] = src[3];\n break;\n }\n }\n }\n }\n}\n\npub fn chop_cubic_at_max_curvature(\n src: &[Point; 4],\n t_values: &mut [NormalizedF32Exclusive; 3],\n dst: &mut [Point],\n) -> usize {\n let mut roots = [NormalizedF32::ZERO; 3];\n let roots = find_cubic_max_curvature(src, &mut roots);\n\n // Throw out values not inside 0..1.\n let mut count = 0;\n for root in roots {\n if 0.0 < root.get() && root.get() < 1.0 {\n t_values[count] = NormalizedF32Exclusive::new_bounded(root.get());\n count += 1;\n }\n }\n\n if count == 0 {\n dst[0..4].copy_from_slice(src);\n } else {\n chop_cubic_at(src, &t_values[0..count], dst);\n }\n\n count + 1\n}\n\npub fn chop_mono_cubic_at_x(src: &[Point; 4], x: f32, dst: &mut [Point; 7]) -> bool {\n cubic_dchop_at_intercept(src, x, true, dst)\n}\n\npub fn chop_mono_cubic_at_y(src: &[Point; 4], y: f32, dst: &mut [Point; 7]) -> bool {\n cubic_dchop_at_intercept(src, y, false, dst)\n}\n\nfn cubic_dchop_at_intercept(\n src: &[Point; 4],\n intercept: f32,\n is_vertical: bool,\n dst: &mut [Point; 7],\n) -> bool {\n use crate::path64::{cubic64::Cubic64, line_cubic_intersections, point64::Point64};\n\n let src = [\n Point64::from_point(src[0]),\n Point64::from_point(src[1]),\n Point64::from_point(src[2]),\n Point64::from_point(src[3]),\n ];\n\n let cubic = Cubic64::new(src);\n let mut roots = [0.0; 3];\n let count = if is_vertical {\n line_cubic_intersections::vertical_intersect(&cubic, f64::from(intercept), &mut roots)\n } else {\n line_cubic_intersections::horizontal_intersect(&cubic, f64::from(intercept), &mut roots)\n };\n\n if count > 0 {\n let pair = cubic.chop_at(roots[0]);\n for i in 0..7 {\n dst[i] = pair.points[i].to_point();\n }\n\n true\n } else {\n false\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn chop_cubic_at_y_extrema_1() {\n let src = [\n Point::from_xy(10.0, 20.0),\n Point::from_xy(67.0, 437.0),\n Point::from_xy(298.0, 213.0),\n Point::from_xy(401.0, 214.0),\n ];\n\n let mut dst = [Point::zero(); 10];\n let n = chop_cubic_at_y_extrema(&src, &mut dst);\n assert_eq!(n, 2);\n assert_eq!(dst[0], Point::from_xy(10.0, 20.0));\n assert_eq!(dst[1], Point::from_xy(37.508274, 221.24475));\n assert_eq!(dst[2], Point::from_xy(105.541855, 273.19803));\n assert_eq!(dst[3], Point::from_xy(180.15599, 273.19803));\n assert_eq!(dst[4], Point::from_xy(259.80502, 273.19803));\n assert_eq!(dst[5], Point::from_xy(346.9527, 213.99666));\n assert_eq!(dst[6], Point::from_xy(400.30844, 213.99666));\n assert_eq!(dst[7], Point::from_xy(400.53958, 213.99666));\n assert_eq!(dst[8], Point::from_xy(400.7701, 213.99777));\n assert_eq!(dst[9], Point::from_xy(401.0, 214.0));\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/pipeline/blitter.rs", "sha256": "959da6b806093b45ba3e8a368d4d81e6f3921a309b5ea8c39e7c989f2008e6b8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2016 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse crate::{BlendMode, Color, LengthU32, Paint, PixmapRef, PremultipliedColorU8, Shader};\nuse crate::{ALPHA_U8_OPAQUE, ALPHA_U8_TRANSPARENT};\n\nuse crate::alpha_runs::AlphaRun;\nuse crate::blitter::{Blitter, Mask};\nuse crate::color::AlphaU8;\nuse crate::geom::ScreenIntRect;\nuse crate::mask::SubMaskRef;\nuse crate::math::LENGTH_U32_ONE;\nuse crate::pipeline::{self, RasterPipeline, RasterPipelineBuilder};\nuse crate::pixmap::SubPixmapMut;\n\npub struct RasterPipelineBlitter<'a, 'b: 'a> {\n mask: Option>,\n pixmap_src: PixmapRef<'a>,\n pixmap: &'a mut SubPixmapMut<'b>,\n memset2d_color: Option,\n blit_anti_h_rp: RasterPipeline,\n blit_rect_rp: RasterPipeline,\n blit_mask_rp: RasterPipeline,\n is_mask: bool,\n}\n\nimpl<'a, 'b: 'a> RasterPipelineBlitter<'a, 'b> {\n pub fn new(\n paint: &Paint<'a>,\n mask: Option>,\n pixmap: &'a mut SubPixmapMut<'b>,\n ) -> Option {\n // Make sure that `mask` has the same size as `pixmap`.\n if let Some(mask) = mask {\n if mask.size.width() != pixmap.size.width()\n || mask.size.height() != pixmap.size.height()\n {\n log::warn!(\"Pixmap and Mask are expected to have the same size\");\n return None;\n }\n }\n\n // Fast-reject.\n // This is basically SkInterpretXfermode().\n match paint.blend_mode {\n // `Destination` keep the pixmap unchanged. Nothing to do here.\n BlendMode::Destination => return None,\n BlendMode::DestinationIn if paint.shader.is_opaque() && paint.is_solid_color() => {\n return None\n }\n _ => {}\n }\n\n // We can strength-reduce SourceOver into Source when opaque.\n let mut blend_mode = paint.blend_mode;\n if paint.shader.is_opaque() && blend_mode == BlendMode::SourceOver && mask.is_none() {\n blend_mode = BlendMode::Source;\n }\n\n // When we're drawing a constant color in Source mode, we can sometimes just memset.\n let mut memset2d_color = None;\n if paint.is_solid_color() && blend_mode == BlendMode::Source && mask.is_none() {\n // Unlike Skia, our shader cannot be constant.\n // Therefore there is no need to run a raster pipeline to get shader's color.\n if let Shader::SolidColor(ref color) = paint.shader {\n memset2d_color = Some(color.premultiply().to_color_u8());\n }\n };\n\n // Clear is just a transparent color memset.\n if blend_mode == BlendMode::Clear && !paint.anti_alias && mask.is_none() {\n blend_mode = BlendMode::Source;\n memset2d_color = Some(PremultipliedColorU8::TRANSPARENT);\n }\n\n let blit_anti_h_rp = {\n let mut p = RasterPipelineBuilder::new();\n p.set_force_hq_pipeline(paint.force_hq_pipeline);\n if !paint.shader.push_stages(&mut p) {\n return None;\n }\n\n if mask.is_some() {\n p.push(pipeline::Stage::MaskU8);\n }\n\n if blend_mode.should_pre_scale_coverage() {\n p.push(pipeline::Stage::Scale1Float);\n p.push(pipeline::Stage::LoadDestination);\n if let Some(blend_stage) = blend_mode.to_stage() {\n p.push(blend_stage);\n }\n } else {\n p.push(pipeline::Stage::LoadDestination);\n if let Some(blend_stage) = blend_mode.to_stage() {\n p.push(blend_stage);\n }\n\n p.push(pipeline::Stage::Lerp1Float);\n }\n\n p.push(pipeline::Stage::Store);\n\n p.compile()\n };\n\n let blit_rect_rp = {\n let mut p = RasterPipelineBuilder::new();\n p.set_force_hq_pipeline(paint.force_hq_pipeline);\n if !paint.shader.push_stages(&mut p) {\n return None;\n }\n\n if mask.is_some() {\n p.push(pipeline::Stage::MaskU8);\n }\n\n if blend_mode == BlendMode::SourceOver && mask.is_none() {\n // TODO: ignore when dither_rate is non-zero\n p.push(pipeline::Stage::SourceOverRgba);\n } else {\n if blend_mode != BlendMode::Source {\n p.push(pipeline::Stage::LoadDestination);\n if let Some(blend_stage) = blend_mode.to_stage() {\n p.push(blend_stage);\n }\n }\n\n p.push(pipeline::Stage::Store);\n }\n\n p.compile()\n };\n\n let blit_mask_rp = {\n let mut p = RasterPipelineBuilder::new();\n p.set_force_hq_pipeline(paint.force_hq_pipeline);\n if !paint.shader.push_stages(&mut p) {\n return None;\n }\n\n if mask.is_some() {\n p.push(pipeline::Stage::MaskU8);\n }\n\n if blend_mode.should_pre_scale_coverage() {\n p.push(pipeline::Stage::ScaleU8);\n p.push(pipeline::Stage::LoadDestination);\n if let Some(blend_stage) = blend_mode.to_stage() {\n p.push(blend_stage);\n }\n } else {\n p.push(pipeline::Stage::LoadDestination);\n if let Some(blend_stage) = blend_mode.to_stage() {\n p.push(blend_stage);\n }\n\n p.push(pipeline::Stage::LerpU8);\n }\n\n p.push(pipeline::Stage::Store);\n\n p.compile()\n };\n\n let pixmap_src = match paint.shader {\n Shader::Pattern(ref patt) => patt.pixmap,\n // Just a dummy one.\n _ => PixmapRef::from_bytes(&[0, 0, 0, 0], 1, 1).unwrap(),\n };\n\n Some(RasterPipelineBlitter {\n mask,\n pixmap_src,\n pixmap,\n memset2d_color,\n blit_anti_h_rp,\n blit_rect_rp,\n blit_mask_rp,\n is_mask: false,\n })\n }\n\n pub fn new_mask(pixmap: &'a mut SubPixmapMut<'b>) -> Option {\n let color = Color::WHITE.premultiply();\n\n let memset2d_color = Some(color.to_color_u8());\n\n let blit_anti_h_rp = {\n let mut p = RasterPipelineBuilder::new();\n p.push_uniform_color(color);\n p.push(pipeline::Stage::LoadDestinationU8);\n p.push(pipeline::Stage::Lerp1Float);\n p.push(pipeline::Stage::StoreU8);\n p.compile()\n };\n\n let blit_rect_rp = {\n let mut p = RasterPipelineBuilder::new();\n p.push_uniform_color(color);\n p.push(pipeline::Stage::StoreU8);\n p.compile()\n };\n\n let blit_mask_rp = {\n let mut p = RasterPipelineBuilder::new();\n p.push_uniform_color(color);\n p.push(pipeline::Stage::LoadDestinationU8);\n p.push(pipeline::Stage::LerpU8);\n p.push(pipeline::Stage::StoreU8);\n p.compile()\n };\n\n Some(RasterPipelineBlitter {\n mask: None,\n pixmap_src: PixmapRef::from_bytes(&[0, 0, 0, 0], 1, 1).unwrap(),\n pixmap,\n memset2d_color,\n blit_anti_h_rp,\n blit_rect_rp,\n blit_mask_rp,\n is_mask: true,\n })\n }\n}\n\nimpl Blitter for RasterPipelineBlitter<'_, '_> {\n fn blit_h(&mut self, x: u32, y: u32, width: LengthU32) {\n let r = ScreenIntRect::from_xywh_safe(x, y, width, LENGTH_U32_ONE);\n self.blit_rect(&r);\n }\n\n fn blit_anti_h(&mut self, mut x: u32, y: u32, aa: &mut [AlphaU8], runs: &mut [AlphaRun]) {\n let mask_ctx = self.mask.map(|c| c.mask_ctx()).unwrap_or_default();\n\n let mut aa_offset = 0;\n let mut run_offset = 0;\n let mut run_opt = runs[0];\n while let Some(run) = run_opt {\n let width = LengthU32::from(run);\n\n match aa[aa_offset] {\n ALPHA_U8_TRANSPARENT => {}\n ALPHA_U8_OPAQUE => {\n self.blit_h(x, y, width);\n }\n alpha => {\n self.blit_anti_h_rp.ctx.current_coverage = alpha as f32 * (1.0 / 255.0);\n\n let rect = ScreenIntRect::from_xywh_safe(x, y, width, LENGTH_U32_ONE);\n self.blit_anti_h_rp.run(\n &rect,\n pipeline::AAMaskCtx::default(),\n mask_ctx,\n self.pixmap_src,\n self.pixmap,\n );\n }\n }\n\n x += width.get();\n run_offset += usize::from(run.get());\n aa_offset += usize::from(run.get());\n run_opt = runs[run_offset];\n }\n }\n\n fn blit_v(&mut self, x: u32, y: u32, height: LengthU32, alpha: AlphaU8) {\n let bounds = ScreenIntRect::from_xywh_safe(x, y, LENGTH_U32_ONE, height);\n\n let mask = Mask {\n image: [alpha, alpha],\n bounds,\n row_bytes: 0, // so we reuse the 1 \"row\" for all of height\n };\n\n self.blit_mask(&mask, &bounds);\n }\n\n fn blit_anti_h2(&mut self, x: u32, y: u32, alpha0: AlphaU8, alpha1: AlphaU8) {\n let bounds = ScreenIntRect::from_xywh(x, y, 2, 1).unwrap();\n\n let mask = Mask {\n image: [alpha0, alpha1],\n bounds,\n row_bytes: 2,\n };\n\n self.blit_mask(&mask, &bounds);\n }\n\n fn blit_anti_v2(&mut self, x: u32, y: u32, alpha0: AlphaU8, alpha1: AlphaU8) {\n let bounds = ScreenIntRect::from_xywh(x, y, 1, 2).unwrap();\n\n let mask = Mask {\n image: [alpha0, alpha1],\n bounds,\n row_bytes: 1,\n };\n\n self.blit_mask(&mask, &bounds);\n }\n\n fn blit_rect(&mut self, rect: &ScreenIntRect) {\n if let Some(c) = self.memset2d_color {\n if self.is_mask {\n for y in 0..rect.height() {\n let start = self\n .pixmap\n .offset(rect.x() as usize, (rect.y() + y) as usize);\n let end = start + rect.width() as usize;\n self.pixmap.data[start..end]\n .iter_mut()\n .for_each(|p| *p = c.alpha());\n }\n } else {\n for y in 0..rect.height() {\n let start = self\n .pixmap\n .offset(rect.x() as usize, (rect.y() + y) as usize);\n let end = start + rect.width() as usize;\n self.pixmap.pixels_mut()[start..end]\n .iter_mut()\n .for_each(|p| *p = c);\n }\n }\n\n return;\n }\n\n let mask_ctx = self.mask.map(|c| c.mask_ctx()).unwrap_or_default();\n\n self.blit_rect_rp.run(\n rect,\n pipeline::AAMaskCtx::default(),\n mask_ctx,\n self.pixmap_src,\n self.pixmap,\n );\n }\n\n fn blit_mask(&mut self, mask: &Mask, clip: &ScreenIntRect) {\n let aa_mask_ctx = pipeline::AAMaskCtx {\n pixels: mask.image,\n stride: mask.row_bytes,\n shift: (mask.bounds.left() + mask.bounds.top() * mask.row_bytes) as usize,\n };\n\n let mask_ctx = self.mask.map(|c| c.mask_ctx()).unwrap_or_default();\n\n self.blit_mask_rp\n .run(clip, aa_mask_ctx, mask_ctx, self.pixmap_src, self.pixmap);\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/pipeline/highp.rs", "sha256": "96f9d086b0ff783adce4e4121f00bf31983213a8edaa94fcce4f5d89ed2ea393", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n/*!\nA high precision raster pipeline implementation.\n\nUnlike lowp, this one implements all stages.\n\nJust like Skia, this pipeline is implemented using f32x8.\n\nFor some reason, we are almost 2x slower. Maybe because Skia uses clang's vector extensions\nand we're using a manual implementation.\n*/\n\nuse crate::{PremultipliedColorU8, SpreadMode, PixmapRef};\n\nuse crate::geom::ScreenIntRect;\nuse crate::pixmap::SubPixmapMut;\nuse crate::wide::{f32x8, i32x8, u32x8};\n\npub const STAGE_WIDTH: usize = 8;\n\npub type StageFn = fn(p: &mut Pipeline);\n\npub struct Pipeline<'a, 'b: 'a> {\n index: usize,\n functions: &'a [StageFn],\n pixmap_src: PixmapRef<'a>,\n pixmap_dst: &'a mut SubPixmapMut<'b>,\n ctx: &'a mut super::Context, // TODO: remove mut\n mask_ctx: super::MaskCtx<'a>,\n aa_mask_ctx: super::AAMaskCtx,\n r: f32x8,\n g: f32x8,\n b: f32x8,\n a: f32x8,\n dr: f32x8,\n dg: f32x8,\n db: f32x8,\n da: f32x8,\n tail: usize,\n dx: usize,\n dy: usize,\n}\n\nimpl Pipeline<'_, '_> {\n #[inline(always)]\n fn next_stage(&mut self) {\n let next: fn(&mut Self) = self.functions[self.index];\n self.index += 1;\n next(self);\n }\n}\n\n// Must be in the same order as raster_pipeline::Stage\npub const STAGES: &[StageFn; super::STAGES_COUNT] = &[\n move_source_to_destination,\n move_destination_to_source,\n clamp_0,\n clamp_a,\n premultiply,\n uniform_color,\n seed_shader,\n load_dst,\n store,\n load_dst_u8,\n store_u8,\n gather,\n load_mask_u8,\n mask_u8,\n scale_u8,\n lerp_u8,\n scale_1_float,\n lerp_1_float,\n destination_atop,\n destination_in,\n destination_out,\n destination_over,\n source_atop,\n source_in,\n source_out,\n source_over,\n clear,\n modulate,\n multiply,\n plus,\n screen,\n xor,\n color_burn,\n color_dodge,\n darken,\n difference,\n exclusion,\n hard_light,\n lighten,\n overlay,\n soft_light,\n hue,\n saturation,\n color,\n luminosity,\n source_over_rgba,\n transform,\n reflect,\n repeat,\n bilinear,\n bicubic,\n pad_x1,\n reflect_x1,\n repeat_x1,\n gradient,\n evenly_spaced_2_stop_gradient,\n xy_to_radius,\n xy_to_2pt_conical_focal_on_circle,\n xy_to_2pt_conical_well_behaved,\n xy_to_2pt_conical_greater,\n mask_2pt_conical_degenerates,\n apply_vector_mask,\n];\n\npub fn fn_ptr(f: StageFn) -> *const () {\n f as *const ()\n}\n\n#[inline(never)]\npub fn start(\n functions: &[StageFn],\n functions_tail: &[StageFn],\n rect: &ScreenIntRect,\n aa_mask_ctx: super::AAMaskCtx,\n mask_ctx: super::MaskCtx,\n ctx: &mut super::Context,\n pixmap_src: PixmapRef,\n pixmap_dst: &mut SubPixmapMut,\n) {\n let mut p = Pipeline {\n index: 0,\n functions: &[],\n pixmap_src,\n pixmap_dst,\n mask_ctx,\n aa_mask_ctx,\n ctx,\n r: f32x8::default(),\n g: f32x8::default(),\n b: f32x8::default(),\n a: f32x8::default(),\n dr: f32x8::default(),\n dg: f32x8::default(),\n db: f32x8::default(),\n da: f32x8::default(),\n tail: 0,\n dx: 0,\n dy: 0,\n };\n\n for y in rect.y()..rect.bottom() {\n let mut x = rect.x() as usize;\n let end = rect.right() as usize;\n\n p.functions = functions;\n while x + STAGE_WIDTH <= end {\n p.index = 0;\n p.dx = x;\n p.dy = y as usize;\n p.tail = STAGE_WIDTH;\n p.next_stage();\n x += STAGE_WIDTH;\n }\n\n if x != end {\n p.index = 0;\n p.functions = functions_tail;\n p.dx = x;\n p.dy = y as usize;\n p.tail = end - x;\n p.next_stage();\n }\n }\n}\n\nfn move_source_to_destination(p: &mut Pipeline) {\n p.dr = p.r;\n p.dg = p.g;\n p.db = p.b;\n p.da = p.a;\n\n p.next_stage();\n}\n\nfn premultiply(p: &mut Pipeline) {\n p.r *= p.a;\n p.g *= p.a;\n p.b *= p.a;\n\n p.next_stage();\n}\n\nfn move_destination_to_source(p: &mut Pipeline) {\n p.r = p.dr;\n p.g = p.dg;\n p.b = p.db;\n p.a = p.da;\n\n p.next_stage();\n}\n\nfn clamp_0(p: &mut Pipeline) {\n p.r = p.r.max(f32x8::default());\n p.g = p.g.max(f32x8::default());\n p.b = p.b.max(f32x8::default());\n p.a = p.a.max(f32x8::default());\n\n p.next_stage();\n}\n\nfn clamp_a(p: &mut Pipeline) {\n p.r = p.r.min(f32x8::splat(1.0));\n p.g = p.g.min(f32x8::splat(1.0));\n p.b = p.b.min(f32x8::splat(1.0));\n p.a = p.a.min(f32x8::splat(1.0));\n\n p.next_stage();\n}\n\nfn uniform_color(p: &mut Pipeline) {\n let ctx = &p.ctx.uniform_color;\n p.r = f32x8::splat(ctx.r);\n p.g = f32x8::splat(ctx.g);\n p.b = f32x8::splat(ctx.b);\n p.a = f32x8::splat(ctx.a);\n\n p.next_stage();\n}\n\nfn seed_shader(p: &mut Pipeline) {\n let iota = f32x8::from([0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5]);\n\n p.r = f32x8::splat(p.dx as f32) + iota;\n p.g = f32x8::splat(p.dy as f32 + 0.5);\n p.b = f32x8::splat(1.0);\n p.a = f32x8::default();\n\n p.dr = f32x8::default();\n p.dg = f32x8::default();\n p.db = f32x8::default();\n p.da = f32x8::default();\n\n p.next_stage();\n}\n\npub fn load_dst(p: &mut Pipeline) {\n load_8888(p.pixmap_dst.slice4_at_xy(p.dx, p.dy), &mut p.dr, &mut p.dg, &mut p.db, &mut p.da);\n p.next_stage();\n}\n\npub fn load_dst_tail(p: &mut Pipeline) {\n load_8888_tail(p.tail, p.pixmap_dst.slice_at_xy(p.dx, p.dy), &mut p.dr, &mut p.dg, &mut p.db, &mut p.da);\n p.next_stage();\n}\n\npub fn store(p: &mut Pipeline) {\n store_8888(&p.r, &p.g, &p.b, &p.a, p.pixmap_dst.slice4_at_xy(p.dx, p.dy));\n p.next_stage();\n}\n\npub fn store_tail(p: &mut Pipeline) {\n store_8888_tail(&p.r, &p.g, &p.b, &p.a, p.tail, p.pixmap_dst.slice_at_xy(p.dx, p.dy));\n p.next_stage();\n}\n\n// Currently, all mask/A8 pixmaps are handled by lowp.\npub fn load_dst_u8(_: &mut Pipeline) {\n // unreachable\n}\n\npub fn load_dst_u8_tail(_: &mut Pipeline) {\n // unreachable\n}\n\npub fn store_u8(_: &mut Pipeline) {\n // unreachable\n}\n\npub fn store_u8_tail(_: &mut Pipeline) {\n // unreachable\n}\n\npub fn gather(p: &mut Pipeline) {\n let ix = gather_ix(p.pixmap_src, p.r, p.g);\n load_8888(&p.pixmap_src.gather(ix), &mut p.r, &mut p.g, &mut p.b, &mut p.a);\n\n p.next_stage();\n}\n\n#[inline(always)]\nfn gather_ix(pixmap: PixmapRef, mut x: f32x8, mut y: f32x8) -> u32x8 {\n // Exclusive -> inclusive.\n let w = ulp_sub(pixmap.width() as f32);\n let h = ulp_sub(pixmap.height() as f32);\n x = x.max(f32x8::default()).min(f32x8::splat(w));\n y = y.max(f32x8::default()).min(f32x8::splat(h));\n\n (y.trunc_int() * i32x8::splat(pixmap.width() as i32) + x.trunc_int()).to_u32x8_bitcast()\n}\n\n#[inline(always)]\nfn ulp_sub(v: f32) -> f32 {\n // Somewhat similar to v - f32::EPSILON\n bytemuck::cast::(bytemuck::cast::(v) - 1)\n}\n\nfn load_mask_u8(_: &mut Pipeline) {\n // unreachable\n}\n\nfn mask_u8(p: &mut Pipeline) {\n let offset = p.mask_ctx.offset(p.dx, p.dy);\n let mut c = [0.0; 8];\n for i in 0..p.tail {\n c[i] = p.mask_ctx.data[offset + i] as f32;\n }\n let c = f32x8::from(c) / f32x8::splat(255.0);\n\n if c == f32x8::default() {\n return;\n }\n\n p.r *= c;\n p.g *= c;\n p.b *= c;\n p.a *= c;\n\n p.next_stage();\n}\n\nfn scale_u8(p: &mut Pipeline) {\n // Load u8xTail and cast it to f32x8.\n let data = p.aa_mask_ctx.copy_at_xy(p.dx, p.dy, p.tail);\n let c = f32x8::from([data[0] as f32, data[1] as f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]);\n let c = c / f32x8::splat(255.0);\n\n p.r *= c;\n p.g *= c;\n p.b *= c;\n p.a *= c;\n\n p.next_stage();\n}\n\nfn lerp_u8(p: &mut Pipeline) {\n // Load u8xTail and cast it to f32x8.\n let data = p.aa_mask_ctx.copy_at_xy(p.dx, p.dy, p.tail);\n let c = f32x8::from([data[0] as f32, data[1] as f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]);\n let c = c / f32x8::splat(255.0);\n\n p.r = lerp(p.dr, p.r, c);\n p.g = lerp(p.dg, p.g, c);\n p.b = lerp(p.db, p.b, c);\n p.a = lerp(p.da, p.a, c);\n\n p.next_stage();\n}\n\nfn scale_1_float(p: &mut Pipeline) {\n let c = f32x8::splat(p.ctx.current_coverage);\n p.r *= c;\n p.g *= c;\n p.b *= c;\n p.a *= c;\n\n p.next_stage();\n}\n\nfn lerp_1_float(p: &mut Pipeline) {\n let c = f32x8::splat(p.ctx.current_coverage);\n p.r = lerp(p.dr, p.r, c);\n p.g = lerp(p.dg, p.g, c);\n p.b = lerp(p.db, p.b, c);\n p.a = lerp(p.da, p.a, c);\n\n p.next_stage();\n}\n\nmacro_rules! blend_fn {\n ($name:ident, $f:expr) => {\n fn $name(p: &mut Pipeline) {\n p.r = $f(p.r, p.dr, p.a, p.da);\n p.g = $f(p.g, p.dg, p.a, p.da);\n p.b = $f(p.b, p.db, p.a, p.da);\n p.a = $f(p.a, p.da, p.a, p.da);\n\n p.next_stage();\n }\n };\n}\n\nblend_fn!(clear, |_, _, _, _| f32x8::default());\nblend_fn!(source_atop, |s, d, sa, da| s * da + d * inv(sa));\nblend_fn!(destination_atop, |s, d, sa, da| d * sa + s * inv(da));\nblend_fn!(source_in, |s, _, _, da| s * da);\nblend_fn!(destination_in, |_, d, sa, _| d * sa);\nblend_fn!(source_out, |s, _, _, da| s * inv(da));\nblend_fn!(destination_out, |_, d, sa, _| d * inv(sa));\nblend_fn!(source_over, |s, d, sa, _| mad(d, inv(sa), s));\nblend_fn!(destination_over, |s, d, _, da| mad(s, inv(da), d));\nblend_fn!(modulate, |s, d, _, _| s * d);\nblend_fn!(multiply, |s, d, sa, da| s * inv(da) + d * inv(sa) + s * d);\nblend_fn!(screen, |s, d, _, _| s + d - s * d);\nblend_fn!(xor, |s, d, sa, da| s * inv(da) + d * inv(sa));\n\n// Wants a type for some reason.\nblend_fn!(plus, |s: f32x8, d: f32x8, _, _| (s + d).min(f32x8::splat(1.0)));\n\nmacro_rules! blend_fn2 {\n ($name:ident, $f:expr) => {\n fn $name(p: &mut Pipeline) {\n // The same logic applied to color, and source_over for alpha.\n p.r = $f(p.r, p.dr, p.a, p.da);\n p.g = $f(p.g, p.dg, p.a, p.da);\n p.b = $f(p.b, p.db, p.a, p.da);\n p.a = mad(p.da, inv(p.a), p.a);\n\n p.next_stage();\n }\n };\n}\n\nblend_fn2!(darken, |s: f32x8, d, sa, da: f32x8| s + d - (s * da).max(d * sa));\nblend_fn2!(lighten, |s: f32x8, d, sa, da: f32x8| s + d - (s * da).min(d * sa));\nblend_fn2!(difference, |s: f32x8, d, sa, da: f32x8| s + d - two((s * da).min(d * sa)));\nblend_fn2!(exclusion, |s: f32x8, d, _, _| s + d - two(s * d));\n\nblend_fn2!(color_burn, |s: f32x8, d: f32x8, sa: f32x8, da: f32x8|\n d.cmp_eq(da).blend(\n d + s * inv(da),\n s.cmp_eq(f32x8::default()).blend(\n d * inv(sa),\n sa * (da - da.min((da - d) * sa * s.recip_fast())) + s * inv(da) + d * inv(sa)\n )\n )\n);\n\nblend_fn2!(color_dodge, |s: f32x8, d: f32x8, sa: f32x8, da: f32x8|\n d.cmp_eq(f32x8::default()).blend(\n s * inv(da),\n s.cmp_eq(sa).blend(\n s + d * inv(sa),\n sa * da.min((d * sa) * (sa - s).recip_fast()) + s * inv(da) + d * inv(sa)\n )\n )\n);\n\nblend_fn2!(hard_light, |s: f32x8, d: f32x8, sa, da|\n s * inv(da) + d * inv(sa) + two(s).cmp_le(sa).blend(\n two(s * d),\n sa * da - two((da - d) * (sa - s))\n )\n);\n\nblend_fn2!(overlay, |s: f32x8, d: f32x8, sa, da|\n s * inv(da) + d * inv(sa) + two(d).cmp_le(da).blend(\n two(s * d),\n sa * da - two((da - d) * (sa - s))\n )\n);\n\nblend_fn2!(soft_light, |s: f32x8, d: f32x8, sa: f32x8, da: f32x8| {\n let m = da.cmp_gt(f32x8::default()).blend(d / da, f32x8::default());\n let s2 = two(s);\n let m4 = two(two(m));\n\n // The logic forks three ways:\n // 1. dark src?\n // 2. light src, dark dst?\n // 3. light src, light dst?\n let dark_src = d * (sa + (s2 - sa) * (f32x8::splat(1.0) - m));\n let dark_dst = (m4 * m4 + m4) * (m - f32x8::splat(1.0)) + f32x8::splat(7.0) * m;\n let lite_dst = m.sqrt() - m;\n let lite_src = d * sa + da * (s2 - sa)\n * two(two(d)).cmp_le(da).blend(dark_dst, lite_dst); // 2 or 3?\n\n s * inv(da) + d * inv(sa) + s2.cmp_le(sa).blend(dark_src, lite_src) // 1 or (2 or 3)?\n});\n\n// We're basing our implementation of non-separable blend modes on\n// https://www.w3.org/TR/compositing-1/#blendingnonseparable.\n// and\n// https://www.khronos.org/registry/OpenGL/specs/es/3.2/es_spec_3.2.pdf\n// They're equivalent, but ES' math has been better simplified.\n//\n// Anything extra we add beyond that is to make the math work with premul inputs.\n\nmacro_rules! blend_fn3 {\n ($name:ident, $f:expr) => {\n fn $name(p: &mut Pipeline) {\n let (tr, tg, tb, ta) = $f(p.r, p.g, p.b, p.a, p.dr, p.dg, p.db, p.da);\n p.r = tr;\n p.g = tg;\n p.b = tb;\n p.a = ta;\n\n p.next_stage();\n }\n };\n}\n\nblend_fn3!(hue, hue_k);\n\n#[inline(always)]\nfn hue_k(\n r: f32x8, g: f32x8, b: f32x8, a: f32x8,\n dr: f32x8, dg: f32x8, db: f32x8, da: f32x8,\n) -> (f32x8, f32x8, f32x8, f32x8) {\n let rr = &mut (r * a);\n let gg = &mut (g * a);\n let bb = &mut (b * a);\n\n set_sat(rr, gg, bb, sat(dr, dg, db) * a);\n set_lum(rr, gg, bb, lum(dr, dg, db) * a);\n clip_color(rr, gg, bb, a * da);\n\n let r = r * inv(da) + dr * inv(a) + *rr;\n let g = g * inv(da) + dg * inv(a) + *gg;\n let b = b * inv(da) + db * inv(a) + *bb;\n let a = a + da - a * da;\n\n (r, g, b, a)\n}\n\nblend_fn3!(saturation, saturation_k);\n\n#[inline(always)]\nfn saturation_k(\n r: f32x8, g: f32x8, b: f32x8, a: f32x8,\n dr: f32x8, dg: f32x8, db: f32x8, da: f32x8,\n) -> (f32x8, f32x8, f32x8, f32x8) {\n let rr = &mut (dr * a);\n let gg = &mut (dg * a);\n let bb = &mut (db * a);\n\n set_sat(rr, gg, bb, sat(r, g, b) * da);\n set_lum(rr, gg, bb, lum(dr, dg, db) * a); // (This is not redundant.)\n clip_color(rr, gg, bb, a * da);\n\n let r = r * inv(da) + dr * inv(a) + *rr;\n let g = g * inv(da) + dg * inv(a) + *gg;\n let b = b * inv(da) + db * inv(a) + *bb;\n let a = a + da - a * da;\n\n (r, g, b, a)\n}\n\nblend_fn3!(color, color_k);\n\n#[inline(always)]\nfn color_k(\n r: f32x8, g: f32x8, b: f32x8, a: f32x8,\n dr: f32x8, dg: f32x8, db: f32x8, da: f32x8,\n) -> (f32x8, f32x8, f32x8, f32x8) {\n let rr = &mut (r * da);\n let gg = &mut (g * da);\n let bb = &mut (b * da);\n\n set_lum(rr, gg, bb, lum(dr, dg, db) * a);\n clip_color(rr, gg, bb, a * da);\n\n let r = r * inv(da) + dr * inv(a) + *rr;\n let g = g * inv(da) + dg * inv(a) + *gg;\n let b = b * inv(da) + db * inv(a) + *bb;\n let a = a + da - a * da;\n\n (r, g, b, a)\n}\n\nblend_fn3!(luminosity, luminosity_k);\n\n#[inline(always)]\nfn luminosity_k(\n r: f32x8, g: f32x8, b: f32x8, a: f32x8,\n dr: f32x8, dg: f32x8, db: f32x8, da: f32x8,\n) -> (f32x8, f32x8, f32x8, f32x8) {\n let rr = &mut (dr * a);\n let gg = &mut (dg * a);\n let bb = &mut (db * a);\n\n set_lum(rr, gg, bb, lum(r, g, b) * da);\n clip_color(rr, gg, bb, a * da);\n\n let r = r * inv(da) + dr * inv(a) + *rr;\n let g = g * inv(da) + dg * inv(a) + *gg;\n let b = b * inv(da) + db * inv(a) + *bb;\n let a = a + da - a * da;\n\n (r, g, b, a)\n}\n\n#[inline(always)]\nfn sat(r: f32x8, g: f32x8, b: f32x8) -> f32x8 {\n r.max(g.max(b)) - r.min(g.min(b))\n}\n\n#[inline(always)]\nfn lum(r: f32x8, g: f32x8, b: f32x8) -> f32x8 {\n r * f32x8::splat(0.30) + g * f32x8::splat(0.59) + b * f32x8::splat(0.11)\n}\n\n#[inline(always)]\nfn set_sat(r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, s: f32x8) {\n let mn = r.min(g.min(*b));\n let mx = r.max(g.max(*b));\n let sat = mx - mn;\n\n // Map min channel to 0, max channel to s, and scale the middle proportionally.\n let scale = |c| sat.cmp_eq(f32x8::default())\n .blend(f32x8::default(), (c - mn) * s / sat);\n\n *r = scale(*r);\n *g = scale(*g);\n *b = scale(*b);\n}\n\n#[inline(always)]\nfn set_lum(r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, l: f32x8) {\n let diff = l - lum(*r, *g, *b);\n *r += diff;\n *g += diff;\n *b += diff;\n}\n\n#[inline(always)]\nfn clip_color(r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, a: f32x8) {\n let mn = r.min(g.min(*b));\n let mx = r.max(g.max(*b));\n let l = lum(*r, *g, *b);\n\n let clip = |mut c| {\n c = mx.cmp_ge(f32x8::default()).blend(c, l + (c - l) * l / (l - mn));\n c = mx.cmp_gt(a).blend(l + (c - l) * (a - l) / (mx - l), c);\n c = c.max(f32x8::default()); // Sometimes without this we may dip just a little negative.\n c\n };\n\n *r = clip(*r);\n *g = clip(*g);\n *b = clip(*b);\n}\n\npub fn source_over_rgba(p: &mut Pipeline) {\n let pixels = p.pixmap_dst.slice4_at_xy(p.dx, p.dy);\n load_8888(pixels, &mut p.dr, &mut p.dg, &mut p.db, &mut p.da);\n p.r = mad(p.dr, inv(p.a), p.r);\n p.g = mad(p.dg, inv(p.a), p.g);\n p.b = mad(p.db, inv(p.a), p.b);\n p.a = mad(p.da, inv(p.a), p.a);\n store_8888(&p.r, &p.g, &p.b, &p.a, pixels);\n\n p.next_stage();\n}\n\npub fn source_over_rgba_tail(p: &mut Pipeline) {\n let pixels = p.pixmap_dst.slice_at_xy(p.dx, p.dy);\n load_8888_tail(p.tail, pixels, &mut p.dr, &mut p.dg, &mut p.db, &mut p.da);\n p.r = mad(p.dr, inv(p.a), p.r);\n p.g = mad(p.dg, inv(p.a), p.g);\n p.b = mad(p.db, inv(p.a), p.b);\n p.a = mad(p.da, inv(p.a), p.a);\n store_8888_tail(&p.r, &p.g, &p.b, &p.a, p.tail, pixels);\n\n p.next_stage();\n}\n\nfn transform(p: &mut Pipeline) {\n let ts = &p.ctx.transform;\n\n let tr = mad(p.r, f32x8::splat(ts.sx), mad(p.g, f32x8::splat(ts.kx), f32x8::splat(ts.tx)));\n let tg = mad(p.r, f32x8::splat(ts.ky), mad(p.g, f32x8::splat(ts.sy), f32x8::splat(ts.ty)));\n p.r = tr;\n p.g = tg;\n\n p.next_stage();\n}\n\n// Tile x or y to [0,limit) == [0,limit - 1 ulp] (think, sampling from images).\n// The gather stages will hard clamp the output of these stages to [0,limit)...\n// we just need to do the basic repeat or mirroring.\n\nfn reflect(p: &mut Pipeline) {\n let ctx = &p.ctx.limit_x;\n p.r = exclusive_reflect(p.r, ctx.scale, ctx.inv_scale);\n\n let ctx = &p.ctx.limit_y;\n p.g = exclusive_reflect(p.g, ctx.scale, ctx.inv_scale);\n\n p.next_stage();\n}\n\n#[inline(always)]\nfn exclusive_reflect(v: f32x8, limit: f32, inv_limit: f32) -> f32x8 {\n let limit = f32x8::splat(limit);\n let inv_limit = f32x8::splat(inv_limit);\n ((v - limit) - (limit + limit)\n * ((v - limit) * (inv_limit * f32x8::splat(0.5))).floor() - limit).abs()\n}\n\nfn repeat(p: &mut Pipeline) {\n let ctx = &p.ctx.limit_x;\n p.r = exclusive_repeat(p.r, ctx.scale, ctx.inv_scale);\n\n let ctx = &p.ctx.limit_y;\n p.g = exclusive_repeat(p.g, ctx.scale, ctx.inv_scale);\n\n p.next_stage();\n}\n\n#[inline(always)]\nfn exclusive_repeat(v: f32x8, limit: f32, inv_limit: f32) -> f32x8 {\n v - (v * f32x8::splat(inv_limit)).floor() * f32x8::splat(limit)\n}\n\nfn bilinear(p: &mut Pipeline) {\n let x = p.r;\n let fx = (x + f32x8::splat(0.5)).fract();\n let y = p.g;\n let fy = (y + f32x8::splat(0.5)).fract();\n let one = f32x8::splat(1.0);\n let wx = [one - fx, fx];\n let wy = [one - fy, fy];\n\n sampler_2x2(p.pixmap_src, &p.ctx.sampler, x, y, &wx, &wy, &mut p.r, &mut p.g, &mut p.b, &mut p.a);\n\n p.next_stage();\n}\n\nfn bicubic(p: &mut Pipeline) {\n let x = p.r;\n let fx = (x + f32x8::splat(0.5)).fract();\n let y = p.g;\n let fy = (y + f32x8::splat(0.5)).fract();\n let one = f32x8::splat(1.0);\n let wx = [bicubic_far(one - fx), bicubic_near(one - fx), bicubic_near(fx), bicubic_far(fx)];\n let wy = [bicubic_far(one - fy), bicubic_near(one - fy), bicubic_near(fy), bicubic_far(fy)];\n\n sampler_4x4(p.pixmap_src, &p.ctx.sampler, x, y, &wx, &wy, &mut p.r, &mut p.g, &mut p.b, &mut p.a);\n\n p.next_stage();\n}\n\n// In bicubic interpolation, the 16 pixels and +/- 0.5 and +/- 1.5 offsets from the sample\n// pixel center are combined with a non-uniform cubic filter, with higher values near the center.\n//\n// We break this function into two parts, one for near 0.5 offsets and one for far 1.5 offsets.\n\n#[inline(always)]\nfn bicubic_near(t: f32x8) -> f32x8 {\n // 1/18 + 9/18t + 27/18t^2 - 21/18t^3 == t ( t ( -21/18t + 27/18) + 9/18) + 1/18\n mad(\n t,\n mad(t,\n mad(\n f32x8::splat(-21.0/18.0),\n t,\n f32x8::splat(27.0/18.0),\n ),\n f32x8::splat(9.0/18.0),\n ),\n f32x8::splat(1.0/18.0),\n )\n}\n\n#[inline(always)]\nfn bicubic_far(t: f32x8) -> f32x8 {\n // 0/18 + 0/18*t - 6/18t^2 + 7/18t^3 == t^2 (7/18t - 6/18)\n (t * t) * mad(f32x8::splat(7.0/18.0), t, f32x8::splat(-6.0/18.0))\n}\n\n#[inline(always)]\nfn sampler_2x2(\n pixmap: PixmapRef,\n ctx: &super::SamplerCtx,\n cx: f32x8, cy: f32x8,\n wx: &[f32x8; 2], wy: &[f32x8; 2],\n r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, a: &mut f32x8,\n) {\n *r = f32x8::default();\n *g = f32x8::default();\n *b = f32x8::default();\n *a = f32x8::default();\n\n let one = f32x8::splat(1.0);\n let start = -0.5;\n let mut y = cy + f32x8::splat(start);\n for j in 0..2 {\n let mut x = cx + f32x8::splat(start);\n for i in 0..2 {\n let mut rr = f32x8::default();\n let mut gg = f32x8::default();\n let mut bb = f32x8::default();\n let mut aa = f32x8::default();\n sample(pixmap, ctx, x,y, &mut rr, &mut gg, &mut bb, &mut aa);\n\n let w = wx[i] * wy[j];\n *r = mad(w, rr, *r);\n *g = mad(w, gg, *g);\n *b = mad(w, bb, *b);\n *a = mad(w, aa, *a);\n\n x += one;\n }\n\n y += one;\n }\n}\n\n#[inline(always)]\nfn sampler_4x4(\n pixmap: PixmapRef,\n ctx: &super::SamplerCtx,\n cx: f32x8, cy: f32x8,\n wx: &[f32x8; 4], wy: &[f32x8; 4],\n r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, a: &mut f32x8,\n) {\n *r = f32x8::default();\n *g = f32x8::default();\n *b = f32x8::default();\n *a = f32x8::default();\n\n let one = f32x8::splat(1.0);\n let start = -1.5;\n let mut y = cy + f32x8::splat(start);\n for j in 0..4 {\n let mut x = cx + f32x8::splat(start);\n for i in 0..4 {\n let mut rr = f32x8::default();\n let mut gg = f32x8::default();\n let mut bb = f32x8::default();\n let mut aa = f32x8::default();\n sample(pixmap, ctx, x,y, &mut rr, &mut gg, &mut bb, &mut aa);\n\n let w = wx[i] * wy[j];\n *r = mad(w, rr, *r);\n *g = mad(w, gg, *g);\n *b = mad(w, bb, *b);\n *a = mad(w, aa, *a);\n\n x += one;\n }\n\n y += one;\n }\n}\n\n#[inline(always)]\nfn sample(\n pixmap: PixmapRef, ctx: &super::SamplerCtx, mut x: f32x8, mut y: f32x8,\n r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, a: &mut f32x8,\n) {\n x = tile(x, ctx.spread_mode, pixmap.width() as f32, ctx.inv_width);\n y = tile(y, ctx.spread_mode, pixmap.height() as f32, ctx.inv_height);\n\n let ix = gather_ix(pixmap, x, y);\n load_8888(&pixmap.gather(ix), r, g, b, a);\n}\n\n#[inline(always)]\nfn tile(v: f32x8, mode: SpreadMode, limit: f32, inv_limit: f32) -> f32x8 {\n match mode {\n SpreadMode::Pad => v,\n SpreadMode::Repeat => exclusive_repeat(v, limit, inv_limit),\n SpreadMode::Reflect => exclusive_reflect(v, limit, inv_limit),\n }\n}\n\nfn pad_x1(p: &mut Pipeline) {\n p.r = p.r.normalize();\n\n p.next_stage();\n}\n\nfn reflect_x1(p: &mut Pipeline) {\n p.r = (\n (p.r - f32x8::splat(1.0))\n - two(((p.r - f32x8::splat(1.0)) * f32x8::splat(0.5)).floor())\n - f32x8::splat(1.0)\n ).abs().normalize();\n\n p.next_stage();\n}\n\nfn repeat_x1(p: &mut Pipeline) {\n p.r = (p.r - p.r.floor()).normalize();\n\n p.next_stage();\n}\n\nfn gradient(p: &mut Pipeline) {\n let ctx = &p.ctx.gradient;\n\n // N.B. The loop starts at 1 because idx 0 is the color to use before the first stop.\n let t: [f32; 8] = p.r.into();\n let mut idx = u32x8::default();\n for i in 1..ctx.len {\n let tt = ctx.t_values[i].get();\n let n: u32x8 = bytemuck::cast([\n (t[0] >= tt) as u32,\n (t[1] >= tt) as u32,\n (t[2] >= tt) as u32,\n (t[3] >= tt) as u32,\n (t[4] >= tt) as u32,\n (t[5] >= tt) as u32,\n (t[6] >= tt) as u32,\n (t[7] >= tt) as u32,\n ]);\n idx = idx + n;\n }\n gradient_lookup(ctx, &idx, p.r, &mut p.r, &mut p.g, &mut p.b, &mut p.a);\n\n p.next_stage();\n}\n\nfn gradient_lookup(\n ctx: &super::GradientCtx, idx: &u32x8, t: f32x8,\n r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, a: &mut f32x8,\n) {\n let idx: [u32; 8] = bytemuck::cast(*idx);\n\n macro_rules! gather {\n ($d:expr, $c:ident) => {\n // Surprisingly, but bound checking doesn't affect the performance.\n // And since `idx` can contain any number, we should leave it in place.\n f32x8::from([\n $d[idx[0] as usize].$c,\n $d[idx[1] as usize].$c,\n $d[idx[2] as usize].$c,\n $d[idx[3] as usize].$c,\n $d[idx[4] as usize].$c,\n $d[idx[5] as usize].$c,\n $d[idx[6] as usize].$c,\n $d[idx[7] as usize].$c,\n ])\n };\n }\n\n let fr = gather!(&ctx.factors, r);\n let fg = gather!(&ctx.factors, g);\n let fb = gather!(&ctx.factors, b);\n let fa = gather!(&ctx.factors, a);\n\n let br = gather!(&ctx.biases, r);\n let bg = gather!(&ctx.biases, g);\n let bb = gather!(&ctx.biases, b);\n let ba = gather!(&ctx.biases, a);\n\n *r = mad(t, fr, br);\n *g = mad(t, fg, bg);\n *b = mad(t, fb, bb);\n *a = mad(t, fa, ba);\n}\n\nfn evenly_spaced_2_stop_gradient(p: &mut Pipeline) {\n let ctx = &p.ctx.evenly_spaced_2_stop_gradient;\n\n let t = p.r;\n p.r = mad(t, f32x8::splat(ctx.factor.r), f32x8::splat(ctx.bias.r));\n p.g = mad(t, f32x8::splat(ctx.factor.g), f32x8::splat(ctx.bias.g));\n p.b = mad(t, f32x8::splat(ctx.factor.b), f32x8::splat(ctx.bias.b));\n p.a = mad(t, f32x8::splat(ctx.factor.a), f32x8::splat(ctx.bias.a));\n\n p.next_stage();\n}\n\nfn xy_to_radius(p: &mut Pipeline) {\n let x2 = p.r * p.r;\n let y2 = p.g * p.g;\n p.r = (x2 + y2).sqrt();\n\n p.next_stage();\n}\n\nfn xy_to_2pt_conical_focal_on_circle(p: &mut Pipeline) {\n let x = p.r;\n let y = p.g;\n p.r = x + y * y / x;\n\n p.next_stage();\n}\n\nfn xy_to_2pt_conical_well_behaved(p: &mut Pipeline) {\n let ctx = &p.ctx.two_point_conical_gradient;\n\n let x = p.r;\n let y = p.g;\n p.r = (x * x + y * y).sqrt() - x * f32x8::splat(ctx.p0);\n\n p.next_stage();\n}\n\nfn xy_to_2pt_conical_greater(p: &mut Pipeline) {\n let ctx = &p.ctx.two_point_conical_gradient;\n\n let x = p.r;\n let y = p.g;\n p.r = (x * x - y * y).sqrt() - x * f32x8::splat(ctx.p0);\n\n p.next_stage();\n}\n\nfn mask_2pt_conical_degenerates(p: &mut Pipeline) {\n let ctx = &mut p.ctx.two_point_conical_gradient;\n\n let t = p.r;\n let is_degenerate = t.cmp_le(f32x8::default()) | t.cmp_ne(t);\n p.r = is_degenerate.blend(f32x8::default(), t);\n\n let is_not_degenerate = !is_degenerate.to_u32x8_bitcast();\n let is_not_degenerate: [u32; 8] = bytemuck::cast(is_not_degenerate);\n ctx.mask = bytemuck::cast([\n if is_not_degenerate[0] != 0 { !0 } else { 0 },\n if is_not_degenerate[1] != 0 { !0 } else { 0 },\n if is_not_degenerate[2] != 0 { !0 } else { 0 },\n if is_not_degenerate[3] != 0 { !0 } else { 0 },\n if is_not_degenerate[4] != 0 { !0 } else { 0 },\n if is_not_degenerate[5] != 0 { !0 } else { 0 },\n if is_not_degenerate[6] != 0 { !0 } else { 0 },\n if is_not_degenerate[7] != 0 { !0 } else { 0 },\n ]);\n\n p.next_stage();\n}\n\nfn apply_vector_mask(p: &mut Pipeline) {\n let ctx = &p.ctx.two_point_conical_gradient;\n\n p.r = (p.r.to_u32x8_bitcast() & ctx.mask).to_f32x8_bitcast();\n p.g = (p.g.to_u32x8_bitcast() & ctx.mask).to_f32x8_bitcast();\n p.b = (p.b.to_u32x8_bitcast() & ctx.mask).to_f32x8_bitcast();\n p.a = (p.a.to_u32x8_bitcast() & ctx.mask).to_f32x8_bitcast();\n\n p.next_stage();\n}\n\npub fn just_return(_: &mut Pipeline) {\n // Ends the loop.\n}\n\n#[inline(always)]\nfn load_8888(\n data: &[PremultipliedColorU8; STAGE_WIDTH],\n r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, a: &mut f32x8,\n) {\n // Surprisingly, `f32 * FACTOR` is way faster than `f32x8 * f32x8::splat(FACTOR)`.\n\n const FACTOR: f32 = 1.0 / 255.0;\n\n *r = f32x8::from([\n data[0].red() as f32 * FACTOR, data[1].red() as f32 * FACTOR,\n data[2].red() as f32 * FACTOR, data[3].red() as f32 * FACTOR,\n data[4].red() as f32 * FACTOR, data[5].red() as f32 * FACTOR,\n data[6].red() as f32 * FACTOR, data[7].red() as f32 * FACTOR,\n ]);\n\n *g = f32x8::from([\n data[0].green() as f32 * FACTOR, data[1].green() as f32 * FACTOR,\n data[2].green() as f32 * FACTOR, data[3].green() as f32 * FACTOR,\n data[4].green() as f32 * FACTOR, data[5].green() as f32 * FACTOR,\n data[6].green() as f32 * FACTOR, data[7].green() as f32 * FACTOR,\n ]);\n\n *b = f32x8::from([\n data[0].blue() as f32 * FACTOR, data[1].blue() as f32 * FACTOR,\n data[2].blue() as f32 * FACTOR, data[3].blue() as f32 * FACTOR,\n data[4].blue() as f32 * FACTOR, data[5].blue() as f32 * FACTOR,\n data[6].blue() as f32 * FACTOR, data[7].blue() as f32 * FACTOR,\n ]);\n\n *a = f32x8::from([\n data[0].alpha() as f32 * FACTOR, data[1].alpha() as f32 * FACTOR,\n data[2].alpha() as f32 * FACTOR, data[3].alpha() as f32 * FACTOR,\n data[4].alpha() as f32 * FACTOR, data[5].alpha() as f32 * FACTOR,\n data[6].alpha() as f32 * FACTOR, data[7].alpha() as f32 * FACTOR,\n ]);\n}\n\n#[inline(always)]\nfn load_8888_tail(\n tail: usize, data: &[PremultipliedColorU8],\n r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, a: &mut f32x8,\n) {\n // Fill a dummy array with `tail` values. `tail` is always in a 1..STAGE_WIDTH-1 range.\n // This way we can reuse the `load_8888_` method and remove any branches.\n let mut tmp = [PremultipliedColorU8::TRANSPARENT; STAGE_WIDTH];\n tmp[0..tail].copy_from_slice(&data[0..tail]);\n load_8888(&tmp, r, g, b, a);\n}\n\n#[inline(always)]\nfn store_8888(\n r: &f32x8, g: &f32x8, b: &f32x8, a: &f32x8,\n data: &mut [PremultipliedColorU8; STAGE_WIDTH],\n) {\n let r: [i32; 8] = unnorm(r).into();\n let g: [i32; 8] = unnorm(g).into();\n let b: [i32; 8] = unnorm(b).into();\n let a: [i32; 8] = unnorm(a).into();\n\n let conv = |rr, gg, bb, aa|\n PremultipliedColorU8::from_rgba_unchecked(rr as u8, gg as u8, bb as u8, aa as u8);\n\n data[0] = conv(r[0], g[0], b[0], a[0]);\n data[1] = conv(r[1], g[1], b[1], a[1]);\n data[2] = conv(r[2], g[2], b[2], a[2]);\n data[3] = conv(r[3], g[3], b[3], a[3]);\n data[4] = conv(r[4], g[4], b[4], a[4]);\n data[5] = conv(r[5], g[5], b[5], a[5]);\n data[6] = conv(r[6], g[6], b[6], a[6]);\n data[7] = conv(r[7], g[7], b[7], a[7]);\n}\n\n#[inline(always)]\nfn store_8888_tail(\n r: &f32x8, g: &f32x8, b: &f32x8, a: &f32x8,\n tail: usize, data: &mut [PremultipliedColorU8],\n) {\n let r: [i32; 8] = unnorm(r).into();\n let g: [i32; 8] = unnorm(g).into();\n let b: [i32; 8] = unnorm(b).into();\n let a: [i32; 8] = unnorm(a).into();\n\n // This is better than `for i in 0..tail`, because this way the compiler\n // knows that we have only 4 steps and slices access is guarantee to be valid.\n // This removes bounds checking and a possible panic call.\n for i in 0..STAGE_WIDTH {\n data[i] = PremultipliedColorU8::from_rgba_unchecked(\n r[i] as u8, g[i] as u8, b[i] as u8, a[i] as u8,\n );\n\n if i + 1 == tail {\n break;\n }\n }\n}\n\n#[inline(always)]\nfn unnorm(v: &f32x8) -> i32x8 {\n (v.max(f32x8::default()).min(f32x8::splat(1.0)) * f32x8::splat(255.0)).round_int()\n}\n\n#[inline(always)]\nfn inv(v: f32x8) -> f32x8 {\n f32x8::splat(1.0) - v\n}\n\n#[inline(always)]\nfn two(v: f32x8) -> f32x8 {\n v + v\n}\n\n#[inline(always)]\nfn mad(f: f32x8, m: f32x8, a: f32x8) -> f32x8 {\n f * m + a\n}\n\n#[inline(always)]\nfn lerp(from: f32x8, to: f32x8, t: f32x8) -> f32x8 {\n mad(to - from, t, from)\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/pipeline/lowp.rs", "sha256": "878f4d05da7a5b101ff5bb2af87b5ccc208b94fc85c108644b95654198d66ab8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2018 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n/*!\nA low precision raster pipeline implementation.\n\nA lowp pipeline uses u16 instead of f32 for math.\nBecause of that, it doesn't implement stages that require high precision.\nThe pipeline compiler will automatically decide which one to use.\n\nSkia uses u16x8 (128bit) types for a generic CPU and u16x16 (256bit) for modern x86 CPUs.\nBut instead of explicit SIMD instructions, it mainly relies on clang's vector extensions.\nAnd since they are unavailable in Rust, we have to do everything manually.\n\nAccording to our benchmarks, a SIMD-accelerated u16x8 in Rust is almost 2x slower than in Skia.\nNot sure why. For example, there are no div instruction for u16x8, so we have to use\na basic scalar version. Which means unnecessary load/store. No idea what clang does in this case.\nSurprisingly, a SIMD-accelerated u16x8 is even slower than a scalar one. Again, not sure why.\n\nTherefore we are using scalar u16x16 by default and relying on rustc/llvm auto vectorization instead.\nWhen targeting a generic CPU, we're just 5-10% slower than Skia. While u16x8 is 30-40% slower.\nAnd while `-C target-cpu=haswell` boosts our performance by around 25%,\nwe are still 40-60% behind Skia built for Haswell.\n\nOn ARM AArch64 the story is different and explicit SIMD make our code up to 2-3x faster.\n*/\n\nuse crate::PremultipliedColorU8;\n\nuse crate::pixmap::SubPixmapMut;\nuse crate::wide::{f32x8, u16x16, f32x16};\nuse crate::geom::ScreenIntRect;\n\npub const STAGE_WIDTH: usize = 16;\n\npub type StageFn = fn(p: &mut Pipeline);\n\npub struct Pipeline<'a, 'b: 'a> {\n index: usize,\n functions: &'a [StageFn],\n pixmap: &'a mut SubPixmapMut<'b>,\n mask_ctx: super::MaskCtx<'a>,\n aa_mask_ctx: super::AAMaskCtx,\n ctx: &'a mut super::Context,\n r: u16x16,\n g: u16x16,\n b: u16x16,\n a: u16x16,\n dr: u16x16,\n dg: u16x16,\n db: u16x16,\n da: u16x16,\n tail: usize,\n dx: usize,\n dy: usize,\n}\n\nimpl Pipeline<'_, '_> {\n #[inline(always)]\n fn next_stage(&mut self) {\n let next: fn(&mut Self) = self.functions[self.index];\n self.index += 1;\n next(self);\n }\n}\n\n\n// Must be in the same order as raster_pipeline::Stage\npub const STAGES: &[StageFn; super::STAGES_COUNT] = &[\n move_source_to_destination,\n move_destination_to_source,\n null_fn, // Clamp0\n null_fn, // ClampA\n premultiply,\n uniform_color,\n seed_shader,\n load_dst,\n store,\n load_dst_u8,\n store_u8,\n null_fn, // Gather\n load_mask_u8,\n mask_u8,\n scale_u8,\n lerp_u8,\n scale_1_float,\n lerp_1_float,\n destination_atop,\n destination_in,\n destination_out,\n destination_over,\n source_atop,\n source_in,\n source_out,\n source_over,\n clear,\n modulate,\n multiply,\n plus,\n screen,\n xor,\n null_fn, // ColorBurn\n null_fn, // ColorDodge\n darken,\n difference,\n exclusion,\n hard_light,\n lighten,\n overlay,\n null_fn, // SoftLight\n null_fn, // Hue\n null_fn, // Saturation\n null_fn, // Color\n null_fn, // Luminosity\n source_over_rgba,\n transform,\n null_fn, // Reflect\n null_fn, // Repeat\n null_fn, // Bilinear\n null_fn, // Bicubic\n pad_x1,\n reflect_x1,\n repeat_x1,\n gradient,\n evenly_spaced_2_stop_gradient,\n xy_to_radius,\n null_fn, // XYTo2PtConicalFocalOnCircle\n null_fn, // XYTo2PtConicalWellBehaved\n null_fn, // XYTo2PtConicalGreater\n null_fn, // Mask2PtConicalDegenerates\n null_fn, // ApplyVectorMask\n];\n\npub fn fn_ptr(f: StageFn) -> *const () {\n f as *const ()\n}\n\npub fn fn_ptr_eq(f1: StageFn, f2: StageFn) -> bool {\n core::ptr::eq(f1 as *const (), f2 as *const ())\n}\n\n#[inline(never)]\npub fn start(\n functions: &[StageFn],\n functions_tail: &[StageFn],\n rect: &ScreenIntRect,\n aa_mask_ctx: super::AAMaskCtx,\n mask_ctx: super::MaskCtx,\n ctx: &mut super::Context,\n pixmap: &mut SubPixmapMut,\n) {\n let mut p = Pipeline {\n index: 0,\n functions: &[],\n pixmap,\n mask_ctx,\n aa_mask_ctx,\n ctx,\n r: u16x16::default(),\n g: u16x16::default(),\n b: u16x16::default(),\n a: u16x16::default(),\n dr: u16x16::default(),\n dg: u16x16::default(),\n db: u16x16::default(),\n da: u16x16::default(),\n tail: 0,\n dx: 0,\n dy: 0,\n };\n\n for y in rect.y()..rect.bottom() {\n let mut x = rect.x() as usize;\n let end = rect.right() as usize;\n\n p.functions = functions;\n while x + STAGE_WIDTH <= end {\n p.index = 0;\n p.dx = x;\n p.dy = y as usize;\n p.tail = STAGE_WIDTH;\n p.next_stage();\n x += STAGE_WIDTH;\n }\n\n if x != end {\n p.index = 0;\n p.functions = functions_tail;\n p.dx = x;\n p.dy = y as usize;\n p.tail = end - x;\n p.next_stage();\n }\n }\n}\n\nfn move_source_to_destination(p: &mut Pipeline) {\n p.dr = p.r;\n p.dg = p.g;\n p.db = p.b;\n p.da = p.a;\n\n p.next_stage();\n}\n\nfn move_destination_to_source(p: &mut Pipeline) {\n p.r = p.dr;\n p.g = p.dg;\n p.b = p.db;\n p.a = p.da;\n\n p.next_stage();\n}\n\nfn premultiply(p: &mut Pipeline) {\n p.r = div255(p.r * p.a);\n p.g = div255(p.g * p.a);\n p.b = div255(p.b * p.a);\n\n p.next_stage();\n}\n\nfn uniform_color(p: &mut Pipeline) {\n let ctx = p.ctx.uniform_color;\n p.r = u16x16::splat(ctx.rgba[0]);\n p.g = u16x16::splat(ctx.rgba[1]);\n p.b = u16x16::splat(ctx.rgba[2]);\n p.a = u16x16::splat(ctx.rgba[3]);\n\n p.next_stage();\n}\n\nfn seed_shader(p: &mut Pipeline) {\n let iota = f32x16(\n f32x8::from([0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5]),\n f32x8::from([8.5, 9.5, 10.5, 11.5, 12.5, 13.5, 14.5, 15.5]),\n );\n\n let x = f32x16::splat(p.dx as f32) + iota;\n let y = f32x16::splat(p.dy as f32 + 0.5);\n split(&x, &mut p.r, &mut p.g);\n split(&y, &mut p.b, &mut p.a);\n\n p.next_stage();\n}\n\npub fn load_dst(p: &mut Pipeline) {\n load_8888(p.pixmap.slice16_at_xy(p.dx, p.dy), &mut p.dr, &mut p.dg, &mut p.db, &mut p.da);\n p.next_stage();\n}\n\npub fn load_dst_tail(p: &mut Pipeline) {\n load_8888_tail(p.tail, p.pixmap.slice_at_xy(p.dx, p.dy), &mut p.dr, &mut p.dg, &mut p.db, &mut p.da);\n p.next_stage();\n}\n\npub fn store(p: &mut Pipeline) {\n store_8888(&p.r, &p.g, &p.b, &p.a, p.pixmap.slice16_at_xy(p.dx, p.dy));\n p.next_stage();\n}\n\npub fn store_tail(p: &mut Pipeline) {\n store_8888_tail(&p.r, &p.g, &p.b, &p.a, p.tail, p.pixmap.slice_at_xy(p.dx, p.dy));\n p.next_stage();\n}\n\npub fn load_dst_u8(p: &mut Pipeline) {\n load_8(p.pixmap.slice16_mask_at_xy(p.dx, p.dy), &mut p.da);\n p.next_stage();\n}\n\npub fn load_dst_u8_tail(p: &mut Pipeline) {\n // Fill a dummy array with `tail` values. `tail` is always in a 1..STAGE_WIDTH-1 range.\n // This way we can reuse the `load_8888__` method and remove any branches.\n let data = p.pixmap.slice_mask_at_xy(p.dx, p.dy);\n let mut tmp = [0u8; STAGE_WIDTH];\n tmp[0..p.tail].copy_from_slice(&data[0..p.tail]);\n load_8(&tmp, &mut p.da);\n\n p.next_stage();\n}\n\npub fn store_u8(p: &mut Pipeline) {\n let data = p.pixmap.slice16_mask_at_xy(p.dx, p.dy);\n let a = p.a.as_slice();\n\n data[ 0] = a[ 0] as u8;\n data[ 1] = a[ 1] as u8;\n data[ 2] = a[ 2] as u8;\n data[ 3] = a[ 3] as u8;\n data[ 4] = a[ 4] as u8;\n data[ 5] = a[ 5] as u8;\n data[ 6] = a[ 6] as u8;\n data[ 7] = a[ 7] as u8;\n data[ 8] = a[ 8] as u8;\n data[ 9] = a[ 9] as u8;\n data[10] = a[10] as u8;\n data[11] = a[11] as u8;\n data[12] = a[12] as u8;\n data[13] = a[13] as u8;\n data[14] = a[14] as u8;\n data[15] = a[15] as u8;\n\n p.next_stage();\n}\n\npub fn store_u8_tail(p: &mut Pipeline) {\n let data = p.pixmap.slice_mask_at_xy(p.dx, p.dy);\n let a = p.a.as_slice();\n\n // This is better than `for i in 0..tail`, because this way the compiler\n // knows that we have only 16 steps and slices access is guarantee to be valid.\n // This removes bounds checking and a possible panic call.\n for i in 0..STAGE_WIDTH {\n data[i] = a[i] as u8;\n\n if i + 1 == p.tail {\n break;\n }\n }\n\n p.next_stage();\n}\n\n// Similar to mask_u8, but only loads the mask values without actually masking the pipeline.\nfn load_mask_u8(p: &mut Pipeline) {\n let offset = p.mask_ctx.offset(p.dx, p.dy);\n\n let mut c = u16x16::default();\n for i in 0..p.tail {\n c.0[i] = u16::from(p.mask_ctx.data[offset + i]);\n }\n\n p.r = u16x16::splat(0);\n p.g = u16x16::splat(0);\n p.b = u16x16::splat(0);\n p.a = c;\n\n p.next_stage();\n}\n\nfn mask_u8(p: &mut Pipeline) {\n let offset = p.mask_ctx.offset(p.dx, p.dy);\n\n let mut c = u16x16::default();\n for i in 0..p.tail {\n c.0[i] = u16::from(p.mask_ctx.data[offset + i]);\n }\n\n if c == u16x16::default() {\n return;\n }\n\n p.r = div255(p.r * c);\n p.g = div255(p.g * c);\n p.b = div255(p.b * c);\n p.a = div255(p.a * c);\n\n p.next_stage();\n}\n\nfn scale_u8(p: &mut Pipeline) {\n // Load u8xTail and cast it to u16x16.\n let data = p.aa_mask_ctx.copy_at_xy(p.dx, p.dy, p.tail);\n let c = u16x16([\n u16::from(data[0]),\n u16::from(data[1]),\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n ]);\n\n p.r = div255(p.r * c);\n p.g = div255(p.g * c);\n p.b = div255(p.b * c);\n p.a = div255(p.a * c);\n\n p.next_stage();\n}\n\nfn lerp_u8(p: &mut Pipeline) {\n // Load u8xTail and cast it to u16x16.\n let data = p.aa_mask_ctx.copy_at_xy(p.dx, p.dy, p.tail);\n let c = u16x16([\n u16::from(data[0]),\n u16::from(data[1]),\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n ]);\n\n p.r = lerp(p.dr, p.r, c);\n p.g = lerp(p.dg, p.g, c);\n p.b = lerp(p.db, p.b, c);\n p.a = lerp(p.da, p.a, c);\n\n p.next_stage();\n}\n\nfn scale_1_float(p: &mut Pipeline) {\n let c = from_float(p.ctx.current_coverage);\n p.r = div255(p.r * c);\n p.g = div255(p.g * c);\n p.b = div255(p.b * c);\n p.a = div255(p.a * c);\n\n p.next_stage();\n}\n\nfn lerp_1_float(p: &mut Pipeline) {\n let c = from_float(p.ctx.current_coverage);\n p.r = lerp(p.dr, p.r, c);\n p.g = lerp(p.dg, p.g, c);\n p.b = lerp(p.db, p.b, c);\n p.a = lerp(p.da, p.a, c);\n\n p.next_stage();\n}\n\nmacro_rules! blend_fn {\n ($name:ident, $f:expr) => {\n fn $name(p: &mut Pipeline) {\n p.r = $f(p.r, p.dr, p.a, p.da);\n p.g = $f(p.g, p.dg, p.a, p.da);\n p.b = $f(p.b, p.db, p.a, p.da);\n p.a = $f(p.a, p.da, p.a, p.da);\n\n p.next_stage();\n }\n };\n}\n\nblend_fn!(clear, |_, _, _, _| u16x16::splat(0));\nblend_fn!(source_atop, |s, d, sa, da| div255(s * da + d * inv(sa)));\nblend_fn!(destination_atop, |s, d, sa, da| div255(d * sa + s * inv(da)));\nblend_fn!(source_in, |s, _, _, da| div255(s * da));\nblend_fn!(destination_in, |_, d, sa, _| div255(d * sa));\nblend_fn!(source_out, |s, _, _, da| div255(s * inv(da)));\nblend_fn!(destination_out, |_, d, sa, _| div255(d * inv(sa)));\nblend_fn!(source_over, |s, d, sa, _| s + div255(d * inv(sa)));\nblend_fn!(destination_over, |s, d, _, da| d + div255(s * inv(da)));\nblend_fn!(modulate, |s, d, _, _| div255(s * d));\nblend_fn!(multiply, |s, d, sa, da| div255(s * inv(da) + d * inv(sa) + s * d));\nblend_fn!(screen, |s, d, _, _| s + d - div255(s * d));\nblend_fn!(xor, |s, d, sa, da| div255(s * inv(da) + d * inv(sa)));\n\n// Wants a type for some reason.\nblend_fn!(plus, |s: u16x16, d, _, _| (s + d).min(&u16x16::splat(255)));\n\n\nmacro_rules! blend_fn2 {\n ($name:ident, $f:expr) => {\n fn $name(p: &mut Pipeline) {\n // The same logic applied to color, and source_over for alpha.\n p.r = $f(p.r, p.dr, p.a, p.da);\n p.g = $f(p.g, p.dg, p.a, p.da);\n p.b = $f(p.b, p.db, p.a, p.da);\n p.a = p.a + div255(p.da * inv(p.a));\n\n p.next_stage();\n }\n };\n}\n\nblend_fn2!(darken, |s: u16x16, d, sa, da| s + d - div255((s * da).max(&(d * sa))));\nblend_fn2!(lighten, |s: u16x16, d, sa, da| s + d - div255((s * da).min(&(d * sa))));\nblend_fn2!(exclusion, |s: u16x16, d, _, _| s + d - u16x16::splat(2) * div255(s * d));\n\nblend_fn2!(difference, |s: u16x16, d, sa, da|\n s + d - u16x16::splat(2) * div255((s * da).min(&(d * sa))));\n\nblend_fn2!(hard_light, |s: u16x16, d: u16x16, sa, da| {\n div255(s * inv(da) + d * inv(sa)\n + (s+s).cmp_le(&sa).blend(\n u16x16::splat(2) * s * d,\n sa * da - u16x16::splat(2) * (sa-s)*(da-d)\n )\n )\n});\n\nblend_fn2!(overlay, |s: u16x16, d: u16x16, sa, da| {\n div255(s * inv(da) + d * inv(sa)\n + (d+d).cmp_le(&da).blend(\n u16x16::splat(2) * s * d,\n sa * da - u16x16::splat(2) * (sa-s)*(da-d)\n )\n )\n});\n\npub fn source_over_rgba(p: &mut Pipeline) {\n let pixels = p.pixmap.slice16_at_xy(p.dx, p.dy);\n load_8888(pixels, &mut p.dr, &mut p.dg, &mut p.db, &mut p.da);\n p.r = p.r + div255(p.dr * inv(p.a));\n p.g = p.g + div255(p.dg * inv(p.a));\n p.b = p.b + div255(p.db * inv(p.a));\n p.a = p.a + div255(p.da * inv(p.a));\n store_8888(&p.r, &p.g, &p.b, &p.a, pixels);\n\n p.next_stage();\n}\n\npub fn source_over_rgba_tail(p: &mut Pipeline) {\n let pixels = p.pixmap.slice_at_xy(p.dx, p.dy);\n load_8888_tail(p.tail, pixels, &mut p.dr, &mut p.dg, &mut p.db, &mut p.da);\n p.r = p.r + div255(p.dr * inv(p.a));\n p.g = p.g + div255(p.dg * inv(p.a));\n p.b = p.b + div255(p.db * inv(p.a));\n p.a = p.a + div255(p.da * inv(p.a));\n store_8888_tail(&p.r, &p.g, &p.b, &p.a, p.tail, pixels);\n\n p.next_stage();\n}\n\nfn transform(p: &mut Pipeline) {\n let ts = &p.ctx.transform;\n\n let x = join(&p.r, &p.g);\n let y = join(&p.b, &p.a);\n\n let nx = mad(x, f32x16::splat(ts.sx), mad(y, f32x16::splat(ts.kx), f32x16::splat(ts.tx)));\n let ny = mad(x, f32x16::splat(ts.ky), mad(y, f32x16::splat(ts.sy), f32x16::splat(ts.ty)));\n\n split(&nx, &mut p.r, &mut p.g);\n split(&ny, &mut p.b, &mut p.a);\n\n p.next_stage();\n}\n\nfn pad_x1(p: &mut Pipeline) {\n let x = join(&p.r, &p.g);\n let x = x.normalize();\n split(&x, &mut p.r, &mut p.g);\n\n p.next_stage();\n}\n\nfn reflect_x1(p: &mut Pipeline) {\n let x = join(&p.r, &p.g);\n let two = |x| x + x;\n let x = (\n (x - f32x16::splat(1.0))\n - two(((x - f32x16::splat(1.0)) * f32x16::splat(0.5)).floor())\n - f32x16::splat(1.0)\n ).abs().normalize();\n split(&x, &mut p.r, &mut p.g);\n\n p.next_stage();\n}\n\nfn repeat_x1(p: &mut Pipeline) {\n let x = join(&p.r, &p.g);\n let x = (x - x.floor()).normalize();\n split(&x, &mut p.r, &mut p.g);\n\n p.next_stage();\n}\n\nfn gradient(p: &mut Pipeline) {\n let ctx = &p.ctx.gradient;\n\n // N.B. The loop starts at 1 because idx 0 is the color to use before the first stop.\n let t = join(&p.r, &p.g);\n let mut idx = u16x16::splat(0);\n for i in 1..ctx.len {\n let tt = ctx.t_values[i].get();\n let t0: [f32; 8] = t.0.into();\n let t1: [f32; 8] = t.1.into();\n idx.0[ 0] += (t0[0] >= tt) as u16;\n idx.0[ 1] += (t0[1] >= tt) as u16;\n idx.0[ 2] += (t0[2] >= tt) as u16;\n idx.0[ 3] += (t0[3] >= tt) as u16;\n idx.0[ 4] += (t0[4] >= tt) as u16;\n idx.0[ 5] += (t0[5] >= tt) as u16;\n idx.0[ 6] += (t0[6] >= tt) as u16;\n idx.0[ 7] += (t0[7] >= tt) as u16;\n idx.0[ 8] += (t1[0] >= tt) as u16;\n idx.0[ 9] += (t1[1] >= tt) as u16;\n idx.0[10] += (t1[2] >= tt) as u16;\n idx.0[11] += (t1[3] >= tt) as u16;\n idx.0[12] += (t1[4] >= tt) as u16;\n idx.0[13] += (t1[5] >= tt) as u16;\n idx.0[14] += (t1[6] >= tt) as u16;\n idx.0[15] += (t1[7] >= tt) as u16;\n }\n gradient_lookup(ctx, &idx, t, &mut p.r, &mut p.g, &mut p.b, &mut p.a);\n\n p.next_stage();\n}\n\nfn evenly_spaced_2_stop_gradient(p: &mut Pipeline) {\n let ctx = &p.ctx.evenly_spaced_2_stop_gradient;\n\n let t = join(&p.r, &p.g);\n round_f32_to_u16(\n mad(t, f32x16::splat(ctx.factor.r), f32x16::splat(ctx.bias.r)),\n mad(t, f32x16::splat(ctx.factor.g), f32x16::splat(ctx.bias.g)),\n mad(t, f32x16::splat(ctx.factor.b), f32x16::splat(ctx.bias.b)),\n mad(t, f32x16::splat(ctx.factor.a), f32x16::splat(ctx.bias.a)),\n &mut p.r, &mut p.g, &mut p.b, &mut p.a,\n );\n\n p.next_stage();\n}\n\nfn xy_to_radius(p: &mut Pipeline) {\n let x = join(&p.r, &p.g);\n let y = join(&p.b, &p.a);\n let x = (x*x + y*y).sqrt();\n split(&x, &mut p.r, &mut p.g);\n split(&y, &mut p.b, &mut p.a);\n\n p.next_stage();\n}\n\n// We are using u16 for index, not u32 as Skia, to simplify the code a bit.\n// The gradient creation code will not allow that many stops anyway.\nfn gradient_lookup(\n ctx: &super::GradientCtx, idx: &u16x16, t: f32x16,\n r: &mut u16x16, g: &mut u16x16, b: &mut u16x16, a: &mut u16x16,\n) {\n macro_rules! gather {\n ($d:expr, $c:ident) => {\n // Surprisingly, but bound checking doesn't affect the performance.\n // And since `idx` can contain any number, we should leave it in place.\n f32x16(\n f32x8::from([\n $d[idx.0[ 0] as usize].$c,\n $d[idx.0[ 1] as usize].$c,\n $d[idx.0[ 2] as usize].$c,\n $d[idx.0[ 3] as usize].$c,\n $d[idx.0[ 4] as usize].$c,\n $d[idx.0[ 5] as usize].$c,\n $d[idx.0[ 6] as usize].$c,\n $d[idx.0[ 7] as usize].$c,\n ]),\n f32x8::from([\n $d[idx.0[ 8] as usize].$c,\n $d[idx.0[ 9] as usize].$c,\n $d[idx.0[10] as usize].$c,\n $d[idx.0[11] as usize].$c,\n $d[idx.0[12] as usize].$c,\n $d[idx.0[13] as usize].$c,\n $d[idx.0[14] as usize].$c,\n $d[idx.0[15] as usize].$c,\n ]),\n )\n };\n }\n\n let fr = gather!(&ctx.factors, r);\n let fg = gather!(&ctx.factors, g);\n let fb = gather!(&ctx.factors, b);\n let fa = gather!(&ctx.factors, a);\n\n let br = gather!(&ctx.biases, r);\n let bg = gather!(&ctx.biases, g);\n let bb = gather!(&ctx.biases, b);\n let ba = gather!(&ctx.biases, a);\n\n round_f32_to_u16(\n mad(t, fr, br),\n mad(t, fg, bg),\n mad(t, fb, bb),\n mad(t, fa, ba),\n r, g, b, a,\n );\n}\n\n#[inline(always)]\nfn round_f32_to_u16(\n rf: f32x16, gf: f32x16, bf: f32x16, af: f32x16,\n r: &mut u16x16, g: &mut u16x16, b: &mut u16x16, a: &mut u16x16,\n) {\n // TODO: may produce a slightly different result to Skia\n // affects the two_stops_linear_mirror test\n\n let rf = rf.normalize() * f32x16::splat(255.0) + f32x16::splat(0.5);\n let gf = gf.normalize() * f32x16::splat(255.0) + f32x16::splat(0.5);\n let bf = bf.normalize() * f32x16::splat(255.0) + f32x16::splat(0.5);\n let af = af * f32x16::splat(255.0) + f32x16::splat(0.5);\n\n rf.save_to_u16x16(r);\n gf.save_to_u16x16(g);\n bf.save_to_u16x16(b);\n af.save_to_u16x16(a);\n}\n\npub fn just_return(_: &mut Pipeline) {\n // Ends the loop.\n}\n\npub fn null_fn(_: &mut Pipeline) {\n // Just for unsupported functions in STAGES.\n}\n\n#[inline(always)]\nfn load_8888(\n data: &[PremultipliedColorU8; STAGE_WIDTH],\n r: &mut u16x16, g: &mut u16x16, b: &mut u16x16, a: &mut u16x16,\n) {\n *r = u16x16([\n data[ 0].red() as u16, data[ 1].red() as u16, data[ 2].red() as u16, data[ 3].red() as u16,\n data[ 4].red() as u16, data[ 5].red() as u16, data[ 6].red() as u16, data[ 7].red() as u16,\n data[ 8].red() as u16, data[ 9].red() as u16, data[10].red() as u16, data[11].red() as u16,\n data[12].red() as u16, data[13].red() as u16, data[14].red() as u16, data[15].red() as u16,\n ]);\n\n *g = u16x16([\n data[ 0].green() as u16, data[ 1].green() as u16, data[ 2].green() as u16, data[ 3].green() as u16,\n data[ 4].green() as u16, data[ 5].green() as u16, data[ 6].green() as u16, data[ 7].green() as u16,\n data[ 8].green() as u16, data[ 9].green() as u16, data[10].green() as u16, data[11].green() as u16,\n data[12].green() as u16, data[13].green() as u16, data[14].green() as u16, data[15].green() as u16,\n ]);\n\n *b = u16x16([\n data[ 0].blue() as u16, data[ 1].blue() as u16, data[ 2].blue() as u16, data[ 3].blue() as u16,\n data[ 4].blue() as u16, data[ 5].blue() as u16, data[ 6].blue() as u16, data[ 7].blue() as u16,\n data[ 8].blue() as u16, data[ 9].blue() as u16, data[10].blue() as u16, data[11].blue() as u16,\n data[12].blue() as u16, data[13].blue() as u16, data[14].blue() as u16, data[15].blue() as u16,\n ]);\n\n *a = u16x16([\n data[ 0].alpha() as u16, data[ 1].alpha() as u16, data[ 2].alpha() as u16, data[ 3].alpha() as u16,\n data[ 4].alpha() as u16, data[ 5].alpha() as u16, data[ 6].alpha() as u16, data[ 7].alpha() as u16,\n data[ 8].alpha() as u16, data[ 9].alpha() as u16, data[10].alpha() as u16, data[11].alpha() as u16,\n data[12].alpha() as u16, data[13].alpha() as u16, data[14].alpha() as u16, data[15].alpha() as u16,\n ]);\n}\n\n#[inline(always)]\nfn load_8888_tail(\n tail: usize, data: &[PremultipliedColorU8],\n r: &mut u16x16, g: &mut u16x16, b: &mut u16x16, a: &mut u16x16,\n) {\n // Fill a dummy array with `tail` values. `tail` is always in a 1..STAGE_WIDTH-1 range.\n // This way we can reuse the `load_8888__` method and remove any branches.\n let mut tmp = [PremultipliedColorU8::TRANSPARENT; STAGE_WIDTH];\n tmp[0..tail].copy_from_slice(&data[0..tail]);\n load_8888(&tmp, r, g, b, a);\n}\n\n#[inline(always)]\nfn store_8888(\n r: &u16x16, g: &u16x16, b: &u16x16, a: &u16x16,\n data: &mut [PremultipliedColorU8; STAGE_WIDTH],\n) {\n let r = r.as_slice();\n let g = g.as_slice();\n let b = b.as_slice();\n let a = a.as_slice();\n\n data[ 0] = PremultipliedColorU8::from_rgba_unchecked(r[ 0] as u8, g[ 0] as u8, b[ 0] as u8, a[ 0] as u8);\n data[ 1] = PremultipliedColorU8::from_rgba_unchecked(r[ 1] as u8, g[ 1] as u8, b[ 1] as u8, a[ 1] as u8);\n data[ 2] = PremultipliedColorU8::from_rgba_unchecked(r[ 2] as u8, g[ 2] as u8, b[ 2] as u8, a[ 2] as u8);\n data[ 3] = PremultipliedColorU8::from_rgba_unchecked(r[ 3] as u8, g[ 3] as u8, b[ 3] as u8, a[ 3] as u8);\n data[ 4] = PremultipliedColorU8::from_rgba_unchecked(r[ 4] as u8, g[ 4] as u8, b[ 4] as u8, a[ 4] as u8);\n data[ 5] = PremultipliedColorU8::from_rgba_unchecked(r[ 5] as u8, g[ 5] as u8, b[ 5] as u8, a[ 5] as u8);\n data[ 6] = PremultipliedColorU8::from_rgba_unchecked(r[ 6] as u8, g[ 6] as u8, b[ 6] as u8, a[ 6] as u8);\n data[ 7] = PremultipliedColorU8::from_rgba_unchecked(r[ 7] as u8, g[ 7] as u8, b[ 7] as u8, a[ 7] as u8);\n data[ 8] = PremultipliedColorU8::from_rgba_unchecked(r[ 8] as u8, g[ 8] as u8, b[ 8] as u8, a[ 8] as u8);\n data[ 9] = PremultipliedColorU8::from_rgba_unchecked(r[ 9] as u8, g[ 9] as u8, b[ 9] as u8, a[ 9] as u8);\n data[10] = PremultipliedColorU8::from_rgba_unchecked(r[10] as u8, g[10] as u8, b[10] as u8, a[10] as u8);\n data[11] = PremultipliedColorU8::from_rgba_unchecked(r[11] as u8, g[11] as u8, b[11] as u8, a[11] as u8);\n data[12] = PremultipliedColorU8::from_rgba_unchecked(r[12] as u8, g[12] as u8, b[12] as u8, a[12] as u8);\n data[13] = PremultipliedColorU8::from_rgba_unchecked(r[13] as u8, g[13] as u8, b[13] as u8, a[13] as u8);\n data[14] = PremultipliedColorU8::from_rgba_unchecked(r[14] as u8, g[14] as u8, b[14] as u8, a[14] as u8);\n data[15] = PremultipliedColorU8::from_rgba_unchecked(r[15] as u8, g[15] as u8, b[15] as u8, a[15] as u8);\n}\n\n#[inline(always)]\nfn store_8888_tail(\n r: &u16x16, g: &u16x16, b: &u16x16, a: &u16x16,\n tail: usize, data: &mut [PremultipliedColorU8],\n) {\n let r = r.as_slice();\n let g = g.as_slice();\n let b = b.as_slice();\n let a = a.as_slice();\n\n // This is better than `for i in 0..tail`, because this way the compiler\n // knows that we have only 16 steps and slices access is guarantee to be valid.\n // This removes bounds checking and a possible panic call.\n for i in 0..STAGE_WIDTH {\n data[i] = PremultipliedColorU8::from_rgba_unchecked(\n r[i] as u8, g[i] as u8, b[i] as u8, a[i] as u8,\n );\n\n if i + 1 == tail {\n break;\n }\n }\n}\n\n#[inline(always)]\nfn load_8(data: &[u8; STAGE_WIDTH], a: &mut u16x16) {\n *a = u16x16([\n data[ 0] as u16, data[ 1] as u16, data[ 2] as u16, data[ 3] as u16,\n data[ 4] as u16, data[ 5] as u16, data[ 6] as u16, data[ 7] as u16,\n data[ 8] as u16, data[ 9] as u16, data[10] as u16, data[11] as u16,\n data[12] as u16, data[13] as u16, data[14] as u16, data[15] as u16,\n ]);\n}\n\n#[inline(always)]\nfn div255(v: u16x16) -> u16x16 {\n // Skia uses `vrshrq_n_u16(vrsraq_n_u16(v, v, 8), 8)` here when NEON is available,\n // but it doesn't affect performance much and breaks reproducible result. Ignore it.\n // NOTE: the compiler does not replace the devision with a shift.\n (v + u16x16::splat(255)) >> u16x16::splat(8) // / u16x16::splat(256)\n}\n\n#[inline(always)]\nfn inv(v: u16x16) -> u16x16 {\n u16x16::splat(255) - v\n}\n\n#[inline(always)]\nfn from_float(f: f32) -> u16x16 {\n u16x16::splat((f * 255.0 + 0.5) as u16)\n}\n\n#[inline(always)]\nfn lerp(from: u16x16, to: u16x16, t: u16x16) -> u16x16 {\n div255(from * inv(t) + to * t)\n}\n\n#[inline(always)]\nfn split(v: &f32x16, lo: &mut u16x16, hi: &mut u16x16) {\n // We're splitting f32x16 (512bit) into two u16x16 (256 bit).\n let data: [u8; 64] = bytemuck::cast(*v);\n let d0: &mut [u8; 32] = bytemuck::cast_mut(&mut lo.0);\n let d1: &mut [u8; 32] = bytemuck::cast_mut(&mut hi.0);\n\n d0.copy_from_slice(&data[0..32]);\n d1.copy_from_slice(&data[32..64]);\n}\n\n#[inline(always)]\nfn join(lo: &u16x16, hi: &u16x16) -> f32x16 {\n // We're joining two u16x16 (256 bit) into f32x16 (512bit).\n\n let d0: [u8; 32] = bytemuck::cast(lo.0);\n let d1: [u8; 32] = bytemuck::cast(hi.0);\n\n let mut v = f32x16::default();\n let data: &mut [u8; 64] = bytemuck::cast_mut(&mut v);\n\n data[0..32].copy_from_slice(&d0);\n data[32..64].copy_from_slice(&d1);\n\n v\n}\n\n#[inline(always)]\nfn mad(f: f32x16, m: f32x16, a: f32x16) -> f32x16 {\n // NEON vmlaq_f32 doesn't seem to affect performance in any way. Ignore it.\n f * m + a\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/pipeline/mod.rs", "sha256": "300bfe14f5223aed89c9d2f31b087b942eba6ed28b0aa0100d7a0e21a25c7fa7", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2016 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n/*!\nA raster pipeline implementation.\n\nDespite having a lot of changes compared to `SkRasterPipeline`,\nthe core principles are the same:\n\n1. A pipeline consists of stages.\n1. A pipeline has a global context shared by all stages.\n Unlike Skia, were each stage has it's own, possibly shared, context.\n1. Each stage has a high precision implementation. See `highp.rs`.\n1. Some stages have a low precision implementation. See `lowp.rs`.\n1. Each stage calls the \"next\" stage after its done.\n1. During pipeline \"compilation\", if **all** stages have a lowp implementation,\n the lowp pipeline will be used. Otherwise, the highp variant will be used.\n1. The pipeline \"compilation\" produces a list of function pointer.\n The last pointer is a pointer to the \"return\" function,\n which simply stops the execution of the pipeline.\n\nThis implementation is a bit tricky, but it gives the maximum performance.\nA simple and straightforward implementation using traits and loops, like:\n\n```ignore\ntrait StageTrait {\n fn apply(&mut self, pixels: &mut [Pixel]);\n}\n\nlet stages: Vec<&mut dyn StageTrait>;\nfor stage in stages {\n stage.apply(pixels);\n}\n```\n\nwill be at least 20-30% slower. Not really sure why.\n\nAlso, since this module is all about performance, any kind of branching is\nstrictly forbidden. All stage functions must not use `if`, `match` or loops.\nThere are still some exceptions, which are basically an imperfect implementations\nand should be optimized out in the future.\n*/\n\nuse alloc::vec::Vec;\n\nuse arrayvec::ArrayVec;\n\nuse tiny_skia_path::NormalizedF32;\n\nuse crate::{Color, PremultipliedColor, PremultipliedColorU8, SpreadMode};\nuse crate::{PixmapRef, Transform};\n\npub use blitter::RasterPipelineBlitter;\n\nuse crate::geom::ScreenIntRect;\nuse crate::pixmap::SubPixmapMut;\nuse crate::wide::u32x8;\n\nmod blitter;\n#[rustfmt::skip] mod highp;\n#[rustfmt::skip] mod lowp;\n\nconst MAX_STAGES: usize = 32; // More than enough.\n\n#[allow(dead_code)]\n#[derive(Copy, Clone, Debug)]\npub enum Stage {\n MoveSourceToDestination = 0,\n MoveDestinationToSource,\n Clamp0,\n ClampA,\n Premultiply,\n UniformColor,\n SeedShader,\n LoadDestination,\n Store,\n LoadDestinationU8,\n StoreU8,\n Gather,\n LoadMaskU8,\n MaskU8,\n ScaleU8,\n LerpU8,\n Scale1Float,\n Lerp1Float,\n DestinationAtop,\n DestinationIn,\n DestinationOut,\n DestinationOver,\n SourceAtop,\n SourceIn,\n SourceOut,\n SourceOver,\n Clear,\n Modulate,\n Multiply,\n Plus,\n Screen,\n Xor,\n ColorBurn,\n ColorDodge,\n Darken,\n Difference,\n Exclusion,\n HardLight,\n Lighten,\n Overlay,\n SoftLight,\n Hue,\n Saturation,\n Color,\n Luminosity,\n SourceOverRgba,\n Transform,\n Reflect,\n Repeat,\n Bilinear,\n Bicubic,\n PadX1,\n ReflectX1,\n RepeatX1,\n Gradient,\n EvenlySpaced2StopGradient,\n XYToRadius,\n XYTo2PtConicalFocalOnCircle,\n XYTo2PtConicalWellBehaved,\n XYTo2PtConicalGreater,\n Mask2PtConicalDegenerates,\n ApplyVectorMask,\n}\n\npub const STAGES_COUNT: usize = Stage::ApplyVectorMask as usize + 1;\n\nimpl<'a> PixmapRef<'a> {\n #[inline(always)]\n pub(crate) fn gather(&self, index: u32x8) -> [PremultipliedColorU8; highp::STAGE_WIDTH] {\n let index: [u32; 8] = bytemuck::cast(index);\n let pixels = self.pixels();\n [\n pixels[index[0] as usize],\n pixels[index[1] as usize],\n pixels[index[2] as usize],\n pixels[index[3] as usize],\n pixels[index[4] as usize],\n pixels[index[5] as usize],\n pixels[index[6] as usize],\n pixels[index[7] as usize],\n ]\n }\n}\n\nimpl<'a> SubPixmapMut<'a> {\n #[inline(always)]\n pub(crate) fn offset(&self, dx: usize, dy: usize) -> usize {\n self.real_width * dy + dx\n }\n\n #[inline(always)]\n pub(crate) fn slice_at_xy(&mut self, dx: usize, dy: usize) -> &mut [PremultipliedColorU8] {\n let offset = self.offset(dx, dy);\n &mut self.pixels_mut()[offset..]\n }\n\n #[inline(always)]\n pub(crate) fn slice_mask_at_xy(&mut self, dx: usize, dy: usize) -> &mut [u8] {\n let offset = self.offset(dx, dy);\n &mut self.data[offset..]\n }\n\n #[inline(always)]\n pub(crate) fn slice4_at_xy(\n &mut self,\n dx: usize,\n dy: usize,\n ) -> &mut [PremultipliedColorU8; highp::STAGE_WIDTH] {\n arrayref::array_mut_ref!(self.pixels_mut(), self.offset(dx, dy), highp::STAGE_WIDTH)\n }\n\n #[inline(always)]\n pub(crate) fn slice16_at_xy(\n &mut self,\n dx: usize,\n dy: usize,\n ) -> &mut [PremultipliedColorU8; lowp::STAGE_WIDTH] {\n arrayref::array_mut_ref!(self.pixels_mut(), self.offset(dx, dy), lowp::STAGE_WIDTH)\n }\n\n #[inline(always)]\n pub(crate) fn slice16_mask_at_xy(\n &mut self,\n dx: usize,\n dy: usize,\n ) -> &mut [u8; lowp::STAGE_WIDTH] {\n arrayref::array_mut_ref!(self.data, self.offset(dx, dy), lowp::STAGE_WIDTH)\n }\n}\n\n#[derive(Default, Debug)]\npub struct AAMaskCtx {\n pub pixels: [u8; 2],\n pub stride: u32, // can be zero\n pub shift: usize, // mask offset/position in pixmap coordinates\n}\n\nimpl AAMaskCtx {\n #[inline(always)]\n pub fn copy_at_xy(&self, dx: usize, dy: usize, tail: usize) -> [u8; 2] {\n let offset = (self.stride as usize * dy + dx) - self.shift;\n // We have only 3 variants, so unroll them.\n match (offset, tail) {\n (0, 1) => [self.pixels[0], 0],\n (0, 2) => [self.pixels[0], self.pixels[1]],\n (1, 1) => [self.pixels[1], 0],\n _ => [0, 0], // unreachable\n }\n }\n}\n\n#[derive(Copy, Clone, Debug)]\npub struct MaskCtx<'a> {\n pub data: &'a [u8],\n pub real_width: u32,\n}\n\nimpl Default for MaskCtx<'_> {\n fn default() -> Self {\n MaskCtx {\n data: &[],\n real_width: 0,\n }\n }\n}\n\nimpl MaskCtx<'_> {\n #[inline(always)]\n fn offset(&self, dx: usize, dy: usize) -> usize {\n self.real_width as usize * dy + dx\n }\n}\n\n#[derive(Default)]\npub struct Context {\n pub current_coverage: f32,\n pub sampler: SamplerCtx,\n pub uniform_color: UniformColorCtx,\n pub evenly_spaced_2_stop_gradient: EvenlySpaced2StopGradientCtx,\n pub gradient: GradientCtx,\n pub two_point_conical_gradient: TwoPointConicalGradientCtx,\n pub limit_x: TileCtx,\n pub limit_y: TileCtx,\n pub transform: Transform,\n}\n\n#[derive(Copy, Clone, Default, Debug)]\npub struct SamplerCtx {\n pub spread_mode: SpreadMode,\n pub inv_width: f32,\n pub inv_height: f32,\n}\n\n#[derive(Copy, Clone, Default, Debug)]\npub struct UniformColorCtx {\n pub r: f32,\n pub g: f32,\n pub b: f32,\n pub a: f32,\n pub rgba: [u16; 4], // [0,255] in a 16-bit lane.\n}\n\n// A gradient color is an unpremultiplied RGBA not in a 0..1 range.\n// It basically can have any float value.\n#[derive(Copy, Clone, Default, Debug)]\npub struct GradientColor {\n pub r: f32,\n pub g: f32,\n pub b: f32,\n pub a: f32,\n}\n\nimpl GradientColor {\n pub fn new(r: f32, g: f32, b: f32, a: f32) -> Self {\n GradientColor { r, g, b, a }\n }\n}\n\nimpl From for GradientColor {\n fn from(c: Color) -> Self {\n GradientColor {\n r: c.red(),\n g: c.green(),\n b: c.blue(),\n a: c.alpha(),\n }\n }\n}\n\n#[derive(Copy, Clone, Default, Debug)]\npub struct EvenlySpaced2StopGradientCtx {\n pub factor: GradientColor,\n pub bias: GradientColor,\n}\n\n#[derive(Clone, Default, Debug)]\npub struct GradientCtx {\n /// This value stores the actual colors count.\n /// `factors` and `biases` must store at least 16 values,\n /// since this is the length of a lowp pipeline stage.\n /// So any any value past `len` is just zeros.\n pub len: usize,\n pub factors: Vec,\n pub biases: Vec,\n pub t_values: Vec,\n}\n\nimpl GradientCtx {\n pub fn push_const_color(&mut self, color: GradientColor) {\n self.factors.push(GradientColor::new(0.0, 0.0, 0.0, 0.0));\n self.biases.push(color);\n }\n}\n\n#[derive(Copy, Clone, Default, Debug)]\npub struct TwoPointConicalGradientCtx {\n // This context is used only in highp, where we use Tx4.\n pub mask: u32x8,\n pub p0: f32,\n}\n\n#[derive(Copy, Clone, Default, Debug)]\npub struct TileCtx {\n pub scale: f32,\n pub inv_scale: f32, // cache of 1/scale\n}\n\npub struct RasterPipelineBuilder {\n stages: ArrayVec,\n force_hq_pipeline: bool,\n pub ctx: Context,\n}\n\nimpl RasterPipelineBuilder {\n pub fn new() -> Self {\n RasterPipelineBuilder {\n stages: ArrayVec::new(),\n force_hq_pipeline: false,\n ctx: Context::default(),\n }\n }\n\n pub fn set_force_hq_pipeline(&mut self, hq: bool) {\n self.force_hq_pipeline = hq;\n }\n\n pub fn push(&mut self, stage: Stage) {\n self.stages.push(stage);\n }\n\n pub fn push_transform(&mut self, ts: Transform) {\n if ts.is_finite() && !ts.is_identity() {\n self.stages.push(Stage::Transform);\n self.ctx.transform = ts;\n }\n }\n\n pub fn push_uniform_color(&mut self, c: PremultipliedColor) {\n let r = c.red();\n let g = c.green();\n let b = c.blue();\n let a = c.alpha();\n let rgba = [\n (r * 255.0 + 0.5) as u16,\n (g * 255.0 + 0.5) as u16,\n (b * 255.0 + 0.5) as u16,\n (a * 255.0 + 0.5) as u16,\n ];\n\n let ctx = UniformColorCtx { r, g, b, a, rgba };\n\n self.stages.push(Stage::UniformColor);\n self.ctx.uniform_color = ctx;\n }\n\n pub fn compile(self) -> RasterPipeline {\n if self.stages.is_empty() {\n return RasterPipeline {\n kind: RasterPipelineKind::High {\n functions: ArrayVec::new(),\n tail_functions: ArrayVec::new(),\n },\n ctx: Context::default(),\n };\n }\n\n let is_lowp_compatible = self\n .stages\n .iter()\n .all(|stage| !lowp::fn_ptr_eq(lowp::STAGES[*stage as usize], lowp::null_fn));\n\n if self.force_hq_pipeline || !is_lowp_compatible {\n let mut functions: ArrayVec<_, MAX_STAGES> = self\n .stages\n .iter()\n .map(|stage| highp::STAGES[*stage as usize] as highp::StageFn)\n .collect();\n functions.push(highp::just_return as highp::StageFn);\n\n // I wasn't able to reproduce Skia's load_8888_/store_8888_ performance.\n // Skia uses fallthrough switch, which is probably the reason.\n // In Rust, any branching in load/store code drastically affects the performance.\n // So instead, we're using two \"programs\": one for \"full stages\" and one for \"tail stages\".\n // While the only difference is the load/store methods.\n let mut tail_functions = functions.clone();\n for fun in &mut tail_functions {\n if highp::fn_ptr(*fun) == highp::fn_ptr(highp::load_dst) {\n *fun = highp::load_dst_tail as highp::StageFn;\n } else if highp::fn_ptr(*fun) == highp::fn_ptr(highp::store) {\n *fun = highp::store_tail as highp::StageFn;\n } else if highp::fn_ptr(*fun) == highp::fn_ptr(highp::load_dst_u8) {\n *fun = highp::load_dst_u8_tail as highp::StageFn;\n } else if highp::fn_ptr(*fun) == highp::fn_ptr(highp::store_u8) {\n *fun = highp::store_u8_tail as highp::StageFn;\n } else if highp::fn_ptr(*fun) == highp::fn_ptr(highp::source_over_rgba) {\n // SourceOverRgba calls load/store manually, without the pipeline,\n // therefore we have to switch it too.\n *fun = highp::source_over_rgba_tail as highp::StageFn;\n }\n }\n\n RasterPipeline {\n kind: RasterPipelineKind::High {\n functions,\n tail_functions,\n },\n ctx: self.ctx,\n }\n } else {\n let mut functions: ArrayVec<_, MAX_STAGES> = self\n .stages\n .iter()\n .map(|stage| lowp::STAGES[*stage as usize] as lowp::StageFn)\n .collect();\n functions.push(lowp::just_return as lowp::StageFn);\n\n // See above.\n let mut tail_functions = functions.clone();\n for fun in &mut tail_functions {\n if lowp::fn_ptr(*fun) == lowp::fn_ptr(lowp::load_dst) {\n *fun = lowp::load_dst_tail as lowp::StageFn;\n } else if lowp::fn_ptr(*fun) == lowp::fn_ptr(lowp::store) {\n *fun = lowp::store_tail as lowp::StageFn;\n } else if lowp::fn_ptr(*fun) == lowp::fn_ptr(lowp::load_dst_u8) {\n *fun = lowp::load_dst_u8_tail as lowp::StageFn;\n } else if lowp::fn_ptr(*fun) == lowp::fn_ptr(lowp::store_u8) {\n *fun = lowp::store_u8_tail as lowp::StageFn;\n } else if lowp::fn_ptr(*fun) == lowp::fn_ptr(lowp::source_over_rgba) {\n // SourceOverRgba calls load/store manually, without the pipeline,\n // therefore we have to switch it too.\n *fun = lowp::source_over_rgba_tail as lowp::StageFn;\n }\n }\n\n RasterPipeline {\n kind: RasterPipelineKind::Low {\n functions,\n tail_functions,\n },\n ctx: self.ctx,\n }\n }\n }\n}\n\npub enum RasterPipelineKind {\n High {\n functions: ArrayVec,\n tail_functions: ArrayVec,\n },\n Low {\n functions: ArrayVec,\n tail_functions: ArrayVec,\n },\n}\n\npub struct RasterPipeline {\n kind: RasterPipelineKind,\n pub ctx: Context,\n}\n\nimpl RasterPipeline {\n pub fn run(\n &mut self,\n rect: &ScreenIntRect,\n aa_mask_ctx: AAMaskCtx,\n mask_ctx: MaskCtx,\n pixmap_src: PixmapRef,\n pixmap_dst: &mut SubPixmapMut,\n ) {\n match self.kind {\n RasterPipelineKind::High {\n ref functions,\n ref tail_functions,\n } => {\n highp::start(\n functions.as_slice(),\n tail_functions.as_slice(),\n rect,\n aa_mask_ctx,\n mask_ctx,\n &mut self.ctx,\n pixmap_src,\n pixmap_dst,\n );\n }\n RasterPipelineKind::Low {\n ref functions,\n ref tail_functions,\n } => {\n lowp::start(\n functions.as_slice(),\n tail_functions.as_slice(),\n rect,\n aa_mask_ctx,\n mask_ctx,\n &mut self.ctx,\n // lowp doesn't support pattern, so no `pixmap_src` for it.\n pixmap_dst,\n );\n }\n }\n }\n}\n\n#[rustfmt::skip]\n#[cfg(test)]\nmod blend_tests {\n // Test blending modes.\n //\n // Skia has two kinds of a raster pipeline: high and low precision.\n // \"High\" uses f32 and \"low\" uses u16.\n // And for basic operations we don't need f32 and u16 simply faster.\n // But those modes are not identical. They can produce slightly different results\n // due rounding.\n\n use super::*;\n use crate::{BlendMode, Color, Pixmap, PremultipliedColorU8};\n use crate::geom::IntSizeExt;\n\n macro_rules! test_blend {\n ($name:ident, $mode:expr, $is_highp:expr, $r:expr, $g:expr, $b:expr, $a:expr) => {\n #[test]\n fn $name() {\n let mut pixmap = Pixmap::new(1, 1).unwrap();\n pixmap.fill(Color::from_rgba8(50, 127, 150, 200));\n\n let pixmap_src = PixmapRef::from_bytes(&[0, 0, 0, 0], 1, 1).unwrap();\n\n let mut p = RasterPipelineBuilder::new();\n p.set_force_hq_pipeline($is_highp);\n p.push_uniform_color(Color::from_rgba8(220, 140, 75, 180).premultiply());\n p.push(Stage::LoadDestination);\n p.push($mode.to_stage().unwrap());\n p.push(Stage::Store);\n let mut p = p.compile();\n let rect = pixmap.size().to_screen_int_rect(0, 0);\n p.run(&rect, AAMaskCtx::default(), MaskCtx::default(), pixmap_src,\n &mut pixmap.as_mut().as_subpixmap());\n\n assert_eq!(\n pixmap.as_ref().pixel(0, 0).unwrap(),\n PremultipliedColorU8::from_rgba($r, $g, $b, $a).unwrap()\n );\n }\n };\n }\n\n macro_rules! test_blend_lowp {\n ($name:ident, $mode:expr, $r:expr, $g:expr, $b:expr, $a:expr) => (\n test_blend!{$name, $mode, false, $r, $g, $b, $a}\n )\n }\n\n macro_rules! test_blend_highp {\n ($name:ident, $mode:expr, $r:expr, $g:expr, $b:expr, $a:expr) => (\n test_blend!{$name, $mode, true, $r, $g, $b, $a}\n )\n }\n\n test_blend_lowp!(clear_lowp, BlendMode::Clear, 0, 0, 0, 0);\n // Source is a no-op\n test_blend_lowp!(destination_lowp, BlendMode::Destination, 39, 100, 118, 200);\n test_blend_lowp!(source_over_lowp, BlendMode::SourceOver, 167, 129, 88, 239);\n test_blend_lowp!(destination_over_lowp, BlendMode::DestinationOver, 73, 122, 130, 239);\n test_blend_lowp!(source_in_lowp, BlendMode::SourceIn, 122, 78, 42, 141);\n test_blend_lowp!(destination_in_lowp, BlendMode::DestinationIn, 28, 71, 83, 141);\n test_blend_lowp!(source_out_lowp, BlendMode::SourceOut, 34, 22, 12, 39);\n test_blend_lowp!(destination_out_lowp, BlendMode::DestinationOut, 12, 30, 35, 59);\n test_blend_lowp!(source_atop_lowp, BlendMode::SourceAtop, 133, 107, 76, 200);\n test_blend_lowp!(destination_atop_lowp, BlendMode::DestinationAtop, 61, 92, 95, 180);\n test_blend_lowp!(xor_lowp, BlendMode::Xor, 45, 51, 46, 98);\n test_blend_lowp!(plus_lowp, BlendMode::Plus, 194, 199, 171, 255);\n test_blend_lowp!(modulate_lowp, BlendMode::Modulate, 24, 39, 25, 141);\n test_blend_lowp!(screen_lowp, BlendMode::Screen, 170, 160, 146, 239);\n test_blend_lowp!(overlay_lowp, BlendMode::Overlay, 92, 128, 106, 239);\n test_blend_lowp!(darken_lowp, BlendMode::Darken, 72, 121, 88, 239);\n test_blend_lowp!(lighten_lowp, BlendMode::Lighten, 166, 128, 129, 239);\n // ColorDodge in not available for lowp.\n // ColorBurn in not available for lowp.\n test_blend_lowp!(hard_light_lowp, BlendMode::HardLight, 154, 128, 95, 239);\n // SoftLight in not available for lowp.\n test_blend_lowp!(difference_lowp, BlendMode::Difference, 138, 57, 87, 239);\n test_blend_lowp!(exclusion_lowp, BlendMode::Exclusion, 146, 121, 121, 239);\n test_blend_lowp!(multiply_lowp, BlendMode::Multiply, 69, 90, 71, 238);\n // Hue in not available for lowp.\n // Saturation in not available for lowp.\n // Color in not available for lowp.\n // Luminosity in not available for lowp.\n\n test_blend_highp!(clear_highp, BlendMode::Clear, 0, 0, 0, 0);\n // Source is a no-op\n test_blend_highp!(destination_highp, BlendMode::Destination, 39, 100, 118, 200);\n test_blend_highp!(source_over_highp, BlendMode::SourceOver, 167, 128, 88, 239);\n test_blend_highp!(destination_over_highp, BlendMode::DestinationOver, 72, 121, 129, 239);\n test_blend_highp!(source_in_highp, BlendMode::SourceIn, 122, 78, 42, 141);\n test_blend_highp!(destination_in_highp, BlendMode::DestinationIn, 28, 71, 83, 141);\n test_blend_highp!(source_out_highp, BlendMode::SourceOut, 33, 21, 11, 39);\n test_blend_highp!(destination_out_highp, BlendMode::DestinationOut, 11, 29, 35, 59);\n test_blend_highp!(source_atop_highp, BlendMode::SourceAtop, 133, 107, 76, 200);\n test_blend_highp!(destination_atop_highp, BlendMode::DestinationAtop, 61, 92, 95, 180);\n test_blend_highp!(xor_highp, BlendMode::Xor, 45, 51, 46, 98);\n test_blend_highp!(plus_highp, BlendMode::Plus, 194, 199, 171, 255);\n test_blend_highp!(modulate_highp, BlendMode::Modulate, 24, 39, 24, 141);\n test_blend_highp!(screen_highp, BlendMode::Screen, 171, 160, 146, 239);\n test_blend_highp!(overlay_highp, BlendMode::Overlay, 92, 128, 106, 239);\n test_blend_highp!(darken_highp, BlendMode::Darken, 72, 121, 88, 239);\n test_blend_highp!(lighten_highp, BlendMode::Lighten, 167, 128, 129, 239);\n test_blend_highp!(color_dodge_highp, BlendMode::ColorDodge, 186, 192, 164, 239);\n test_blend_highp!(color_burn_highp, BlendMode::ColorBurn, 54, 63, 46, 239);\n test_blend_highp!(hard_light_highp, BlendMode::HardLight, 155, 128, 95, 239);\n test_blend_highp!(soft_light_highp, BlendMode::SoftLight, 98, 124, 115, 239);\n test_blend_highp!(difference_highp, BlendMode::Difference, 139, 58, 88, 239);\n test_blend_highp!(exclusion_highp, BlendMode::Exclusion, 147, 121, 122, 239);\n test_blend_highp!(multiply_highp, BlendMode::Multiply, 69, 89, 71, 239);\n test_blend_highp!(hue_highp, BlendMode::Hue, 128, 103, 74, 239);\n test_blend_highp!(saturation_highp, BlendMode::Saturation, 59, 126, 140, 239);\n test_blend_highp!(color_highp, BlendMode::Color, 139, 100, 60, 239);\n test_blend_highp!(luminosity_highp, BlendMode::Luminosity, 100, 149, 157, 239);\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/pixmap.rs", "sha256": "4918eda1ac2844e277252f2b3b4113cddb74f471c5e940bec863b9aac6975175", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse alloc::vec;\nuse alloc::vec::Vec;\n\nuse core::convert::TryFrom;\nuse core::num::NonZeroUsize;\n\nuse tiny_skia_path::IntSize;\n\nuse crate::{Color, IntRect};\n\nuse crate::color::PremultipliedColorU8;\nuse crate::geom::{IntSizeExt, ScreenIntRect};\n\n#[cfg(feature = \"png-format\")]\nuse crate::color::{premultiply_u8, ALPHA_U8_OPAQUE};\n\n/// Number of bytes per pixel.\npub const BYTES_PER_PIXEL: usize = 4;\n\n/// A container that owns premultiplied RGBA pixels.\n///\n/// The data is not aligned, therefore width == stride.\n#[derive(Clone, PartialEq)]\npub struct Pixmap {\n data: Vec,\n size: IntSize,\n}\n\nimpl Pixmap {\n /// Allocates a new pixmap.\n ///\n /// A pixmap is filled with transparent black by default, aka (0, 0, 0, 0).\n ///\n /// Zero size in an error.\n ///\n /// Pixmap's width is limited by i32::MAX/4.\n pub fn new(width: u32, height: u32) -> Option {\n let size = IntSize::from_wh(width, height)?;\n let data_len = data_len_for_size(size)?;\n\n // We cannot check that allocation was successful yet.\n // We have to wait for https://github.com/rust-lang/rust/issues/48043\n\n Some(Pixmap {\n data: vec![0; data_len],\n size,\n })\n }\n\n /// Creates a new pixmap by taking ownership over an image buffer\n /// (premultiplied RGBA pixels).\n ///\n /// The size needs to match the data provided.\n ///\n /// Pixmap's width is limited by i32::MAX/4.\n pub fn from_vec(data: Vec, size: IntSize) -> Option {\n let data_len = data_len_for_size(size)?;\n if data.len() != data_len {\n return None;\n }\n\n Some(Pixmap { data, size })\n }\n\n /// Decodes a PNG data into a `Pixmap`.\n ///\n /// Only 8-bit images are supported.\n /// Index PNGs are not supported.\n #[cfg(feature = \"png-format\")]\n pub fn decode_png(data: &[u8]) -> Result {\n fn make_custom_png_error(msg: &str) -> png::DecodingError {\n std::io::Error::new(std::io::ErrorKind::Other, msg).into()\n }\n\n let mut decoder = png::Decoder::new(data);\n decoder.set_transformations(png::Transformations::normalize_to_color8());\n let mut reader = decoder.read_info()?;\n let mut img_data = vec![0; reader.output_buffer_size()];\n let info = reader.next_frame(&mut img_data)?;\n\n if info.bit_depth != png::BitDepth::Eight {\n return Err(make_custom_png_error(\"unsupported bit depth\"));\n }\n\n let size = IntSize::from_wh(info.width, info.height)\n .ok_or_else(|| make_custom_png_error(\"invalid image size\"))?;\n let data_len =\n data_len_for_size(size).ok_or_else(|| make_custom_png_error(\"image is too big\"))?;\n\n img_data = match info.color_type {\n png::ColorType::Rgb => {\n let mut rgba_data = Vec::with_capacity(data_len);\n for rgb in img_data.chunks(3) {\n rgba_data.push(rgb[0]);\n rgba_data.push(rgb[1]);\n rgba_data.push(rgb[2]);\n rgba_data.push(ALPHA_U8_OPAQUE);\n }\n\n rgba_data\n }\n png::ColorType::Rgba => img_data,\n png::ColorType::Grayscale => {\n let mut rgba_data = Vec::with_capacity(data_len);\n for gray in img_data {\n rgba_data.push(gray);\n rgba_data.push(gray);\n rgba_data.push(gray);\n rgba_data.push(ALPHA_U8_OPAQUE);\n }\n\n rgba_data\n }\n png::ColorType::GrayscaleAlpha => {\n let mut rgba_data = Vec::with_capacity(data_len);\n for slice in img_data.chunks(2) {\n let gray = slice[0];\n let alpha = slice[1];\n rgba_data.push(gray);\n rgba_data.push(gray);\n rgba_data.push(gray);\n rgba_data.push(alpha);\n }\n\n rgba_data\n }\n png::ColorType::Indexed => {\n return Err(make_custom_png_error(\"indexed PNG is not supported\"));\n }\n };\n\n // Premultiply alpha.\n //\n // We cannon use RasterPipeline here, which is faster,\n // because it produces slightly different results.\n // Seems like Skia does the same.\n //\n // Also, in our tests unsafe version (no bound checking)\n // had roughly the same performance. So we keep the safe one.\n for pixel in img_data.as_mut_slice().chunks_mut(BYTES_PER_PIXEL) {\n let a = pixel[3];\n pixel[0] = premultiply_u8(pixel[0], a);\n pixel[1] = premultiply_u8(pixel[1], a);\n pixel[2] = premultiply_u8(pixel[2], a);\n }\n\n Pixmap::from_vec(img_data, size)\n .ok_or_else(|| make_custom_png_error(\"failed to create a pixmap\"))\n }\n\n /// Loads a PNG file into a `Pixmap`.\n ///\n /// Only 8-bit images are supported.\n /// Index PNGs are not supported.\n #[cfg(feature = \"png-format\")]\n pub fn load_png>(path: P) -> Result {\n // `png::Decoder` is generic over input, which means that it will instance\n // two copies: one for `&[]` and one for `File`. Which will simply bloat the code.\n // Therefore we're using only one type for input.\n let data = std::fs::read(path)?;\n Self::decode_png(&data)\n }\n\n /// Encodes pixmap into a PNG data.\n #[cfg(feature = \"png-format\")]\n pub fn encode_png(&self) -> Result, png::EncodingError> {\n self.as_ref().encode_png()\n }\n\n /// Saves pixmap as a PNG file.\n #[cfg(feature = \"png-format\")]\n pub fn save_png>(&self, path: P) -> Result<(), png::EncodingError> {\n self.as_ref().save_png(path)\n }\n\n /// Returns a container that references Pixmap's data.\n pub fn as_ref(&self) -> PixmapRef {\n PixmapRef {\n data: &self.data,\n size: self.size,\n }\n }\n\n /// Returns a container that references Pixmap's data.\n pub fn as_mut(&mut self) -> PixmapMut {\n PixmapMut {\n data: &mut self.data,\n size: self.size,\n }\n }\n\n /// Returns pixmap's width.\n #[inline]\n pub fn width(&self) -> u32 {\n self.size.width()\n }\n\n /// Returns pixmap's height.\n #[inline]\n pub fn height(&self) -> u32 {\n self.size.height()\n }\n\n /// Returns pixmap's size.\n #[allow(dead_code)]\n pub(crate) fn size(&self) -> IntSize {\n self.size\n }\n\n /// Fills the entire pixmap with a specified color.\n pub fn fill(&mut self, color: Color) {\n let c = color.premultiply().to_color_u8();\n for p in self.as_mut().pixels_mut() {\n *p = c;\n }\n }\n\n /// Returns the internal data.\n ///\n /// Byteorder: RGBA\n pub fn data(&self) -> &[u8] {\n self.data.as_slice()\n }\n\n /// Returns the mutable internal data.\n ///\n /// Byteorder: RGBA\n pub fn data_mut(&mut self) -> &mut [u8] {\n self.data.as_mut_slice()\n }\n\n /// Returns a pixel color.\n ///\n /// Returns `None` when position is out of bounds.\n pub fn pixel(&self, x: u32, y: u32) -> Option {\n let idx = self.width().checked_mul(y)?.checked_add(x)?;\n self.pixels().get(idx as usize).cloned()\n }\n\n /// Returns a mutable slice of pixels.\n pub fn pixels_mut(&mut self) -> &mut [PremultipliedColorU8] {\n bytemuck::cast_slice_mut(self.data_mut())\n }\n\n /// Returns a slice of pixels.\n pub fn pixels(&self) -> &[PremultipliedColorU8] {\n bytemuck::cast_slice(self.data())\n }\n\n /// Consumes the internal data.\n ///\n /// Byteorder: RGBA\n pub fn take(self) -> Vec {\n self.data\n }\n\n /// Returns a copy of the pixmap that intersects the `rect`.\n ///\n /// Returns `None` when `Pixmap`'s rect doesn't contain `rect`.\n pub fn clone_rect(&self, rect: IntRect) -> Option {\n self.as_ref().clone_rect(rect)\n }\n}\n\nimpl core::fmt::Debug for Pixmap {\n fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n f.debug_struct(\"Pixmap\")\n .field(\"data\", &\"...\")\n .field(\"width\", &self.size.width())\n .field(\"height\", &self.size.height())\n .finish()\n }\n}\n\n/// A container that references premultiplied RGBA pixels.\n///\n/// Can be created from `Pixmap` or from a user provided data.\n///\n/// The data is not aligned, therefore width == stride.\n#[derive(Clone, Copy, PartialEq)]\npub struct PixmapRef<'a> {\n data: &'a [u8],\n size: IntSize,\n}\n\nimpl<'a> PixmapRef<'a> {\n /// Creates a new `PixmapRef` from bytes.\n ///\n /// The size must be at least `size.width() * size.height() * BYTES_PER_PIXEL`.\n /// Zero size in an error. Width is limited by i32::MAX/4.\n ///\n /// The `data` is assumed to have premultiplied RGBA pixels (byteorder: RGBA).\n pub fn from_bytes(data: &'a [u8], width: u32, height: u32) -> Option {\n let size = IntSize::from_wh(width, height)?;\n let data_len = data_len_for_size(size)?;\n if data.len() < data_len {\n return None;\n }\n\n Some(PixmapRef { data, size })\n }\n\n /// Creates a new `Pixmap` from the current data.\n ///\n /// Clones the underlying data.\n pub fn to_owned(&self) -> Pixmap {\n Pixmap {\n data: self.data.to_vec(),\n size: self.size,\n }\n }\n\n /// Returns pixmap's width.\n #[inline]\n pub fn width(&self) -> u32 {\n self.size.width()\n }\n\n /// Returns pixmap's height.\n #[inline]\n pub fn height(&self) -> u32 {\n self.size.height()\n }\n\n /// Returns pixmap's size.\n pub(crate) fn size(&self) -> IntSize {\n self.size\n }\n\n /// Returns pixmap's rect.\n pub(crate) fn rect(&self) -> ScreenIntRect {\n self.size.to_screen_int_rect(0, 0)\n }\n\n /// Returns the internal data.\n ///\n /// Byteorder: RGBA\n pub fn data(&self) -> &'a [u8] {\n self.data\n }\n\n /// Returns a pixel color.\n ///\n /// Returns `None` when position is out of bounds.\n pub fn pixel(&self, x: u32, y: u32) -> Option {\n let idx = self.width().checked_mul(y)?.checked_add(x)?;\n self.pixels().get(idx as usize).cloned()\n }\n\n /// Returns a slice of pixels.\n pub fn pixels(&self) -> &'a [PremultipliedColorU8] {\n bytemuck::cast_slice(self.data())\n }\n\n // TODO: add rows() iterator\n\n /// Returns a copy of the pixmap that intersects the `rect`.\n ///\n /// Returns `None` when `Pixmap`'s rect doesn't contain `rect`.\n pub fn clone_rect(&self, rect: IntRect) -> Option {\n // TODO: to ScreenIntRect?\n\n let rect = self.rect().to_int_rect().intersect(&rect)?;\n let mut new = Pixmap::new(rect.width(), rect.height())?;\n {\n let old_pixels = self.pixels();\n let mut new_mut = new.as_mut();\n let new_pixels = new_mut.pixels_mut();\n\n // TODO: optimize\n for y in 0..rect.height() {\n for x in 0..rect.width() {\n let old_idx = (y + rect.y() as u32) * self.width() + (x + rect.x() as u32);\n let new_idx = y * rect.width() + x;\n new_pixels[new_idx as usize] = old_pixels[old_idx as usize];\n }\n }\n }\n\n Some(new)\n }\n\n /// Encodes pixmap into a PNG data.\n #[cfg(feature = \"png-format\")]\n pub fn encode_png(&self) -> Result, png::EncodingError> {\n // Skia uses skcms here, which is somewhat similar to RasterPipeline.\n\n // Sadly, we have to copy the pixmap here, because of demultiplication.\n // Not sure how to avoid this.\n // TODO: remove allocation\n let mut tmp_pixmap = self.to_owned();\n\n // Demultiply alpha.\n //\n // RasterPipeline is 15% faster here, but produces slightly different results\n // due to rounding. So we stick with this method for now.\n for pixel in tmp_pixmap.pixels_mut() {\n let c = pixel.demultiply();\n *pixel =\n PremultipliedColorU8::from_rgba_unchecked(c.red(), c.green(), c.blue(), c.alpha());\n }\n\n let mut data = Vec::new();\n {\n let mut encoder = png::Encoder::new(&mut data, self.width(), self.height());\n encoder.set_color(png::ColorType::Rgba);\n encoder.set_depth(png::BitDepth::Eight);\n let mut writer = encoder.write_header()?;\n writer.write_image_data(&tmp_pixmap.data)?;\n }\n\n Ok(data)\n }\n\n /// Saves pixmap as a PNG file.\n #[cfg(feature = \"png-format\")]\n pub fn save_png>(&self, path: P) -> Result<(), png::EncodingError> {\n let data = self.encode_png()?;\n std::fs::write(path, data)?;\n Ok(())\n }\n}\n\nimpl core::fmt::Debug for PixmapRef<'_> {\n fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n f.debug_struct(\"PixmapRef\")\n .field(\"data\", &\"...\")\n .field(\"width\", &self.size.width())\n .field(\"height\", &self.size.height())\n .finish()\n }\n}\n\n/// A container that references mutable premultiplied RGBA pixels.\n///\n/// Can be created from `Pixmap` or from a user provided data.\n///\n/// The data is not aligned, therefore width == stride.\n#[derive(PartialEq)]\npub struct PixmapMut<'a> {\n data: &'a mut [u8],\n size: IntSize,\n}\n\nimpl<'a> PixmapMut<'a> {\n /// Creates a new `PixmapMut` from bytes.\n ///\n /// The size must be at least `size.width() * size.height() * BYTES_PER_PIXEL`.\n /// Zero size in an error. Width is limited by i32::MAX/4.\n ///\n /// The `data` is assumed to have premultiplied RGBA pixels (byteorder: RGBA).\n pub fn from_bytes(data: &'a mut [u8], width: u32, height: u32) -> Option {\n let size = IntSize::from_wh(width, height)?;\n let data_len = data_len_for_size(size)?;\n if data.len() < data_len {\n return None;\n }\n\n Some(PixmapMut { data, size })\n }\n\n /// Creates a new `Pixmap` from the current data.\n ///\n /// Clones the underlying data.\n pub fn to_owned(&self) -> Pixmap {\n Pixmap {\n data: self.data.to_vec(),\n size: self.size,\n }\n }\n\n /// Returns a container that references Pixmap's data.\n pub fn as_ref(&self) -> PixmapRef {\n PixmapRef {\n data: self.data,\n size: self.size,\n }\n }\n\n /// Returns pixmap's width.\n #[inline]\n pub fn width(&self) -> u32 {\n self.size.width()\n }\n\n /// Returns pixmap's height.\n #[inline]\n pub fn height(&self) -> u32 {\n self.size.height()\n }\n\n /// Returns pixmap's size.\n pub(crate) fn size(&self) -> IntSize {\n self.size\n }\n\n /// Fills the entire pixmap with a specified color.\n pub fn fill(&mut self, color: Color) {\n let c = color.premultiply().to_color_u8();\n for p in self.pixels_mut() {\n *p = c;\n }\n }\n\n /// Returns the mutable internal data.\n ///\n /// Byteorder: RGBA\n pub fn data_mut(&mut self) -> &mut [u8] {\n self.data\n }\n\n /// Returns a mutable slice of pixels.\n pub fn pixels_mut(&mut self) -> &mut [PremultipliedColorU8] {\n bytemuck::cast_slice_mut(self.data_mut())\n }\n\n /// Creates `SubPixmapMut` that contains the whole `PixmapMut`.\n pub(crate) fn as_subpixmap(&mut self) -> SubPixmapMut {\n SubPixmapMut {\n size: self.size(),\n real_width: self.width() as usize,\n data: &mut self.data,\n }\n }\n\n /// Returns a mutable reference to the pixmap region that intersects the `rect`.\n ///\n /// Returns `None` when `Pixmap`'s rect doesn't contain `rect`.\n pub(crate) fn subpixmap(&mut self, rect: IntRect) -> Option {\n let rect = self.size.to_int_rect(0, 0).intersect(&rect)?;\n let row_bytes = self.width() as usize * BYTES_PER_PIXEL;\n let offset = rect.top() as usize * row_bytes + rect.left() as usize * BYTES_PER_PIXEL;\n\n Some(SubPixmapMut {\n size: rect.size(),\n real_width: self.width() as usize,\n data: &mut self.data[offset..],\n })\n }\n}\n\nimpl core::fmt::Debug for PixmapMut<'_> {\n fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n f.debug_struct(\"PixmapMut\")\n .field(\"data\", &\"...\")\n .field(\"width\", &self.size.width())\n .field(\"height\", &self.size.height())\n .finish()\n }\n}\n\n/// A `PixmapMut` subregion.\n///\n/// Unlike `PixmapMut`, contains `real_width` which references the parent `PixmapMut` width.\n/// This way we can operate on a `PixmapMut` subregion without reallocations.\n/// Primarily required because of `DrawTiler`.\n///\n/// We cannot implement it in `PixmapMut` directly, because it will brake `fill`, `data_mut`\n/// `pixels_mut` and other similar methods.\n/// This is because `SubPixmapMut.data` references more \"data\" than it actually allowed to access.\n/// On the other hand, `PixmapMut.data` can access all it's data and it's stored linearly.\npub struct SubPixmapMut<'a> {\n pub data: &'a mut [u8],\n pub size: IntSize,\n pub real_width: usize,\n}\n\nimpl<'a> SubPixmapMut<'a> {\n /// Returns a mutable slice of pixels.\n pub fn pixels_mut(&mut self) -> &mut [PremultipliedColorU8] {\n bytemuck::cast_slice_mut(self.data)\n }\n}\n\n/// Returns minimum bytes per row as usize.\n///\n/// Pixmap's maximum value for row bytes must fit in 31 bits.\nfn min_row_bytes(size: IntSize) -> Option {\n let w = i32::try_from(size.width()).ok()?;\n let w = w.checked_mul(BYTES_PER_PIXEL as i32)?;\n NonZeroUsize::new(w as usize)\n}\n\n/// Returns storage size required by pixel array.\nfn compute_data_len(size: IntSize, row_bytes: usize) -> Option {\n let h = size.height().checked_sub(1)?;\n let h = (h as usize).checked_mul(row_bytes)?;\n\n let w = (size.width() as usize).checked_mul(BYTES_PER_PIXEL)?;\n\n h.checked_add(w)\n}\n\nfn data_len_for_size(size: IntSize) -> Option {\n let row_bytes = min_row_bytes(size)?;\n compute_data_len(size, row_bytes.get())\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/scan/hairline.rs", "sha256": "aa50d030fc207e198d0b106b9792dd9fc606bd6ccc11384327ce768d879ab5b1", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse core::convert::TryInto;\n\nuse tiny_skia_path::{f32x2, PathVerb, SaturateCast, Scalar};\n\nuse crate::{IntRect, LineCap, Path, PathSegment, Point, Rect};\n\nuse crate::blitter::Blitter;\nuse crate::fixed_point::{fdot16, fdot6};\nuse crate::geom::ScreenIntRect;\nuse crate::line_clipper;\nuse crate::math::LENGTH_U32_ONE;\nuse crate::path_geometry;\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\nconst FLOAT_PI: f32 = 3.14159265;\n\npub type LineProc = fn(&[Point], Option<&ScreenIntRect>, &mut dyn Blitter);\n\nconst MAX_CUBIC_SUBDIVIDE_LEVEL: u8 = 9;\nconst MAX_QUAD_SUBDIVIDE_LEVEL: u8 = 5;\n\npub fn stroke_path(\n path: &Path,\n line_cap: LineCap,\n clip: &ScreenIntRect,\n blitter: &mut dyn Blitter,\n) {\n super::hairline::stroke_path_impl(path, line_cap, clip, hair_line_rgn, blitter)\n}\n\nfn hair_line_rgn(points: &[Point], clip: Option<&ScreenIntRect>, blitter: &mut dyn Blitter) {\n let max = 32767.0;\n let fixed_bounds = Rect::from_ltrb(-max, -max, max, max).unwrap();\n\n let clip_bounds = clip.map(|c| c.to_rect());\n\n for i in 0..points.len() - 1 {\n let mut pts = [Point::zero(); 2];\n\n // We have to pre-clip the line to fit in a Fixed, so we just chop the line.\n if !line_clipper::intersect(&[points[i], points[i + 1]], &fixed_bounds, &mut pts) {\n continue;\n }\n\n if let Some(clip_bounds) = clip_bounds {\n let tmp = pts.clone();\n // Perform a clip in scalar space, so we catch huge values which might\n // be missed after we convert to FDot6 (overflow).\n if !line_clipper::intersect(&tmp, &clip_bounds, &mut pts) {\n continue;\n }\n }\n\n let mut x0 = fdot6::from_f32(pts[0].x);\n let mut y0 = fdot6::from_f32(pts[0].y);\n let mut x1 = fdot6::from_f32(pts[1].x);\n let mut y1 = fdot6::from_f32(pts[1].y);\n\n debug_assert!(fdot6::can_convert_to_fdot16(x0));\n debug_assert!(fdot6::can_convert_to_fdot16(y0));\n debug_assert!(fdot6::can_convert_to_fdot16(x1));\n debug_assert!(fdot6::can_convert_to_fdot16(y1));\n\n let dx = x1 - x0;\n let dy = y1 - y0;\n\n if dx.abs() > dy.abs() {\n // mostly horizontal\n\n if x0 > x1 {\n // we want to go left-to-right\n core::mem::swap(&mut x0, &mut x1);\n core::mem::swap(&mut y0, &mut y1);\n }\n\n let mut ix0 = fdot6::round(x0);\n let ix1 = fdot6::round(x1);\n if ix0 == ix1 {\n // too short to draw\n continue;\n }\n\n let slope = fdot16::div(dy, dx);\n #[allow(clippy::precedence)]\n let mut start_y = fdot6::to_fdot16(y0) + (slope * ((32 - x0) & 63) >> 6);\n\n // In some cases, probably due to precision/rounding issues,\n // `start_y` can become equal to the image height,\n // which will lead to panic, because we would be accessing pixels outside\n // the current memory buffer.\n // This is tiny-skia specific issue. Skia handles this part differently.\n let max_y = if let Some(clip_bounds) = clip_bounds {\n fdot16::from_f32(clip_bounds.bottom())\n } else {\n i32::MAX\n };\n\n debug_assert!(ix0 < ix1);\n loop {\n if ix0 >= 0 && start_y >= 0 && start_y < max_y {\n blitter.blit_h(ix0 as u32, (start_y >> 16) as u32, LENGTH_U32_ONE);\n }\n\n start_y += slope;\n ix0 += 1;\n if ix0 >= ix1 {\n break;\n }\n }\n } else {\n // mostly vertical\n\n if y0 > y1 {\n // we want to go top-to-bottom\n core::mem::swap(&mut x0, &mut x1);\n core::mem::swap(&mut y0, &mut y1);\n }\n\n let mut iy0 = fdot6::round(y0);\n let iy1 = fdot6::round(y1);\n if iy0 == iy1 {\n // too short to draw\n continue;\n }\n\n let slope = fdot16::div(dx, dy);\n #[allow(clippy::precedence)]\n let mut start_x = fdot6::to_fdot16(x0) + (slope * ((32 - y0) & 63) >> 6);\n\n debug_assert!(iy0 < iy1);\n loop {\n if start_x >= 0 && iy0 >= 0 {\n blitter.blit_h((start_x >> 16) as u32, iy0 as u32, LENGTH_U32_ONE);\n }\n\n start_x += slope;\n iy0 += 1;\n if iy0 >= iy1 {\n break;\n }\n }\n }\n }\n}\n\npub fn stroke_path_impl(\n path: &Path,\n line_cap: LineCap,\n clip: &ScreenIntRect,\n line_proc: LineProc,\n blitter: &mut dyn Blitter,\n) {\n let mut inset_clip = None;\n let mut outset_clip = None;\n\n {\n let cap_out = if line_cap == LineCap::Butt { 1.0 } else { 2.0 };\n let ibounds = match path\n .bounds()\n .outset(cap_out, cap_out)\n .and_then(|r| r.round_out())\n {\n Some(v) => v,\n None => return,\n };\n if clip.to_int_rect().intersect(&ibounds).is_none() {\n return;\n }\n\n if !clip.to_int_rect().contains(&ibounds) {\n // We now cache two scalar rects, to use for culling per-segment (e.g. cubic).\n // Since we're hairlining, the \"bounds\" of the control points isn't necessairly the\n // limit of where a segment can draw (it might draw up to 1 pixel beyond in aa-hairs).\n //\n // Compute the pt-bounds per segment is easy, so we do that, and then inversely adjust\n // the culling bounds so we can just do a straight compare per segment.\n //\n // insetClip is use for quick-accept (i.e. the segment is not clipped), so we inset\n // it from the clip-bounds (since segment bounds can be off by 1).\n //\n // outsetClip is used for quick-reject (i.e. the segment is entirely outside), so we\n // outset it from the clip-bounds.\n match clip.to_int_rect().make_outset(1, 1) {\n Some(v) => outset_clip = Some(v),\n None => return,\n }\n match clip.to_int_rect().inset(1, 1) {\n Some(v) => inset_clip = Some(v),\n None => return,\n }\n }\n }\n\n let clip = Some(clip);\n let mut prev_verb = PathVerb::Move;\n let mut first_pt = Point::zero();\n let mut last_pt = Point::zero();\n\n let mut iter = path.segments();\n while let Some(segment) = iter.next() {\n let verb = iter.curr_verb();\n let next_verb = iter.next_verb();\n let last_pt2;\n match segment {\n PathSegment::MoveTo(p) => {\n first_pt = p;\n last_pt = p;\n last_pt2 = p;\n }\n PathSegment::LineTo(p) => {\n let mut points = [last_pt, p];\n if line_cap != LineCap::Butt {\n extend_pts(line_cap, prev_verb, next_verb, &mut points);\n }\n\n line_proc(&points, clip, blitter);\n last_pt = p;\n last_pt2 = points[0];\n }\n PathSegment::QuadTo(p0, p1) => {\n let mut points = [last_pt, p0, p1];\n if line_cap != LineCap::Butt {\n extend_pts(line_cap, prev_verb, next_verb, &mut points);\n }\n\n hair_quad(\n &points,\n clip,\n inset_clip.as_ref(),\n outset_clip.as_ref(),\n compute_quad_level(&points),\n line_proc,\n blitter,\n );\n\n last_pt = p1;\n last_pt2 = points[0];\n }\n PathSegment::CubicTo(p0, p1, p2) => {\n let mut points = [last_pt, p0, p1, p2];\n if line_cap != LineCap::Butt {\n extend_pts(line_cap, prev_verb, next_verb, &mut points);\n }\n\n hair_cubic(\n &points,\n clip,\n inset_clip.as_ref(),\n outset_clip.as_ref(),\n line_proc,\n blitter,\n );\n\n last_pt = p2;\n last_pt2 = points[0];\n }\n PathSegment::Close => {\n let mut points = [last_pt, first_pt];\n if line_cap != LineCap::Butt && prev_verb == PathVerb::Move {\n // cap moveTo/close to match svg expectations for degenerate segments\n extend_pts(line_cap, prev_verb, next_verb, &mut points);\n }\n line_proc(&points, clip, blitter);\n last_pt2 = points[0];\n }\n }\n\n if line_cap != LineCap::Butt {\n if prev_verb == PathVerb::Move\n && matches!(verb, PathVerb::Line | PathVerb::Quad | PathVerb::Cubic)\n {\n first_pt = last_pt2; // the curve moved the initial point, so close to it instead\n }\n\n prev_verb = verb;\n }\n }\n}\n\n/// Extend the points in the direction of the starting or ending tangent by 1/2 unit to\n/// account for a round or square cap.\n///\n/// If there's no distance between the end point and\n/// the control point, use the next control point to create a tangent. If the curve\n/// is degenerate, move the cap out 1/2 unit horizontally.\nfn extend_pts(\n line_cap: LineCap,\n prev_verb: PathVerb,\n next_verb: Option,\n points: &mut [Point],\n) {\n debug_assert!(!points.is_empty()); // TODO: use non-zero slice\n debug_assert!(line_cap != LineCap::Butt);\n\n // The area of a circle is PI*R*R. For a unit circle, R=1/2, and the cap covers half of that.\n let cap_outset = if line_cap == LineCap::Square {\n 0.5\n } else {\n FLOAT_PI / 8.0\n };\n if prev_verb == PathVerb::Move {\n let first = points[0];\n let mut offset = 0;\n let mut controls = points.len() - 1;\n let mut tangent;\n loop {\n offset += 1;\n tangent = first - points[offset];\n\n if !tangent.is_zero() {\n break;\n }\n\n controls -= 1;\n if controls == 0 {\n break;\n }\n }\n\n if tangent.is_zero() {\n tangent = Point::from_xy(1.0, 0.0);\n controls = points.len() - 1; // If all points are equal, move all but one.\n } else {\n tangent.normalize();\n }\n\n offset = 0;\n loop {\n // If the end point and control points are equal, loop to move them in tandem.\n points[offset].x += tangent.x * cap_outset;\n points[offset].y += tangent.y * cap_outset;\n\n offset += 1;\n controls += 1;\n if controls >= points.len() {\n break;\n }\n }\n }\n\n if matches!(\n next_verb,\n Some(PathVerb::Move) | Some(PathVerb::Close) | None\n ) {\n let last = points.last().unwrap().clone();\n let mut offset = points.len() - 1;\n let mut controls = points.len() - 1;\n let mut tangent;\n loop {\n offset -= 1;\n tangent = last - points[offset];\n\n if !tangent.is_zero() {\n break;\n }\n\n controls -= 1;\n if controls == 0 {\n break;\n }\n }\n\n if tangent.is_zero() {\n tangent = Point::from_xy(-1.0, 0.0);\n controls = points.len() - 1;\n } else {\n tangent.normalize();\n }\n\n offset = points.len() - 1;\n loop {\n points[offset].x += tangent.x * cap_outset;\n points[offset].y += tangent.y * cap_outset;\n\n offset -= 1;\n controls += 1;\n if controls >= points.len() {\n break;\n }\n }\n }\n}\n\nfn hair_quad(\n points: &[Point; 3],\n mut clip: Option<&ScreenIntRect>,\n inset_clip: Option<&IntRect>,\n outset_clip: Option<&IntRect>,\n level: u8,\n line_proc: LineProc,\n blitter: &mut dyn Blitter,\n) {\n if let Some(inset_clip) = inset_clip {\n debug_assert!(outset_clip.is_some());\n let inset_clip = inset_clip.to_rect();\n let outset_clip = match outset_clip {\n Some(v) => v.to_rect(),\n None => return,\n };\n\n let bounds = match compute_nocheck_quad_bounds(points) {\n Some(v) => v,\n None => return,\n };\n if !geometric_overlap(&outset_clip, &bounds) {\n return; // nothing to do\n } else if geometric_contains(&inset_clip, &bounds) {\n clip = None;\n }\n }\n\n hair_quad2(points, clip, level, line_proc, blitter);\n}\n\nfn compute_nocheck_quad_bounds(points: &[Point; 3]) -> Option {\n debug_assert!(points[0].is_finite());\n debug_assert!(points[1].is_finite());\n debug_assert!(points[2].is_finite());\n\n let mut min = points[0].to_f32x2();\n let mut max = min;\n for i in 1..3 {\n let pair = points[i].to_f32x2();\n min = min.min(pair);\n max = max.max(pair);\n }\n\n Rect::from_ltrb(min.x(), min.y(), max.x(), max.y())\n}\n\nfn geometric_overlap(a: &Rect, b: &Rect) -> bool {\n a.left() < b.right() && b.left() < a.right() && a.top() < b.bottom() && b.top() < a.bottom()\n}\n\nfn geometric_contains(outer: &Rect, inner: &Rect) -> bool {\n inner.right() <= outer.right()\n && inner.left() >= outer.left()\n && inner.bottom() <= outer.bottom()\n && inner.top() >= outer.top()\n}\n\nfn hair_quad2(\n points: &[Point; 3],\n clip: Option<&ScreenIntRect>,\n level: u8,\n line_proc: LineProc,\n blitter: &mut dyn Blitter,\n) {\n debug_assert!(level <= MAX_QUAD_SUBDIVIDE_LEVEL); // TODO: to type\n\n let coeff = path_geometry::QuadCoeff::from_points(points);\n\n const MAX_POINTS: usize = (1 << MAX_QUAD_SUBDIVIDE_LEVEL) + 1;\n let lines = 1 << level;\n debug_assert!(lines < MAX_POINTS);\n\n let mut tmp = [Point::zero(); MAX_POINTS];\n tmp[0] = points[0];\n\n let mut t = f32x2::default();\n let dt = f32x2::splat(1.0 / lines as f32);\n for i in 1..lines {\n t = t + dt;\n let v = (coeff.a * t + coeff.b) * t + coeff.c;\n tmp[i] = Point::from_xy(v.x(), v.y());\n }\n\n tmp[lines] = points[2];\n line_proc(&tmp[0..lines + 1], clip, blitter);\n}\n\nfn compute_quad_level(points: &[Point; 3]) -> u8 {\n let d = compute_int_quad_dist(points);\n // Quadratics approach the line connecting their start and end points\n // 4x closer with each subdivision, so we compute the number of\n // subdivisions to be the minimum need to get that distance to be less\n // than a pixel.\n let mut level = (33 - d.leading_zeros()) >> 1;\n // sanity check on level (from the previous version)\n if level > MAX_QUAD_SUBDIVIDE_LEVEL as u32 {\n level = MAX_QUAD_SUBDIVIDE_LEVEL as u32;\n }\n\n level as u8\n}\n\nfn compute_int_quad_dist(points: &[Point; 3]) -> u32 {\n // compute the vector between the control point ([1]) and the middle of the\n // line connecting the start and end ([0] and [2])\n let dx = ((points[0].x + points[2].x).half() - points[1].x).abs();\n let dy = ((points[0].y + points[2].y).half() - points[1].y).abs();\n\n // convert to whole pixel values (use ceiling to be conservative).\n // assign to unsigned so we can safely add 1/2 of the smaller and still fit in\n // u32, since T::saturate_from() returns 31 bits at most.\n let idx = i32::saturate_from(dx.ceil()) as u32;\n let idy = i32::saturate_from(dy.ceil()) as u32;\n\n // use the cheap approx for distance\n if idx > idy {\n idx + (idy >> 1)\n } else {\n idy + (idx >> 1)\n }\n}\n\nfn hair_cubic(\n points: &[Point; 4],\n mut clip: Option<&ScreenIntRect>,\n inset_clip: Option<&IntRect>,\n outset_clip: Option<&IntRect>,\n line_proc: LineProc,\n blitter: &mut dyn Blitter,\n) {\n if let Some(inset_clip) = inset_clip {\n debug_assert!(outset_clip.is_some());\n let inset_clip = inset_clip.to_rect();\n let outset_clip = match outset_clip {\n Some(v) => v.to_rect(),\n None => return,\n };\n\n let bounds = match compute_nocheck_cubic_bounds(points) {\n Some(v) => v,\n None => return,\n };\n if !geometric_overlap(&outset_clip, &bounds) {\n return; // noting to do\n } else if geometric_contains(&inset_clip, &bounds) {\n clip = None;\n }\n }\n\n if quick_cubic_niceness_check(points) {\n hair_cubic2(points, clip, line_proc, blitter);\n } else {\n let mut tmp = [Point::zero(); 13];\n let mut t_values = path_geometry::new_t_values();\n\n let count = path_geometry::chop_cubic_at_max_curvature(points, &mut t_values, &mut tmp);\n for i in 0..count {\n let offset = i * 3;\n let new_points: [Point; 4] = tmp[offset..offset + 4].try_into().unwrap();\n hair_cubic2(&new_points, clip, line_proc, blitter);\n }\n }\n}\n\nfn compute_nocheck_cubic_bounds(points: &[Point; 4]) -> Option {\n debug_assert!(points[0].is_finite());\n debug_assert!(points[1].is_finite());\n debug_assert!(points[2].is_finite());\n debug_assert!(points[3].is_finite());\n\n let mut min = points[0].to_f32x2();\n let mut max = min;\n for i in 1..4 {\n let pair = points[i].to_f32x2();\n min = min.min(pair);\n max = max.max(pair);\n }\n\n Rect::from_ltrb(min.x(), min.y(), max.x(), max.y())\n}\n\n// The off-curve points are \"inside\" the limits of the on-curve points.\nfn quick_cubic_niceness_check(points: &[Point; 4]) -> bool {\n lt_90(points[1], points[0], points[3])\n && lt_90(points[2], points[0], points[3])\n && lt_90(points[1], points[3], points[0])\n && lt_90(points[2], points[3], points[0])\n}\n\nfn lt_90(p0: Point, pivot: Point, p2: Point) -> bool {\n (p0 - pivot).dot(p2 - pivot) >= 0.0\n}\n\nfn hair_cubic2(\n points: &[Point; 4],\n clip: Option<&ScreenIntRect>,\n line_proc: LineProc,\n blitter: &mut dyn Blitter,\n) {\n let lines = compute_cubic_segments(points);\n debug_assert!(lines > 0);\n if lines == 1 {\n line_proc(&[points[0], points[3]], clip, blitter);\n return;\n }\n\n let coeff = path_geometry::CubicCoeff::from_points(points);\n\n const MAX_POINTS: usize = (1 << MAX_CUBIC_SUBDIVIDE_LEVEL) + 1;\n debug_assert!(lines < MAX_POINTS);\n let mut tmp = [Point::zero(); MAX_POINTS];\n\n let dt = f32x2::splat(1.0 / lines as f32);\n let mut t = f32x2::default();\n\n tmp[0] = points[0];\n for i in 1..lines {\n t = t + dt;\n tmp[i] = Point::from_f32x2(((coeff.a * t + coeff.b) * t + coeff.c) * t + coeff.d);\n }\n\n if tmp.iter().all(|p| p.is_finite()) {\n tmp[lines] = points[3];\n line_proc(&tmp[0..lines + 1], clip, blitter);\n } else {\n // else some point(s) are non-finite, so don't draw\n }\n}\n\nfn compute_cubic_segments(points: &[Point; 4]) -> usize {\n let p0 = points[0].to_f32x2();\n let p1 = points[1].to_f32x2();\n let p2 = points[2].to_f32x2();\n let p3 = points[3].to_f32x2();\n\n let one_third = f32x2::splat(1.0 / 3.0);\n let two_third = f32x2::splat(2.0 / 3.0);\n\n let p13 = one_third * p3 + two_third * p0;\n let p23 = one_third * p0 + two_third * p3;\n\n let diff = (p1 - p13).abs().max((p2 - p23).abs()).max_component();\n let mut tol = 1.0 / 8.0;\n\n for i in 0..MAX_CUBIC_SUBDIVIDE_LEVEL {\n if diff < tol {\n return 1 << i;\n }\n\n tol *= 4.0;\n }\n\n 1 << MAX_CUBIC_SUBDIVIDE_LEVEL\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/scan/hairline_aa.rs", "sha256": "50f1948aa413647e5ad8a3d2020cf758c29bd5b54f8c829190d453fad8e39549", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2011 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse core::convert::TryFrom;\nuse core::num::NonZeroU16;\n\nuse crate::{IntRect, LengthU32, LineCap, Path, Point, Rect};\n\nuse crate::alpha_runs::{AlphaRun, AlphaRuns};\nuse crate::blitter::Blitter;\nuse crate::color::AlphaU8;\nuse crate::fixed_point::{fdot16, fdot6, fdot8, FDot16, FDot6, FDot8};\nuse crate::geom::{IntRectExt, ScreenIntRect};\nuse crate::line_clipper;\nuse crate::math::LENGTH_U32_ONE;\n\n#[derive(Copy, Clone, Debug)]\nstruct FixedRect {\n left: FDot16,\n top: FDot16,\n right: FDot16,\n bottom: FDot16,\n}\n\nimpl FixedRect {\n fn from_rect(src: &Rect) -> Self {\n FixedRect {\n left: fdot16::from_f32(src.left()),\n top: fdot16::from_f32(src.top()),\n right: fdot16::from_f32(src.right()),\n bottom: fdot16::from_f32(src.bottom()),\n }\n }\n}\n\n/// Multiplies value by 0..256, and shift the result down 8\n/// (i.e. return (value * alpha256) >> 8)\nfn alpha_mul(value: AlphaU8, alpha256: i32) -> u8 {\n let a = (i32::from(value) * alpha256) >> 8;\n debug_assert!(a >= 0 && a <= 255);\n a as u8\n}\n\npub fn fill_rect(rect: &Rect, clip: &ScreenIntRect, blitter: &mut dyn Blitter) {\n let rect = match rect.intersect(&clip.to_rect()) {\n Some(v) => v,\n None => return, // everything was clipped out\n };\n\n let fr = FixedRect::from_rect(&rect);\n fill_fixed_rect(&fr, blitter);\n}\n\nfn fill_fixed_rect(rect: &FixedRect, blitter: &mut dyn Blitter) {\n fill_dot8(\n fdot8::from_fdot16(rect.left),\n fdot8::from_fdot16(rect.top),\n fdot8::from_fdot16(rect.right),\n fdot8::from_fdot16(rect.bottom),\n true,\n blitter,\n )\n}\n\nfn fill_dot8(l: FDot8, t: FDot8, r: FDot8, b: FDot8, fill_inner: bool, blitter: &mut dyn Blitter) {\n fn to_alpha(a: i32) -> u8 {\n debug_assert!(a >= 0 && a <= 255);\n a as u8\n }\n\n // check for empty now that we're in our reduced precision space\n if l >= r || t >= b {\n return;\n }\n\n let mut top = t >> 8;\n if top == ((b - 1) >> 8) {\n // just one scanline high\n do_scanline(l, top, r, to_alpha(b - t - 1), blitter);\n return;\n }\n\n if t & 0xFF != 0 {\n do_scanline(l, top, r, to_alpha(256 - (t & 0xFF)), blitter);\n top += 1;\n }\n\n let bottom = b >> 8;\n let height = bottom - top;\n if let Some(height) = u32::try_from(height).ok().and_then(LengthU32::new) {\n let mut left = l >> 8;\n if left == ((r - 1) >> 8) {\n // just 1-pixel wide\n if let (Ok(left), Ok(top)) = (u32::try_from(left), u32::try_from(top)) {\n blitter.blit_v(left, top, height, to_alpha(r - l - 1));\n } else {\n debug_assert!(false);\n }\n } else {\n if l & 0xFF != 0 {\n if let (Ok(left), Ok(top)) = (u32::try_from(left), u32::try_from(top)) {\n blitter.blit_v(left, top, height, to_alpha(256 - (l & 0xFF)));\n } else {\n debug_assert!(false);\n }\n\n left += 1;\n }\n\n let right = r >> 8;\n let width = right - left;\n if fill_inner {\n if let Some(width) = u32::try_from(width).ok().and_then(LengthU32::new) {\n if let (Ok(left), Ok(top)) = (u32::try_from(left), u32::try_from(top)) {\n let rect = ScreenIntRect::from_xywh_safe(left, top, width, height);\n blitter.blit_rect(&rect);\n } else {\n debug_assert!(false);\n }\n } else {\n debug_assert!(false);\n }\n }\n\n if r & 0xFF != 0 {\n if let (Ok(right), Ok(top)) = (u32::try_from(right), u32::try_from(top)) {\n blitter.blit_v(right, top, height, to_alpha(r & 0xFF));\n } else {\n debug_assert!(false);\n }\n }\n }\n }\n\n if b & 0xFF != 0 {\n do_scanline(l, bottom, r, to_alpha(b & 0xFF), blitter);\n }\n}\n\nfn do_scanline(l: FDot8, top: i32, r: FDot8, alpha: AlphaU8, blitter: &mut dyn Blitter) {\n debug_assert!(l < r);\n\n let one_len = LENGTH_U32_ONE;\n let top = match u32::try_from(top) {\n Ok(n) => n,\n _ => return,\n };\n\n if (l >> 8) == ((r - 1) >> 8) {\n // 1x1 pixel\n if let Ok(left) = u32::try_from(l >> 8) {\n blitter.blit_v(left, top, one_len, alpha_mul(alpha, r - l));\n }\n\n return;\n }\n\n let mut left = l >> 8;\n\n if l & 0xFF != 0 {\n if let Ok(left) = u32::try_from(l >> 8) {\n blitter.blit_v(left, top, one_len, alpha_mul(alpha, 256 - (l & 0xFF)));\n }\n\n left += 1;\n }\n\n let right = r >> 8;\n let width = right - left;\n if let Some(width) = u32::try_from(width).ok().and_then(LengthU32::new) {\n if let Ok(left) = u32::try_from(left) {\n call_hline_blitter(left, Some(top), width, alpha, blitter);\n }\n }\n\n if r & 0xFF != 0 {\n if let Ok(right) = u32::try_from(right) {\n blitter.blit_v(right, top, one_len, alpha_mul(alpha, r & 0xFF));\n }\n }\n}\n\nfn call_hline_blitter(\n mut x: u32,\n y: Option,\n count: LengthU32,\n alpha: AlphaU8,\n blitter: &mut dyn Blitter,\n) {\n const HLINE_STACK_BUFFER: usize = 100;\n\n let mut runs = [None; HLINE_STACK_BUFFER + 1];\n let mut aa = [0u8; HLINE_STACK_BUFFER];\n\n let mut count = count.get();\n loop {\n // In theory, we should be able to just do this once (outside of the loop),\n // since aa[] and runs[] are supposed\" to be const when we call the blitter.\n // In reality, some wrapper-blitters (e.g. RgnClipBlitter) cast away that\n // constness, and modify the buffers in-place. Hence the need to be defensive\n // here and reseed the aa value.\n aa[0] = alpha;\n\n let mut n = count;\n if n > HLINE_STACK_BUFFER as u32 {\n n = HLINE_STACK_BUFFER as u32;\n }\n\n debug_assert!(n <= core::u16::MAX as u32);\n runs[0] = NonZeroU16::new(n as u16);\n runs[n as usize] = None;\n if let Some(y) = y {\n blitter.blit_anti_h(x, y, &mut aa, &mut runs);\n }\n x += n;\n\n if n >= count || count == 0 {\n break;\n }\n\n count -= n;\n }\n}\n\npub fn stroke_path(\n path: &Path,\n line_cap: LineCap,\n clip: &ScreenIntRect,\n blitter: &mut dyn Blitter,\n) {\n super::hairline::stroke_path_impl(path, line_cap, clip, anti_hair_line_rgn, blitter);\n}\n\nfn anti_hair_line_rgn(points: &[Point], clip: Option<&ScreenIntRect>, blitter: &mut dyn Blitter) {\n let max = 32767.0;\n let fixed_bounds = Rect::from_ltrb(-max, -max, max, max).unwrap();\n\n let clip_bounds = if let Some(clip) = clip {\n // We perform integral clipping later on, but we do a scalar clip first\n // to ensure that our coordinates are expressible in fixed/integers.\n //\n // antialiased hairlines can draw up to 1/2 of a pixel outside of\n // their bounds, so we need to outset the clip before calling the\n // clipper. To make the numerics safer, we outset by a whole pixel,\n // since the 1/2 pixel boundary is important to the antihair blitter,\n // we don't want to risk numerical fate by chopping on that edge.\n clip.to_rect().outset(1.0, 1.0)\n } else {\n None\n };\n\n for i in 0..points.len() - 1 {\n let mut pts = [Point::zero(); 2];\n\n // We have to pre-clip the line to fit in a Fixed, so we just chop the line.\n if !line_clipper::intersect(&[points[i], points[i + 1]], &fixed_bounds, &mut pts) {\n continue;\n }\n\n if let Some(clip_bounds) = clip_bounds {\n let tmp = pts;\n if !line_clipper::intersect(&tmp, &clip_bounds, &mut pts) {\n continue;\n }\n }\n\n let x0 = fdot6::from_f32(pts[0].x);\n let y0 = fdot6::from_f32(pts[0].y);\n let x1 = fdot6::from_f32(pts[1].x);\n let y1 = fdot6::from_f32(pts[1].y);\n\n if let Some(clip) = clip {\n let left = x0.min(x1);\n let top = y0.min(y1);\n let right = x0.max(x1);\n let bottom = y0.max(y1);\n\n let ir = IntRect::from_ltrb(\n fdot6::floor(left) - 1,\n fdot6::floor(top) - 1,\n fdot6::ceil(right) + 1,\n fdot6::ceil(bottom) + 1,\n );\n let ir = match ir {\n Some(v) => v,\n None => return,\n };\n\n if clip.to_int_rect().intersect(&ir).is_none() {\n continue;\n }\n\n if !clip.to_int_rect().contains(&ir) {\n let subclip = ir\n .intersect(&clip.to_int_rect())\n .and_then(|r| r.to_screen_int_rect());\n\n if let Some(subclip) = subclip {\n do_anti_hairline(x0, y0, x1, y1, Some(subclip), blitter);\n }\n\n continue;\n }\n\n // fall through to no-clip case\n }\n\n do_anti_hairline(x0, y0, x1, y1, None, blitter);\n }\n}\n\n#[derive(Copy, Clone, Debug)]\nenum BlitterKind {\n HLine,\n Horish,\n VLine,\n Vertish,\n}\n\nfn do_anti_hairline(\n mut x0: FDot6,\n mut y0: FDot6,\n mut x1: FDot6,\n mut y1: FDot6,\n mut clip_opt: Option,\n blitter: &mut dyn Blitter,\n) {\n // check for integer NaN (0x80000000) which we can't handle (can't negate it)\n // It appears typically from a huge float (inf or nan) being converted to int.\n // If we see it, just don't draw.\n if any_bad_ints(x0, y0, x1, y1) != 0 {\n return;\n }\n\n // The caller must clip the line to [-32767.0 ... 32767.0] ahead of time (in dot6 format)\n debug_assert!(fdot6::can_convert_to_fdot16(x0));\n debug_assert!(fdot6::can_convert_to_fdot16(y0));\n debug_assert!(fdot6::can_convert_to_fdot16(x1));\n debug_assert!(fdot6::can_convert_to_fdot16(y1));\n\n if (x1 - x0).abs() > fdot6::from_i32(511) || (y1 - y0).abs() > fdot6::from_i32(511) {\n // instead of (x0 + x1) >> 1, we shift each separately. This is less\n // precise, but avoids overflowing the intermediate result if the\n // values are huge. A better fix might be to clip the original pts\n // directly (i.e. do the divide), so we don't spend time subdividing\n // huge lines at all.\n let hx = (x0 >> 1) + (x1 >> 1);\n let hy = (y0 >> 1) + (y1 >> 1);\n do_anti_hairline(x0, y0, hx, hy, clip_opt, blitter);\n do_anti_hairline(hx, hy, x1, y1, clip_opt, blitter);\n return; // we're done\n }\n\n let mut scale_start;\n let mut scale_stop;\n let mut istart;\n let mut istop;\n let mut fstart;\n let slope;\n let blitter_kind;\n\n if (x1 - x0).abs() > (y1 - y0).abs() {\n // mostly horizontal\n\n if x0 > x1 {\n // we want to go left-to-right\n core::mem::swap(&mut x0, &mut x1);\n core::mem::swap(&mut y0, &mut y1);\n }\n\n istart = fdot6::floor(x0);\n istop = fdot6::ceil(x1);\n fstart = fdot6::to_fdot16(y0);\n if y0 == y1 {\n // completely horizontal, take fast case\n slope = 0;\n blitter_kind = Some(BlitterKind::HLine);\n } else {\n slope = fdot16::fast_div(y1 - y0, x1 - x0);\n debug_assert!(slope >= -fdot16::ONE && slope <= fdot16::ONE);\n fstart += (slope * (32 - (x0 & 63)) + 32) >> 6;\n blitter_kind = Some(BlitterKind::Horish);\n }\n\n debug_assert!(istop > istart);\n if istop - istart == 1 {\n // we are within a single pixel\n scale_start = x1 - x0;\n debug_assert!(scale_start >= 0 && scale_start <= 64);\n scale_stop = 0;\n } else {\n scale_start = 64 - (x0 & 63);\n scale_stop = x1 & 63;\n }\n\n if let Some(clip) = clip_opt {\n let clip = clip.to_int_rect();\n\n if istart >= clip.right() || istop <= clip.left() {\n return; // we're done\n }\n\n if istart < clip.left() {\n fstart += slope * (clip.left() - istart);\n istart = clip.left();\n scale_start = 64;\n if istop - istart == 1 {\n // we are within a single pixel\n scale_start = contribution_64(x1);\n scale_stop = 0;\n }\n }\n\n if istop > clip.right() {\n istop = clip.right();\n scale_stop = 0; // so we don't draw this last column\n }\n\n debug_assert!(istart <= istop);\n if istart == istop {\n return; // we're done\n }\n\n // now test if our Y values are completely inside the clip\n let (mut top, mut bottom) = if slope >= 0 {\n // T2B\n let top = fdot16::floor_to_i32(fstart - fdot16::HALF);\n let bottom =\n fdot16::ceil_to_i32(fstart + (istop - istart - 1) * slope + fdot16::HALF);\n (top, bottom)\n } else {\n // B2T\n let bottom = fdot16::ceil_to_i32(fstart + fdot16::HALF);\n let top =\n fdot16::floor_to_i32(fstart + (istop - istart - 1) * slope - fdot16::HALF);\n (top, bottom)\n };\n\n top -= 1;\n bottom += 1;\n\n if top >= clip.bottom() || bottom <= clip.top() {\n return; // we're done\n }\n\n if clip.top() <= top && clip.bottom() >= bottom {\n clip_opt = None;\n }\n }\n } else {\n // mostly vertical\n\n if y0 > y1 {\n // we want to go top-to-bottom\n core::mem::swap(&mut x0, &mut x1);\n core::mem::swap(&mut y0, &mut y1);\n }\n\n istart = fdot6::floor(y0);\n istop = fdot6::ceil(y1);\n fstart = fdot6::to_fdot16(x0);\n if x0 == x1 {\n if y0 == y1 {\n // are we zero length? nothing to do\n return; // we're done\n }\n\n slope = 0;\n blitter_kind = Some(BlitterKind::VLine);\n } else {\n slope = fdot16::fast_div(x1 - x0, y1 - y0);\n debug_assert!(slope <= fdot16::ONE && slope >= -fdot16::ONE);\n fstart += (slope * (32 - (y0 & 63)) + 32) >> 6;\n blitter_kind = Some(BlitterKind::Vertish);\n }\n\n debug_assert!(istop > istart);\n if istop - istart == 1 {\n // we are within a single pixel\n scale_start = y1 - y0;\n debug_assert!(scale_start >= 0 && scale_start <= 64);\n scale_stop = 0;\n } else {\n scale_start = 64 - (y0 & 63);\n scale_stop = y1 & 63;\n }\n\n if let Some(clip) = clip_opt {\n let clip = clip.to_int_rect();\n\n if istart >= clip.bottom() || istop <= clip.top() {\n return; // we're done\n }\n\n if istart < clip.top() {\n fstart += slope * (clip.top() - istart);\n istart = clip.top();\n scale_start = 64;\n if istop - istart == 1 {\n // we are within a single pixel\n scale_start = contribution_64(y1);\n scale_stop = 0;\n }\n }\n if istop > clip.bottom() {\n istop = clip.bottom();\n scale_stop = 0; // so we don't draw this last row\n }\n\n debug_assert!(istart <= istop);\n if istart == istop {\n return; // we're done\n }\n\n // now test if our X values are completely inside the clip\n let (mut left, mut right) = if slope >= 0 {\n // L2R\n let left = fdot16::floor_to_i32(fstart - fdot16::HALF);\n let right =\n fdot16::ceil_to_i32(fstart + (istop - istart - 1) * slope + fdot16::HALF);\n (left, right)\n } else {\n // R2L\n let right = fdot16::ceil_to_i32(fstart + fdot16::HALF);\n let left =\n fdot16::floor_to_i32(fstart + (istop - istart - 1) * slope - fdot16::HALF);\n (left, right)\n };\n\n left -= 1;\n right += 1;\n\n if left >= clip.right() || right <= clip.left() {\n return; // we're done\n }\n\n if clip.left() <= left && clip.right() >= right {\n clip_opt = None;\n }\n }\n }\n\n let mut clip_blitter;\n let blitter = if let Some(clip) = clip_opt {\n clip_blitter = RectClipBlitter { blitter, clip };\n &mut clip_blitter\n } else {\n blitter\n };\n\n let blitter_kind = match blitter_kind {\n Some(v) => v,\n None => return,\n };\n\n // A bit ugly, but looks like this is the only way to have stack allocated object trait.\n let mut hline_blitter;\n let mut horish_blitter;\n let mut vline_blitter;\n let mut vertish_blitter;\n let hair_blitter: &mut dyn AntiHairBlitter = match blitter_kind {\n BlitterKind::HLine => {\n hline_blitter = HLineAntiHairBlitter(blitter);\n &mut hline_blitter\n }\n BlitterKind::Horish => {\n horish_blitter = HorishAntiHairBlitter(blitter);\n &mut horish_blitter\n }\n BlitterKind::VLine => {\n vline_blitter = VLineAntiHairBlitter(blitter);\n &mut vline_blitter\n }\n BlitterKind::Vertish => {\n vertish_blitter = VertishAntiHairBlitter(blitter);\n &mut vertish_blitter\n }\n };\n\n debug_assert!(istart >= 0);\n let mut istart = istart as u32;\n\n debug_assert!(istop >= 0);\n let istop = istop as u32;\n\n fstart = hair_blitter.draw_cap(istart, fstart, slope, scale_start);\n istart += 1;\n let full_spans = istop - istart - (scale_stop > 0) as u32;\n if full_spans > 0 {\n fstart = hair_blitter.draw_line(istart, istart + full_spans, fstart, slope);\n }\n\n if scale_stop > 0 {\n hair_blitter.draw_cap(istop - 1, fstart, slope, scale_stop);\n }\n}\n\n// returns high-bit set if x == 0x8000\nfn bad_int(x: i32) -> i32 {\n x & -x\n}\n\nfn any_bad_ints(a: i32, b: i32, c: i32, d: i32) -> i32 {\n (bad_int(a) | bad_int(b) | bad_int(c) | bad_int(d)) >> ((core::mem::size_of::() << 3) - 1)\n}\n\n// We want the fractional part of ordinate, but we want multiples of 64 to\n// return 64, not 0, so we can't just say (ordinate & 63).\n// We basically want to compute those bits, and if they're 0, return 64.\n// We can do that w/o a branch with an extra sub and add.\nfn contribution_64(ordinate: FDot6) -> i32 {\n let result = ((ordinate - 1) & 63) + 1;\n debug_assert!(result > 0 && result <= 64);\n result\n}\n\ntrait AntiHairBlitter {\n fn draw_cap(&mut self, x: u32, fy: FDot16, slope: FDot16, mod64: i32) -> FDot16;\n fn draw_line(&mut self, x: u32, stopx: u32, fy: FDot16, slope: FDot16) -> FDot16;\n}\n\nstruct HLineAntiHairBlitter<'a>(&'a mut dyn Blitter);\n\nimpl AntiHairBlitter for HLineAntiHairBlitter<'_> {\n fn draw_cap(&mut self, x: u32, mut fy: FDot16, _: FDot16, mod64: i32) -> FDot16 {\n fy += fdot16::ONE / 2;\n fy = fy.max(0);\n\n let y = (fy >> 16) as u32;\n let a = i32_to_alpha(fy >> 8);\n\n // lower line\n let mut ma = fdot6::small_scale(a, mod64);\n if ma != 0 {\n call_hline_blitter(x, Some(y), LENGTH_U32_ONE, ma, self.0);\n }\n\n // upper line\n ma = fdot6::small_scale(255 - a, mod64);\n if ma != 0 {\n call_hline_blitter(x, y.checked_sub(1), LENGTH_U32_ONE, ma, self.0);\n }\n\n fy - fdot16::ONE / 2\n }\n\n fn draw_line(&mut self, x: u32, stop_x: u32, mut fy: FDot16, _: FDot16) -> FDot16 {\n let count = match LengthU32::new(stop_x - x) {\n Some(n) => n,\n None => return fy,\n };\n\n fy += fdot16::ONE / 2;\n fy = fy.max(0);\n\n let y = (fy >> 16) as u32;\n let mut a = i32_to_alpha(fy >> 8);\n\n // lower line\n if a != 0 {\n call_hline_blitter(x, Some(y), count, a, self.0);\n }\n\n // upper line\n a = 255 - a;\n if a != 0 {\n call_hline_blitter(x, y.checked_sub(1), count, a, self.0);\n }\n\n fy - fdot16::ONE / 2\n }\n}\n\nstruct HorishAntiHairBlitter<'a>(&'a mut dyn Blitter);\n\nimpl AntiHairBlitter for HorishAntiHairBlitter<'_> {\n fn draw_cap(&mut self, x: u32, mut fy: FDot16, dy: FDot16, mod64: i32) -> FDot16 {\n fy += fdot16::ONE / 2;\n fy = fy.max(0);\n\n let lower_y = (fy >> 16) as u32;\n let a = i32_to_alpha(fy >> 8);\n let a0 = fdot6::small_scale(255 - a, mod64);\n let a1 = fdot6::small_scale(a, mod64);\n self.0.blit_anti_v2(x, lower_y.max(1) - 1, a0, a1);\n\n fy + dy - fdot16::ONE / 2\n }\n\n fn draw_line(&mut self, mut x: u32, stop_x: u32, mut fy: FDot16, dy: FDot16) -> FDot16 {\n debug_assert!(x < stop_x);\n\n fy += fdot16::ONE / 2;\n loop {\n fy = fy.max(0);\n let lower_y = (fy >> 16) as u32;\n let a = i32_to_alpha(fy >> 8);\n self.0.blit_anti_v2(x, lower_y.max(1) - 1, 255 - a, a);\n fy += dy;\n\n x += 1;\n if x >= stop_x {\n break;\n }\n }\n\n fy - fdot16::ONE / 2\n }\n}\n\nstruct VLineAntiHairBlitter<'a>(&'a mut dyn Blitter);\n\nimpl AntiHairBlitter for VLineAntiHairBlitter<'_> {\n fn draw_cap(&mut self, y: u32, mut fx: FDot16, dx: FDot16, mod64: i32) -> FDot16 {\n debug_assert!(dx == 0);\n fx += fdot16::ONE / 2;\n fx = fx.max(0);\n\n let x = (fx >> 16) as u32;\n let a = i32_to_alpha(fx >> 8);\n\n let mut ma = fdot6::small_scale(a, mod64);\n if ma != 0 {\n self.0.blit_v(x, y, LENGTH_U32_ONE, ma);\n }\n\n ma = fdot6::small_scale(255 - a, mod64);\n if ma != 0 {\n self.0.blit_v(x.max(1) - 1, y, LENGTH_U32_ONE, ma);\n }\n\n fx - fdot16::ONE / 2\n }\n\n fn draw_line(&mut self, y: u32, stop_y: u32, mut fx: FDot16, dx: FDot16) -> FDot16 {\n debug_assert!(dx == 0);\n let height = match LengthU32::new(stop_y - y) {\n Some(n) => n,\n None => return fx,\n };\n\n fx += fdot16::ONE / 2;\n fx = fx.max(0);\n\n let x = (fx >> 16) as u32;\n let mut a = i32_to_alpha(fx >> 8);\n\n if a != 0 {\n self.0.blit_v(x, y, height, a);\n }\n\n a = 255 - a;\n if a != 0 {\n self.0.blit_v(x.max(1) - 1, y, height, a);\n }\n\n fx - fdot16::ONE / 2\n }\n}\n\nstruct VertishAntiHairBlitter<'a>(&'a mut dyn Blitter);\n\nimpl AntiHairBlitter for VertishAntiHairBlitter<'_> {\n fn draw_cap(&mut self, y: u32, mut fx: FDot16, dx: FDot16, mod64: i32) -> FDot16 {\n fx += fdot16::ONE / 2;\n fx = fx.max(0);\n\n let x = (fx >> 16) as u32;\n let a = i32_to_alpha(fx >> 8);\n self.0.blit_anti_h2(\n x.max(1) - 1,\n y,\n fdot6::small_scale(255 - a, mod64),\n fdot6::small_scale(a, mod64),\n );\n\n fx + dx - fdot16::ONE / 2\n }\n\n fn draw_line(&mut self, mut y: u32, stop_y: u32, mut fx: FDot16, dx: FDot16) -> FDot16 {\n debug_assert!(y < stop_y);\n\n fx += fdot16::ONE / 2;\n loop {\n fx = fx.max(0);\n let x = (fx >> 16) as u32;\n let a = i32_to_alpha(fx >> 8);\n self.0.blit_anti_h2(x.max(1) - 1, y, 255 - a, a);\n fx += dx;\n\n y += 1;\n if y >= stop_y {\n break;\n }\n }\n\n fx - fdot16::ONE / 2\n }\n}\n\nfn i32_to_alpha(a: i32) -> u8 {\n (a & 0xFF) as u8\n}\n\nstruct RectClipBlitter<'a> {\n blitter: &'a mut dyn Blitter,\n clip: ScreenIntRect,\n}\n\nimpl Blitter for RectClipBlitter<'_> {\n fn blit_anti_h(\n &mut self,\n x: u32,\n y: u32,\n mut antialias: &mut [AlphaU8],\n mut runs: &mut [AlphaRun],\n ) {\n fn y_in_rect(y: u32, rect: ScreenIntRect) -> bool {\n (y - rect.top()) < rect.height()\n }\n\n if !y_in_rect(y, self.clip) || x >= self.clip.right() {\n return;\n }\n\n let mut x0 = x;\n let mut x1 = x + compute_anti_width(runs);\n\n if x1 <= self.clip.left() {\n return;\n }\n\n debug_assert!(x0 < x1);\n if x0 < self.clip.left() {\n let dx = self.clip.left() - x0;\n AlphaRuns::break_at(antialias, runs, dx as i32);\n antialias = &mut antialias[dx as usize..];\n runs = &mut runs[dx as usize..];\n x0 = self.clip.left();\n }\n\n debug_assert!(x0 < x1 && runs[(x1 - x0) as usize].is_none());\n if x1 > self.clip.right() {\n x1 = self.clip.right();\n AlphaRuns::break_at(antialias, runs, (x1 - x0) as i32);\n runs[(x1 - x0) as usize] = None;\n }\n\n debug_assert!(x0 < x1 && runs[(x1 - x0) as usize].is_none());\n debug_assert!(compute_anti_width(runs) == x1 - x0);\n\n self.blitter.blit_anti_h(x0, y, antialias, runs);\n }\n\n fn blit_v(&mut self, x: u32, y: u32, height: LengthU32, alpha: AlphaU8) {\n fn x_in_rect(x: u32, rect: ScreenIntRect) -> bool {\n (x - rect.left()) < rect.width()\n }\n\n if !x_in_rect(x, self.clip) {\n return;\n }\n\n let mut y0 = y;\n let mut y1 = y + height.get();\n\n if y0 < self.clip.top() {\n y0 = self.clip.top();\n }\n\n if y1 > self.clip.bottom() {\n y1 = self.clip.bottom();\n }\n\n if y0 < y1 {\n if let Some(h) = LengthU32::new(y1 - y0) {\n self.blitter.blit_v(x, y0, h, alpha);\n }\n }\n }\n\n fn blit_anti_h2(&mut self, x: u32, y: u32, alpha0: AlphaU8, alpha1: AlphaU8) {\n self.blit_anti_h(\n x,\n y,\n &mut [alpha0, alpha1],\n &mut [NonZeroU16::new(1), NonZeroU16::new(1), None],\n );\n }\n\n fn blit_anti_v2(&mut self, x: u32, y: u32, alpha0: AlphaU8, alpha1: AlphaU8) {\n self.blit_anti_h(x, y, &mut [alpha0], &mut [NonZeroU16::new(1), None]);\n\n self.blit_anti_h(x, y + 1, &mut [alpha1], &mut [NonZeroU16::new(1), None]);\n }\n}\n\nfn compute_anti_width(runs: &[AlphaRun]) -> u32 {\n let mut i = 0;\n let mut width = 0;\n while let Some(count) = runs[i] {\n width += u32::from(count.get());\n i += usize::from(count.get());\n }\n\n width\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/scan/mod.rs", "sha256": "db9ca53fad681f35b4c13dc228e1bab54cb08a0c3efb4d1e4b13d937c5ec1a65", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2011 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\npub mod hairline;\npub mod hairline_aa;\npub mod path;\npub mod path_aa;\n\nuse crate::{IntRect, Rect};\n\nuse crate::blitter::Blitter;\nuse crate::geom::{IntRectExt, ScreenIntRect};\n\npub fn fill_rect(rect: &Rect, clip: &ScreenIntRect, blitter: &mut dyn Blitter) {\n if let Some(rect) = rect.round() {\n fill_int_rect(&rect, clip, blitter);\n }\n}\n\nfn fill_int_rect(rect: &IntRect, clip: &ScreenIntRect, blitter: &mut dyn Blitter) {\n let rect = match rect.intersect(&clip.to_int_rect()) {\n Some(v) => v,\n None => return, // everything was clipped out\n };\n\n let rect = match rect.to_screen_int_rect() {\n Some(v) => v,\n None => return,\n };\n\n blitter.blit_rect(&rect);\n}\n\npub fn fill_rect_aa(rect: &Rect, clip: &ScreenIntRect, blitter: &mut dyn Blitter) {\n hairline_aa::fill_rect(rect, clip, blitter);\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/scan/path.rs", "sha256": "1d8a89c974b120f1a82d8318d1ceedc740be44a3e4fc73bdfe20184ae5c26619", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse core::convert::TryFrom;\n\nuse tiny_skia_path::SaturateCast;\n\nuse crate::{FillRule, IntRect, LengthU32, Path, Rect};\n\nuse crate::blitter::Blitter;\nuse crate::edge::{Edge, LineEdge};\nuse crate::edge_builder::{BasicEdgeBuilder, ShiftedIntRect};\nuse crate::fixed_point::{fdot16, fdot6, FDot16};\nuse crate::geom::{IntRectExt, ScreenIntRect};\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\npub fn fill_path(\n path: &Path,\n fill_rule: FillRule,\n clip: &ScreenIntRect,\n blitter: &mut dyn Blitter,\n) {\n let ir = match conservative_round_to_int(&path.bounds()) {\n Some(v) => v,\n None => return,\n };\n\n let path_contained_in_clip = if let Some(bounds) = ir.to_screen_int_rect() {\n clip.contains(&bounds)\n } else {\n // If bounds cannot be converted into ScreenIntRect,\n // the path is out of clip.\n false\n };\n\n // TODO: SkScanClipper\n\n fill_path_impl(\n path,\n fill_rule,\n clip,\n ir.y(),\n ir.bottom(),\n 0,\n path_contained_in_clip,\n blitter,\n );\n}\n\n// Conservative rounding function, which effectively nudges the int-rect to be slightly larger\n// than Rect::round() might have produced. This is a safety-net for the scan-converter, which\n// inspects the returned int-rect, and may disable clipping (for speed) if it thinks all of the\n// edges will fit inside the clip's bounds. The scan-converter introduces slight numeric errors\n// due to accumulated += of the slope, so this function is used to return a conservatively large\n// int-bounds, and thus we will only disable clipping if we're sure the edges will stay in-bounds.\nfn conservative_round_to_int(src: &Rect) -> Option {\n // We must use `from_ltrb`, otherwise rounding will be incorrect.\n IntRect::from_ltrb(\n round_down_to_int(src.left()),\n round_down_to_int(src.top()),\n round_up_to_int(src.right()),\n round_up_to_int(src.bottom()),\n )\n}\n\n// Bias used for conservative rounding of float rects to int rects, to nudge the irects a little\n// larger, so we don't \"think\" a path's bounds are inside a clip, when (due to numeric drift in\n// the scan-converter) we might walk beyond the predicted limits.\n//\n// This value has been determined trial and error: pick the smallest value (after the 0.5) that\n// fixes any problematic cases (e.g. crbug.com/844457)\n// NOTE: cubics appear to be the main reason for needing this slop. If we could (perhaps) have a\n// more accurate walker for cubics, we may be able to reduce this fudge factor.\nconst CONSERVATIVE_ROUND_BIAS: f64 = 0.5 + 1.5 / fdot6::ONE as f64;\n\n// Round the value down. This is used to round the top and left of a rectangle,\n// and corresponds to the way the scan converter treats the top and left edges.\n// It has a slight bias to make the \"rounded\" int smaller than a normal round, to create a more\n// conservative int-bounds (larger) from a float rect.\nfn round_down_to_int(x: f32) -> i32 {\n let mut xx = x as f64;\n xx -= CONSERVATIVE_ROUND_BIAS;\n i32::saturate_from(xx.ceil())\n}\n\n// Round the value up. This is used to round the right and bottom of a rectangle.\n// It has a slight bias to make the \"rounded\" int smaller than a normal round, to create a more\n// conservative int-bounds (larger) from a float rect.\nfn round_up_to_int(x: f32) -> i32 {\n let mut xx = x as f64;\n xx += CONSERVATIVE_ROUND_BIAS;\n i32::saturate_from(xx.floor())\n}\n\npub fn fill_path_impl(\n path: &Path,\n fill_rule: FillRule,\n clip_rect: &ScreenIntRect,\n mut start_y: i32,\n mut stop_y: i32,\n shift_edges_up: i32,\n path_contained_in_clip: bool,\n blitter: &mut dyn Blitter,\n) {\n let shifted_clip = match ShiftedIntRect::new(clip_rect, shift_edges_up) {\n Some(v) => v,\n None => return,\n };\n\n let clip = if path_contained_in_clip {\n None\n } else {\n Some(&shifted_clip)\n };\n let mut edges = match BasicEdgeBuilder::build_edges(path, clip, shift_edges_up) {\n Some(v) => v,\n None => return, // no edges to render, just return\n };\n\n edges.sort_by(|a, b| {\n let mut value_a = a.as_line().first_y;\n let mut value_b = b.as_line().first_y;\n\n if value_a == value_b {\n value_a = a.as_line().x;\n value_b = b.as_line().x;\n }\n\n value_a.cmp(&value_b)\n });\n\n for i in 0..edges.len() {\n // 0 will be set later, so start with 1.\n edges[i].prev = Some(i as u32 + 0);\n edges[i].next = Some(i as u32 + 2);\n }\n\n const EDGE_HEAD_Y: i32 = i32::MIN;\n const EDGE_TAIL_Y: i32 = i32::MAX;\n\n edges.insert(\n 0,\n Edge::Line(LineEdge {\n prev: None,\n next: Some(1),\n x: i32::MIN,\n first_y: EDGE_HEAD_Y,\n ..LineEdge::default()\n }),\n );\n\n edges.push(Edge::Line(LineEdge {\n prev: Some(edges.len() as u32 - 1),\n next: None,\n first_y: EDGE_TAIL_Y,\n ..LineEdge::default()\n }));\n\n start_y <<= shift_edges_up;\n stop_y <<= shift_edges_up;\n\n let top = shifted_clip.shifted().y() as i32;\n if !path_contained_in_clip && start_y < top {\n start_y = top;\n }\n\n let bottom = shifted_clip.shifted().bottom() as i32;\n if !path_contained_in_clip && stop_y > bottom as i32 {\n stop_y = bottom as i32;\n }\n\n let start_y = match u32::try_from(start_y) {\n Ok(v) => v,\n Err(_) => return,\n };\n let stop_y = match u32::try_from(stop_y) {\n Ok(v) => v,\n Err(_) => return,\n };\n\n // TODO: walk_simple_edges\n\n walk_edges(\n fill_rule,\n start_y,\n stop_y,\n shifted_clip.shifted().right(),\n &mut edges,\n blitter,\n );\n}\n\n// TODO: simplify!\nfn walk_edges(\n fill_rule: FillRule,\n start_y: u32,\n stop_y: u32,\n right_clip: u32,\n edges: &mut [Edge],\n blitter: &mut dyn Blitter,\n) {\n let mut curr_y = start_y;\n let winding_mask = if fill_rule == FillRule::EvenOdd {\n 1\n } else {\n -1\n };\n\n loop {\n let mut w = 0i32;\n let mut left = 0u32;\n let mut prev_x = edges[0].x;\n\n let mut curr_idx = edges[0].next.unwrap() as usize;\n while edges[curr_idx].first_y <= curr_y as i32 {\n debug_assert!(edges[curr_idx].last_y >= curr_y as i32);\n\n let x = fdot16::round_to_i32(edges[curr_idx].x) as u32; // TODO: check\n\n if (w & winding_mask) == 0 {\n // we're starting interval\n left = x;\n }\n\n w += i32::from(edges[curr_idx].winding);\n\n if (w & winding_mask) == 0 {\n // we finished an interval\n if let Some(width) = LengthU32::new(x - left) {\n blitter.blit_h(left, curr_y, width);\n }\n }\n\n let next_idx = edges[curr_idx].next.unwrap();\n let new_x;\n\n if edges[curr_idx].last_y == curr_y as i32 {\n // are we done with this edge?\n match &mut edges[curr_idx] {\n Edge::Line(_) => {\n remove_edge(curr_idx, edges);\n }\n Edge::Quadratic(ref mut quad) => {\n if quad.curve_count > 0 && quad.update() {\n new_x = quad.line.x;\n\n if new_x < prev_x {\n // ripple current edge backwards until it is x-sorted\n backward_insert_edge_based_on_x(curr_idx, edges);\n } else {\n prev_x = new_x;\n }\n } else {\n remove_edge(curr_idx, edges);\n }\n }\n Edge::Cubic(ref mut cubic) => {\n if cubic.curve_count < 0 && cubic.update() {\n debug_assert!(cubic.line.first_y == curr_y as i32 + 1);\n\n new_x = cubic.line.x;\n\n if new_x < prev_x {\n // ripple current edge backwards until it is x-sorted\n backward_insert_edge_based_on_x(curr_idx, edges);\n } else {\n prev_x = new_x;\n }\n } else {\n remove_edge(curr_idx, edges);\n }\n }\n }\n } else {\n debug_assert!(edges[curr_idx].last_y > curr_y as i32);\n new_x = edges[curr_idx].x + edges[curr_idx].dx;\n edges[curr_idx].x = new_x;\n\n if new_x < prev_x {\n // ripple current edge backwards until it is x-sorted\n backward_insert_edge_based_on_x(curr_idx, edges);\n } else {\n prev_x = new_x;\n }\n }\n\n curr_idx = next_idx as usize;\n }\n\n if (w & winding_mask) != 0 {\n // was our right-edge culled away?\n if let Some(width) = LengthU32::new(right_clip - left) {\n blitter.blit_h(left, curr_y, width);\n }\n }\n\n curr_y += 1;\n if curr_y >= stop_y {\n break;\n }\n\n // now current edge points to the first edge with a Yint larger than curr_y\n insert_new_edges(curr_idx, curr_y as i32, edges);\n }\n}\n\nfn remove_edge(curr_idx: usize, edges: &mut [Edge]) {\n let prev = edges[curr_idx].prev.unwrap();\n let next = edges[curr_idx].next.unwrap();\n\n edges[prev as usize].next = Some(next);\n edges[next as usize].prev = Some(prev);\n}\n\nfn backward_insert_edge_based_on_x(curr_idx: usize, edges: &mut [Edge]) {\n let x = edges[curr_idx].x;\n let mut prev_idx = edges[curr_idx].prev.unwrap() as usize;\n while prev_idx != 0 {\n if edges[prev_idx].x > x {\n prev_idx = edges[prev_idx].prev.unwrap() as usize;\n } else {\n break;\n }\n }\n\n let next_idx = edges[prev_idx].next.unwrap() as usize;\n if next_idx != curr_idx {\n remove_edge(curr_idx, edges);\n insert_edge_after(curr_idx, prev_idx, edges);\n }\n}\n\nfn insert_edge_after(curr_idx: usize, after_idx: usize, edges: &mut [Edge]) {\n edges[curr_idx].prev = Some(after_idx as u32);\n edges[curr_idx].next = edges[after_idx].next;\n\n let after_next_idx = edges[after_idx].next.unwrap() as usize;\n edges[after_next_idx].prev = Some(curr_idx as u32);\n edges[after_idx].next = Some(curr_idx as u32);\n}\n\n// Start from the right side, searching backwards for the point to begin the new edge list\n// insertion, marching forwards from here. The implementation could have started from the left\n// of the prior insertion, and search to the right, or with some additional caching, binary\n// search the starting point. More work could be done to determine optimal new edge insertion.\nfn backward_insert_start(mut prev_idx: usize, x: FDot16, edges: &mut [Edge]) -> usize {\n while let Some(prev) = edges[prev_idx].prev {\n prev_idx = prev as usize;\n if edges[prev_idx].x <= x {\n break;\n }\n }\n\n prev_idx\n}\n\nfn insert_new_edges(mut new_idx: usize, curr_y: i32, edges: &mut [Edge]) {\n if edges[new_idx].first_y != curr_y {\n return;\n }\n\n let prev_idx = edges[new_idx].prev.unwrap() as usize;\n if edges[prev_idx].x <= edges[new_idx].x {\n return;\n }\n\n // find first x pos to insert\n let mut start_idx = backward_insert_start(prev_idx, edges[new_idx].x, edges);\n // insert the lot, fixing up the links as we go\n loop {\n let next_idx = edges[new_idx].next.unwrap() as usize;\n let mut keep_edge = false;\n loop {\n let after_idx = edges[start_idx].next.unwrap() as usize;\n if after_idx == new_idx {\n keep_edge = true;\n break;\n }\n\n if edges[after_idx].x >= edges[new_idx].x {\n break;\n }\n\n start_idx = after_idx;\n }\n\n if !keep_edge {\n remove_edge(new_idx, edges);\n insert_edge_after(new_idx, start_idx, edges);\n }\n\n start_idx = new_idx;\n new_idx = next_idx;\n\n if edges[new_idx].first_y != curr_y {\n break;\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/scan/path_aa.rs", "sha256": "d6fee0876986a9225be67d47c9f3e60dc6cf863d1b0ef2949ec499e72f7e893c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse core::convert::TryFrom;\n\nuse crate::{FillRule, IntRect, LengthU32, Path, Rect};\n\nuse crate::alpha_runs::AlphaRuns;\nuse crate::blitter::Blitter;\nuse crate::color::AlphaU8;\nuse crate::geom::{IntRectExt, ScreenIntRect};\nuse crate::math::left_shift;\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\n/// controls how much we super-sample (when we use that scan conversion)\nconst SUPERSAMPLE_SHIFT: u32 = 2;\n\nconst SHIFT: u32 = SUPERSAMPLE_SHIFT;\nconst SCALE: u32 = 1 << SHIFT;\nconst MASK: u32 = SCALE - 1;\n\npub fn fill_path(\n path: &Path,\n fill_rule: FillRule,\n clip: &ScreenIntRect,\n blitter: &mut dyn Blitter,\n) {\n // Unlike `path.bounds.to_rect()?.round_out()`,\n // this method rounds out first and then converts into a Rect.\n let ir = Rect::from_ltrb(\n path.bounds().left().floor(),\n path.bounds().top().floor(),\n path.bounds().right().ceil(),\n path.bounds().bottom().ceil(),\n )\n .and_then(|r| r.round_out());\n let ir = match ir {\n Some(v) => v,\n None => return,\n };\n\n // TODO: remove\n // If the intersection of the path bounds and the clip bounds\n // will overflow 32767 when << by SHIFT, we can't supersample,\n // so draw without antialiasing.\n let clipped_ir = match ir.intersect(&clip.to_int_rect()) {\n Some(v) => v,\n None => return,\n };\n if rect_overflows_short_shift(&clipped_ir, SHIFT as i32) != 0 {\n super::path::fill_path(path, fill_rule, clip, blitter);\n return;\n }\n\n // TODO: remove\n // Our antialiasing can't handle a clip larger than 32767.\n // TODO: skia actually limits the clip to 32767\n {\n const MAX_CLIP_COORD: u32 = 32767;\n if clip.right() > MAX_CLIP_COORD || clip.bottom() > MAX_CLIP_COORD {\n return;\n }\n }\n\n // TODO: SkScanClipper\n // TODO: AAA\n\n fill_path_impl(path, fill_rule, &ir, clip, blitter)\n}\n\n// Would any of the coordinates of this rectangle not fit in a short,\n// when left-shifted by shift?\nfn rect_overflows_short_shift(rect: &IntRect, shift: i32) -> i32 {\n debug_assert!(overflows_short_shift(8191, shift) == 0);\n debug_assert!(overflows_short_shift(8192, shift) != 0);\n debug_assert!(overflows_short_shift(32767, 0) == 0);\n debug_assert!(overflows_short_shift(32768, 0) != 0);\n\n // Since we expect these to succeed, we bit-or together\n // for a tiny extra bit of speed.\n overflows_short_shift(rect.left(), shift)\n | overflows_short_shift(rect.top(), shift)\n | overflows_short_shift(rect.right(), shift)\n | overflows_short_shift(rect.bottom(), shift)\n}\n\nfn overflows_short_shift(value: i32, shift: i32) -> i32 {\n let s = 16 + shift;\n (left_shift(value, s) >> s) - value\n}\n\nfn fill_path_impl(\n path: &Path,\n fill_rule: FillRule,\n bounds: &IntRect,\n clip: &ScreenIntRect,\n blitter: &mut dyn Blitter,\n) {\n // TODO: MaskSuperBlitter\n\n // TODO: 15% slower than skia, find out why\n let mut blitter = match SuperBlitter::new(bounds, clip, blitter) {\n Some(v) => v,\n None => return, // clipped out, nothing else to do\n };\n\n let path_contained_in_clip = if let Some(bounds) = bounds.to_screen_int_rect() {\n clip.contains(&bounds)\n } else {\n // If bounds cannot be converted into ScreenIntRect,\n // the path is out of clip.\n false\n };\n\n super::path::fill_path_impl(\n path,\n fill_rule,\n clip,\n bounds.top(),\n bounds.bottom(),\n SHIFT as i32,\n path_contained_in_clip,\n &mut blitter,\n );\n}\n\nstruct BaseSuperBlitter<'a> {\n real_blitter: &'a mut dyn Blitter,\n\n /// Current y coordinate, in destination coordinates.\n curr_iy: i32,\n /// Widest row of region to be blitted, in destination coordinates.\n width: LengthU32,\n /// Leftmost x coordinate in any row, in destination coordinates.\n left: u32,\n /// Leftmost x coordinate in any row, in supersampled coordinates.\n super_left: u32,\n\n /// Current y coordinate in supersampled coordinates.\n curr_y: i32,\n /// Initial y coordinate (top of bounds).\n top: i32,\n}\n\nimpl<'a> BaseSuperBlitter<'a> {\n fn new(\n bounds: &IntRect,\n clip_rect: &ScreenIntRect,\n blitter: &'a mut dyn Blitter,\n ) -> Option {\n let sect = bounds\n .intersect(&clip_rect.to_int_rect())?\n .to_screen_int_rect()?;\n Some(BaseSuperBlitter {\n real_blitter: blitter,\n curr_iy: sect.top() as i32 - 1,\n width: sect.width_safe(),\n left: sect.left(),\n super_left: sect.left() << SHIFT,\n curr_y: (sect.top() << SHIFT) as i32 - 1,\n top: sect.top() as i32,\n })\n }\n}\n\nstruct SuperBlitter<'a> {\n base: BaseSuperBlitter<'a>,\n runs: AlphaRuns,\n offset_x: usize,\n}\n\nimpl<'a> SuperBlitter<'a> {\n fn new(\n bounds: &IntRect,\n clip_rect: &ScreenIntRect,\n blitter: &'a mut dyn Blitter,\n ) -> Option {\n let base = BaseSuperBlitter::new(bounds, clip_rect, blitter)?;\n let runs_width = base.width;\n Some(SuperBlitter {\n base,\n runs: AlphaRuns::new(runs_width),\n offset_x: 0,\n })\n }\n\n /// Once `runs` contains a complete supersampled row, flush() blits\n /// it out through the wrapped blitter.\n fn flush(&mut self) {\n if self.base.curr_iy >= self.base.top {\n if !self.runs.is_empty() {\n self.base.real_blitter.blit_anti_h(\n self.base.left,\n u32::try_from(self.base.curr_iy).unwrap(),\n &mut self.runs.alpha,\n &mut self.runs.runs,\n );\n self.runs.reset(self.base.width);\n self.offset_x = 0;\n }\n\n self.base.curr_iy = self.base.top - 1;\n }\n }\n}\n\nimpl Drop for SuperBlitter<'_> {\n fn drop(&mut self) {\n self.flush();\n }\n}\n\nimpl Blitter for SuperBlitter<'_> {\n /// Blits a row of pixels, with location and width specified\n /// in supersampled coordinates.\n fn blit_h(&mut self, mut x: u32, y: u32, mut width: LengthU32) {\n let iy = (y >> SHIFT) as i32;\n debug_assert!(iy >= self.base.curr_iy);\n\n // hack, until I figure out why my cubics (I think) go beyond the bounds\n match x.checked_sub(self.base.super_left) {\n Some(n) => x = n,\n None => {\n width = LengthU32::new(x + width.get()).unwrap();\n x = 0;\n }\n }\n\n debug_assert!(y as i32 >= self.base.curr_y);\n if self.base.curr_y != y as i32 {\n self.offset_x = 0;\n self.base.curr_y = y as i32;\n }\n\n if iy != self.base.curr_iy {\n // new scanline\n self.flush();\n self.base.curr_iy = iy;\n }\n\n let start = x;\n let stop = x + width.get();\n\n debug_assert!(stop > start);\n // integer-pixel-aligned ends of blit, rounded out\n let mut fb = start & MASK;\n let mut fe = stop & MASK;\n let mut n: i32 = (stop as i32 >> SHIFT) - (start as i32 >> SHIFT) - 1;\n\n if n < 0 {\n fb = fe - fb;\n n = 0;\n fe = 0;\n } else {\n if fb == 0 {\n n += 1;\n } else {\n fb = SCALE - fb;\n }\n }\n\n let max_value = u8::try_from((1 << (8 - SHIFT)) - (((y & MASK) + 1) >> SHIFT)).unwrap();\n self.offset_x = self.runs.add(\n x >> SHIFT,\n coverage_to_partial_alpha(fb),\n n as usize,\n coverage_to_partial_alpha(fe),\n max_value,\n self.offset_x,\n );\n }\n}\n\n// coverage_to_partial_alpha() is being used by AlphaRuns, which\n// *accumulates* SCALE pixels worth of \"alpha\" in [0,(256/SCALE)]\n// to produce a final value in [0, 255] and handles clamping 256->255\n// itself, with the same (alpha - (alpha >> 8)) correction as\n// coverage_to_exact_alpha().\nfn coverage_to_partial_alpha(mut aa: u32) -> AlphaU8 {\n aa <<= 8 - 2 * SHIFT;\n aa as AlphaU8\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/shaders/gradient.rs", "sha256": "df396a8df82213ea576f031110ce7e786912bc68b1fed6bbea0a79a1e3547d3c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse alloc::vec::Vec;\n\nuse tiny_skia_path::{NormalizedF32, Scalar};\n\nuse crate::{Color, SpreadMode, Transform};\n\nuse crate::pipeline::RasterPipelineBuilder;\nuse crate::pipeline::{self, EvenlySpaced2StopGradientCtx, GradientColor, GradientCtx};\n\n// The default SCALAR_NEARLY_ZERO threshold of .0024 is too big and causes regressions for svg\n// gradients defined in the wild.\npub const DEGENERATE_THRESHOLD: f32 = 1.0 / (1 << 15) as f32;\n\n/// A gradient point.\n#[allow(missing_docs)]\n#[derive(Clone, Copy, PartialEq, Debug)]\npub struct GradientStop {\n pub(crate) position: NormalizedF32,\n pub(crate) color: Color,\n}\n\nimpl GradientStop {\n /// Creates a new gradient point.\n ///\n /// `position` will be clamped to a 0..=1 range.\n pub fn new(position: f32, color: Color) -> Self {\n GradientStop {\n position: NormalizedF32::new_clamped(position),\n color,\n }\n }\n}\n\n#[derive(Clone, PartialEq, Debug)]\npub struct Gradient {\n stops: Vec,\n tile_mode: SpreadMode,\n pub(crate) transform: Transform,\n points_to_unit: Transform,\n pub(crate) colors_are_opaque: bool,\n has_uniform_stops: bool,\n}\n\nimpl Gradient {\n pub fn new(\n mut stops: Vec,\n tile_mode: SpreadMode,\n transform: Transform,\n points_to_unit: Transform,\n ) -> Self {\n debug_assert!(stops.len() > 1);\n\n // Note: we let the caller skip the first and/or last position.\n // i.e. pos[0] = 0.3, pos[1] = 0.7\n // In these cases, we insert dummy entries to ensure that the final data\n // will be bracketed by [0, 1].\n // i.e. our_pos[0] = 0, our_pos[1] = 0.3, our_pos[2] = 0.7, our_pos[3] = 1\n let dummy_first = stops[0].position.get() != 0.0;\n let dummy_last = stops[stops.len() - 1].position.get() != 1.0;\n\n // Now copy over the colors, adding the dummies as needed.\n if dummy_first {\n stops.insert(0, GradientStop::new(0.0, stops[0].color));\n }\n\n if dummy_last {\n stops.push(GradientStop::new(1.0, stops[stops.len() - 1].color));\n }\n\n let colors_are_opaque = stops.iter().all(|p| p.color.is_opaque());\n\n // Pin the last value to 1.0, and make sure positions are monotonic.\n let start_index = if dummy_first { 0 } else { 1 };\n let mut prev = 0.0;\n let mut has_uniform_stops = true;\n let uniform_step = stops[start_index].position.get() - prev;\n for i in start_index..stops.len() {\n let curr = if i + 1 == stops.len() {\n // The last one must be zero.\n 1.0\n } else {\n stops[i].position.get().bound(prev, 1.0)\n };\n\n has_uniform_stops &= uniform_step.is_nearly_equal(curr - prev);\n stops[i].position = NormalizedF32::new_clamped(curr);\n prev = curr;\n }\n\n Gradient {\n stops,\n tile_mode,\n transform,\n points_to_unit,\n colors_are_opaque,\n has_uniform_stops,\n }\n }\n\n pub fn push_stages(\n &self,\n p: &mut RasterPipelineBuilder,\n push_stages_pre: &dyn Fn(&mut RasterPipelineBuilder),\n push_stages_post: &dyn Fn(&mut RasterPipelineBuilder),\n ) -> bool {\n p.push(pipeline::Stage::SeedShader);\n\n let ts = match self.transform.invert() {\n Some(v) => v,\n None => {\n log::warn!(\"failed to invert a gradient transform. Nothing will be rendered\");\n return false;\n }\n };\n let ts = ts.post_concat(self.points_to_unit);\n p.push_transform(ts);\n\n push_stages_pre(p);\n\n match self.tile_mode {\n SpreadMode::Reflect => {\n p.push(pipeline::Stage::ReflectX1);\n }\n SpreadMode::Repeat => {\n p.push(pipeline::Stage::RepeatX1);\n }\n SpreadMode::Pad => {\n if self.has_uniform_stops {\n // We clamp only when the stops are evenly spaced.\n // If not, there may be hard stops, and clamping ruins hard stops at 0 and/or 1.\n // In that case, we must make sure we're using the general \"gradient\" stage,\n // which is the only stage that will correctly handle unclamped t.\n p.push(pipeline::Stage::PadX1);\n }\n }\n }\n\n // The two-stop case with stops at 0 and 1.\n if self.stops.len() == 2 {\n debug_assert!(self.has_uniform_stops);\n\n let c0 = self.stops[0].color;\n let c1 = self.stops[1].color;\n\n p.ctx.evenly_spaced_2_stop_gradient = EvenlySpaced2StopGradientCtx {\n factor: GradientColor::new(\n c1.red() - c0.red(),\n c1.green() - c0.green(),\n c1.blue() - c0.blue(),\n c1.alpha() - c0.alpha(),\n ),\n bias: GradientColor::from(c0),\n };\n\n p.push(pipeline::Stage::EvenlySpaced2StopGradient);\n } else {\n // Unlike Skia, we do not support the `evenly_spaced_gradient` stage.\n // In our case, there is no performance difference.\n\n let mut ctx = GradientCtx::default();\n\n // Note: In order to handle clamps in search, the search assumes\n // a stop conceptually placed at -inf.\n // Therefore, the max number of stops is `self.points.len()+1`.\n //\n // We also need at least 16 values for lowp pipeline.\n ctx.factors.reserve((self.stops.len() + 1).max(16));\n ctx.biases.reserve((self.stops.len() + 1).max(16));\n\n ctx.t_values.reserve(self.stops.len() + 1);\n\n // Remove the dummy stops inserted by Gradient::new\n // because they are naturally handled by the search method.\n let (first_stop, last_stop) = if self.stops.len() > 2 {\n let first = if self.stops[0].color != self.stops[1].color {\n 0\n } else {\n 1\n };\n\n let len = self.stops.len();\n let last = if self.stops[len - 2].color != self.stops[len - 1].color {\n len - 1\n } else {\n len - 2\n };\n (first, last)\n } else {\n (0, 1)\n };\n\n let mut t_l = self.stops[first_stop].position.get();\n let mut c_l = GradientColor::from(self.stops[first_stop].color);\n ctx.push_const_color(c_l);\n ctx.t_values.push(NormalizedF32::ZERO);\n // N.B. lastStop is the index of the last stop, not one after.\n for i in first_stop..last_stop {\n let t_r = self.stops[i + 1].position.get();\n let c_r = GradientColor::from(self.stops[i + 1].color);\n debug_assert!(t_l <= t_r);\n if t_l < t_r {\n // For each stop we calculate a bias B and a scale factor F, such that\n // for any t between stops n and n+1, the color we want is B[n] + F[n]*t.\n let f = GradientColor::new(\n (c_r.r - c_l.r) / (t_r - t_l),\n (c_r.g - c_l.g) / (t_r - t_l),\n (c_r.b - c_l.b) / (t_r - t_l),\n (c_r.a - c_l.a) / (t_r - t_l),\n );\n ctx.factors.push(f);\n\n ctx.biases.push(GradientColor::new(\n c_l.r - f.r * t_l,\n c_l.g - f.g * t_l,\n c_l.b - f.b * t_l,\n c_l.a - f.a * t_l,\n ));\n\n ctx.t_values.push(NormalizedF32::new_clamped(t_l));\n }\n\n t_l = t_r;\n c_l = c_r;\n }\n\n ctx.push_const_color(c_l);\n ctx.t_values.push(NormalizedF32::new_clamped(t_l));\n\n ctx.len = ctx.factors.len();\n\n // All lists must have the same length.\n debug_assert_eq!(ctx.factors.len(), ctx.t_values.len());\n debug_assert_eq!(ctx.biases.len(), ctx.t_values.len());\n\n // Will with zeros until we have enough data to fit into F32x16.\n while ctx.factors.len() < 16 {\n ctx.factors.push(GradientColor::default());\n ctx.biases.push(GradientColor::default());\n }\n\n p.push(pipeline::Stage::Gradient);\n p.ctx.gradient = ctx;\n }\n\n if !self.colors_are_opaque {\n p.push(pipeline::Stage::Premultiply);\n }\n\n push_stages_post(p);\n\n true\n }\n\n pub fn apply_opacity(&mut self, opacity: f32) {\n for stop in &mut self.stops {\n stop.color.apply_opacity(opacity);\n }\n\n self.colors_are_opaque = self.stops.iter().all(|p| p.color.is_opaque());\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/shaders/linear_gradient.rs", "sha256": "2d101e497d51e431f8d31fbf343d0e6eedfad0a1d42d0dcd1b75e742f456bb53", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse alloc::vec::Vec;\n\nuse tiny_skia_path::Scalar;\n\nuse crate::{Color, GradientStop, Point, Shader, SpreadMode, Transform};\n\nuse super::gradient::{Gradient, DEGENERATE_THRESHOLD};\nuse crate::pipeline::RasterPipelineBuilder;\n\n/// A linear gradient shader.\n#[derive(Clone, PartialEq, Debug)]\npub struct LinearGradient {\n pub(crate) base: Gradient,\n}\n\nimpl LinearGradient {\n /// Creates a new linear gradient shader.\n ///\n /// Returns `Shader::SolidColor` when:\n /// - `stops.len()` == 1\n /// - `start` and `end` are very close\n ///\n /// Returns `None` when:\n ///\n /// - `stops` is empty\n /// - `start` == `end`\n /// - `transform` is not invertible\n #[allow(clippy::new_ret_no_self)]\n pub fn new(\n start: Point,\n end: Point,\n stops: Vec,\n mode: SpreadMode,\n transform: Transform,\n ) -> Option> {\n if stops.is_empty() {\n return None;\n }\n\n if stops.len() == 1 {\n return Some(Shader::SolidColor(stops[0].color));\n }\n\n let length = (end - start).length();\n if !length.is_finite() {\n return None;\n }\n\n if length.is_nearly_zero_within_tolerance(DEGENERATE_THRESHOLD) {\n // Degenerate gradient, the only tricky complication is when in clamp mode,\n // the limit of the gradient approaches two half planes of solid color\n // (first and last). However, they are divided by the line perpendicular\n // to the start and end point, which becomes undefined once start and end\n // are exactly the same, so just use the end color for a stable solution.\n\n // Except for special circumstances of clamped gradients,\n // every gradient shape (when degenerate) can be mapped to the same fallbacks.\n // The specific shape factories must account for special clamped conditions separately,\n // this will always return the last color for clamped gradients.\n match mode {\n SpreadMode::Pad => {\n // Depending on how the gradient shape degenerates,\n // there may be a more specialized fallback representation\n // for the factories to use, but this is a reasonable default.\n return Some(Shader::SolidColor(stops.last().unwrap().color));\n }\n SpreadMode::Reflect | SpreadMode::Repeat => {\n // repeat and mirror are treated the same: the border colors are never visible,\n // but approximate the final color as infinite repetitions of the colors, so\n // it can be represented as the average color of the gradient.\n return Some(Shader::SolidColor(average_gradient_color(&stops)));\n }\n }\n }\n\n transform.invert()?;\n\n let unit_ts = points_to_unit_ts(start, end)?;\n Some(Shader::LinearGradient(LinearGradient {\n base: Gradient::new(stops, mode, transform, unit_ts),\n }))\n }\n\n pub(crate) fn is_opaque(&self) -> bool {\n self.base.colors_are_opaque\n }\n\n pub(crate) fn push_stages(&self, p: &mut RasterPipelineBuilder) -> bool {\n self.base.push_stages(p, &|_| {}, &|_| {})\n }\n}\n\nfn points_to_unit_ts(start: Point, end: Point) -> Option {\n let mut vec = end - start;\n let mag = vec.length();\n let inv = if mag != 0.0 { mag.invert() } else { 0.0 };\n\n vec.scale(inv);\n\n let mut ts = ts_from_sin_cos_at(-vec.y, vec.x, start.x, start.y);\n ts = ts.post_translate(-start.x, -start.y);\n ts = ts.post_scale(inv, inv);\n Some(ts)\n}\n\nfn average_gradient_color(points: &[GradientStop]) -> Color {\n use crate::wide::f32x4;\n\n fn load_color(c: Color) -> f32x4 {\n f32x4::from([c.red(), c.green(), c.blue(), c.alpha()])\n }\n\n fn store_color(c: f32x4) -> Color {\n let c: [f32; 4] = c.into();\n Color::from_rgba(c[0], c[1], c[2], c[3]).unwrap()\n }\n\n assert!(!points.is_empty());\n\n // The gradient is a piecewise linear interpolation between colors. For a given interval,\n // the integral between the two endpoints is 0.5 * (ci + cj) * (pj - pi), which provides that\n // intervals average color. The overall average color is thus the sum of each piece. The thing\n // to keep in mind is that the provided gradient definition may implicitly use p=0 and p=1.\n let mut blend = f32x4::splat(0.0);\n\n // Bake 1/(colorCount - 1) uniform stop difference into this scale factor\n let w_scale = f32x4::splat(0.5);\n\n for i in 0..points.len() - 1 {\n // Calculate the average color for the interval between pos(i) and pos(i+1)\n let c0 = load_color(points[i].color);\n let c1 = load_color(points[i + 1].color);\n // when pos == null, there are colorCount uniformly distributed stops, going from 0 to 1,\n // so pos[i + 1] - pos[i] = 1/(colorCount-1)\n let w = points[i + 1].position.get() - points[i].position.get();\n blend += w_scale * f32x4::splat(w) * (c1 + c0);\n }\n\n // Now account for any implicit intervals at the start or end of the stop definitions\n if points[0].position.get() > 0.0 {\n // The first color is fixed between p = 0 to pos[0], so 0.5 * (ci + cj) * (pj - pi)\n // becomes 0.5 * (c + c) * (pj - 0) = c * pj\n let c = load_color(points[0].color);\n blend += f32x4::splat(points[0].position.get()) * c;\n }\n\n let last_idx = points.len() - 1;\n if points[last_idx].position.get() < 1.0 {\n // The last color is fixed between pos[n-1] to p = 1, so 0.5 * (ci + cj) * (pj - pi)\n // becomes 0.5 * (c + c) * (1 - pi) = c * (1 - pi)\n let c = load_color(points[last_idx].color);\n blend += (f32x4::splat(1.0) - f32x4::splat(points[last_idx].position.get())) * c;\n }\n\n store_color(blend)\n}\n\nfn ts_from_sin_cos_at(sin: f32, cos: f32, px: f32, py: f32) -> Transform {\n let cos_inv = 1.0 - cos;\n Transform::from_row(\n cos,\n sin,\n -sin,\n cos,\n sdot(sin, py, cos_inv, px),\n sdot(-sin, px, cos_inv, py),\n )\n}\n\nfn sdot(a: f32, b: f32, c: f32, d: f32) -> f32 {\n a * b + c * d\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/shaders/mod.rs", "sha256": "05b9d88de14df33ad56122bd03ceb717abf2d4223609c58d6b28b49bba214d62", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nmod gradient;\nmod linear_gradient;\nmod pattern;\nmod radial_gradient;\n\nuse tiny_skia_path::{NormalizedF32, Scalar};\n\npub use gradient::GradientStop;\npub use linear_gradient::LinearGradient;\npub use pattern::{FilterQuality, Pattern, PixmapPaint};\npub use radial_gradient::RadialGradient;\n\nuse crate::{Color, Transform};\n\nuse crate::pipeline::RasterPipelineBuilder;\n\n/// A shader spreading mode.\n#[derive(Copy, Clone, PartialEq, Debug)]\npub enum SpreadMode {\n /// Replicate the edge color if the shader draws outside of its\n /// original bounds.\n Pad,\n\n /// Repeat the shader's image horizontally and vertically, alternating\n /// mirror images so that adjacent images always seam.\n Reflect,\n\n /// Repeat the shader's image horizontally and vertically.\n Repeat,\n}\n\nimpl Default for SpreadMode {\n fn default() -> Self {\n SpreadMode::Pad\n }\n}\n\n/// A shader specifies the source color(s) for what is being drawn.\n///\n/// If a paint has no shader, then the paint's color is used. If the paint has a\n/// shader, then the shader's color(s) are use instead, but they are\n/// modulated by the paint's alpha. This makes it easy to create a shader\n/// once (e.g. bitmap tiling or gradient) and then change its transparency\n/// without having to modify the original shader. Only the paint's alpha needs\n/// to be modified.\n#[derive(Clone, PartialEq, Debug)]\npub enum Shader<'a> {\n /// A solid color shader.\n SolidColor(Color),\n /// A linear gradient shader.\n LinearGradient(LinearGradient),\n /// A radial gradient shader.\n RadialGradient(RadialGradient),\n /// A pattern shader.\n Pattern(Pattern<'a>),\n}\n\nimpl<'a> Shader<'a> {\n /// Checks if the shader is guaranteed to produce only opaque colors.\n pub fn is_opaque(&self) -> bool {\n match self {\n Shader::SolidColor(ref c) => c.is_opaque(),\n Shader::LinearGradient(ref g) => g.is_opaque(),\n Shader::RadialGradient(_) => false,\n Shader::Pattern(_) => false,\n }\n }\n\n // Unlike Skia, we do not have is_constant, because we don't have Color shaders.\n\n /// If this returns false, then we draw nothing (do not fall back to shader context)\n #[must_use]\n pub(crate) fn push_stages(&self, p: &mut RasterPipelineBuilder) -> bool {\n match self {\n Shader::SolidColor(color) => {\n p.push_uniform_color(color.premultiply());\n true\n }\n Shader::LinearGradient(ref g) => g.push_stages(p),\n Shader::RadialGradient(ref g) => g.push_stages(p),\n Shader::Pattern(ref patt) => patt.push_stages(p),\n }\n }\n\n /// Transforms the shader.\n pub fn transform(&mut self, ts: Transform) {\n match self {\n Shader::SolidColor(_) => {}\n Shader::LinearGradient(g) => {\n g.base.transform = g.base.transform.post_concat(ts);\n }\n Shader::RadialGradient(g) => {\n g.base.transform = g.base.transform.post_concat(ts);\n }\n Shader::Pattern(p) => {\n p.transform = p.transform.post_concat(ts);\n }\n }\n }\n\n /// Shifts shader's opacity.\n ///\n /// `opacity` will be clamped to the 0..=1 range.\n ///\n /// This is roughly the same as Skia's `SkPaint::setAlpha`.\n ///\n /// Unlike Skia, we do not support global alpha/opacity, which is in Skia\n /// is set via the alpha channel of the `SkPaint::fColor4f`.\n /// Instead, you can shift the opacity of the shader to whatever value you need.\n ///\n /// - For `SolidColor` this function will multiply `color.alpha` by `opacity`.\n /// - For gradients this function will multiply all colors by `opacity`.\n /// - For `Pattern` this function will multiply `Patter::opacity` by `opacity`.\n pub fn apply_opacity(&mut self, opacity: f32) {\n match self {\n Shader::SolidColor(ref mut c) => {\n c.apply_opacity(opacity);\n }\n Shader::LinearGradient(g) => {\n g.base.apply_opacity(opacity);\n }\n Shader::RadialGradient(g) => {\n g.base.apply_opacity(opacity);\n }\n Shader::Pattern(ref mut p) => {\n p.opacity = NormalizedF32::new(p.opacity.get() * opacity.bound(0.0, 1.0)).unwrap();\n }\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/shaders/pattern.rs", "sha256": "96865d92752b85b20024f5aa81d36239efdb606224aa6e23c180125b0d2be1ab", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse tiny_skia_path::NormalizedF32;\n\nuse crate::{BlendMode, PixmapRef, Shader, SpreadMode, Transform};\n\nuse crate::pipeline;\nuse crate::pipeline::RasterPipelineBuilder;\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\n/// Controls how much filtering to be done when transforming images.\n#[derive(Copy, Clone, PartialEq, Debug)]\npub enum FilterQuality {\n /// Nearest-neighbor. Low quality, but fastest.\n Nearest,\n /// Bilinear.\n Bilinear,\n /// Bicubic. High quality, but slow.\n Bicubic,\n}\n\n/// Controls how a pixmap should be blended.\n///\n/// Like `Paint`, but for `Pixmap`.\n#[derive(Copy, Clone, PartialEq, Debug)]\npub struct PixmapPaint {\n /// Pixmap opacity.\n ///\n /// Must be in 0..=1 range.\n ///\n /// Default: 1.0\n pub opacity: f32,\n\n /// Pixmap blending mode.\n ///\n /// Default: SourceOver\n pub blend_mode: BlendMode,\n\n /// Specifies how much filtering to be done when transforming images.\n ///\n /// Default: Nearest\n pub quality: FilterQuality,\n}\n\nimpl Default for PixmapPaint {\n fn default() -> Self {\n PixmapPaint {\n opacity: 1.0,\n blend_mode: BlendMode::default(),\n quality: FilterQuality::Nearest,\n }\n }\n}\n\n/// A pattern shader.\n///\n/// Essentially a `SkImageShader`.\n///\n/// Unlike Skia, we do not support FilterQuality::Medium, because it involves\n/// mipmap generation, which adds too much complexity.\n#[derive(Clone, PartialEq, Debug)]\npub struct Pattern<'a> {\n pub(crate) pixmap: PixmapRef<'a>,\n quality: FilterQuality,\n spread_mode: SpreadMode,\n pub(crate) opacity: NormalizedF32,\n pub(crate) transform: Transform,\n}\n\nimpl<'a> Pattern<'a> {\n /// Creates a new pattern shader.\n ///\n /// `opacity` will be clamped to the 0..=1 range.\n #[allow(clippy::new_ret_no_self)]\n pub fn new(\n pixmap: PixmapRef<'a>,\n spread_mode: SpreadMode,\n quality: FilterQuality,\n opacity: f32,\n transform: Transform,\n ) -> Shader {\n Shader::Pattern(Pattern {\n pixmap,\n spread_mode,\n quality,\n opacity: NormalizedF32::new_clamped(opacity),\n transform,\n })\n }\n\n pub(crate) fn push_stages(&self, p: &mut RasterPipelineBuilder) -> bool {\n let ts = match self.transform.invert() {\n Some(v) => v,\n None => {\n log::warn!(\"failed to invert a pattern transform. Nothing will be rendered\");\n return false;\n }\n };\n\n p.push(pipeline::Stage::SeedShader);\n\n p.push_transform(ts);\n\n let mut quality = self.quality;\n\n if ts.is_identity() || ts.is_translate() {\n quality = FilterQuality::Nearest;\n }\n\n if quality == FilterQuality::Bilinear {\n if ts.is_translate() {\n if ts.tx == ts.tx.trunc() && ts.ty == ts.ty.trunc() {\n // When the matrix is just an integer translate, bilerp == nearest neighbor.\n quality = FilterQuality::Nearest;\n }\n }\n }\n\n // TODO: minimizing scale via mipmap\n\n match quality {\n FilterQuality::Nearest => {\n p.ctx.limit_x = pipeline::TileCtx {\n scale: self.pixmap.width() as f32,\n inv_scale: 1.0 / self.pixmap.width() as f32,\n };\n\n p.ctx.limit_y = pipeline::TileCtx {\n scale: self.pixmap.height() as f32,\n inv_scale: 1.0 / self.pixmap.height() as f32,\n };\n\n match self.spread_mode {\n SpreadMode::Pad => { /* The gather() stage will clamp for us. */ }\n SpreadMode::Repeat => p.push(pipeline::Stage::Repeat),\n SpreadMode::Reflect => p.push(pipeline::Stage::Reflect),\n }\n\n p.push(pipeline::Stage::Gather);\n }\n FilterQuality::Bilinear => {\n p.ctx.sampler = pipeline::SamplerCtx {\n spread_mode: self.spread_mode,\n inv_width: 1.0 / self.pixmap.width() as f32,\n inv_height: 1.0 / self.pixmap.height() as f32,\n };\n p.push(pipeline::Stage::Bilinear);\n }\n FilterQuality::Bicubic => {\n p.ctx.sampler = pipeline::SamplerCtx {\n spread_mode: self.spread_mode,\n inv_width: 1.0 / self.pixmap.width() as f32,\n inv_height: 1.0 / self.pixmap.height() as f32,\n };\n p.push(pipeline::Stage::Bicubic);\n\n // Bicubic filtering naturally produces out of range values on both sides of [0,1].\n p.push(pipeline::Stage::Clamp0);\n p.push(pipeline::Stage::ClampA);\n }\n }\n\n // Unlike Skia, we do not support global opacity and only Pattern allows it.\n if self.opacity != NormalizedF32::ONE {\n debug_assert_eq!(\n core::mem::size_of_val(&self.opacity),\n 4,\n \"alpha must be f32\"\n );\n p.ctx.current_coverage = self.opacity.get();\n p.push(pipeline::Stage::Scale1Float);\n }\n\n true\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/shaders/radial_gradient.rs", "sha256": "ddcd5026ccfa12dc5277e65180511698b2a2db54351032cb7a8cd2e85c84f648", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse alloc::vec::Vec;\n\nuse tiny_skia_path::Scalar;\n\nuse crate::{GradientStop, Point, Shader, SpreadMode, Transform};\n\nuse super::gradient::{Gradient, DEGENERATE_THRESHOLD};\nuse crate::pipeline;\nuse crate::pipeline::RasterPipelineBuilder;\nuse crate::wide::u32x8;\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\n#[derive(Copy, Clone, PartialEq, Debug)]\nstruct FocalData {\n r1: f32, // r1 after mapping focal point to (0, 0)\n}\n\nimpl FocalData {\n // Whether the focal point (0, 0) is on the end circle with center (1, 0) and radius r1. If\n // this is true, it's as if an aircraft is flying at Mach 1 and all circles (soundwaves)\n // will go through the focal point (aircraft). In our previous implementations, this was\n // known as the edge case where the inside circle touches the outside circle (on the focal\n // point). If we were to solve for t bruteforcely using a quadratic equation, this case\n // implies that the quadratic equation degenerates to a linear equation.\n fn is_focal_on_circle(&self) -> bool {\n (1.0 - self.r1).is_nearly_zero()\n }\n\n fn is_well_behaved(&self) -> bool {\n !self.is_focal_on_circle() && self.r1 > 1.0\n }\n}\n\n/// A radial gradient shader.\n///\n/// This is not `SkRadialGradient` like in Skia, but rather `SkTwoPointConicalGradient`\n/// without the start radius.\n#[derive(Clone, PartialEq, Debug)]\npub struct RadialGradient {\n pub(crate) base: Gradient,\n focal_data: Option,\n}\n\nimpl RadialGradient {\n /// Creates a new radial gradient shader.\n ///\n /// Returns `Shader::SolidColor` when:\n /// - `stops.len()` == 1\n ///\n /// Returns `None` when:\n ///\n /// - `stops` is empty\n /// - `radius` <= 0\n /// - `transform` is not invertible\n #[allow(clippy::new_ret_no_self)]\n pub fn new(\n start: Point,\n end: Point,\n radius: f32,\n stops: Vec,\n mode: SpreadMode,\n transform: Transform,\n ) -> Option> {\n // From SkGradientShader::MakeTwoPointConical\n\n if radius < 0.0 || radius.is_nearly_zero() {\n return None;\n }\n\n if stops.is_empty() {\n return None;\n }\n\n if stops.len() == 1 {\n return Some(Shader::SolidColor(stops[0].color));\n }\n\n transform.invert()?;\n\n let length = (end - start).length();\n if !length.is_finite() {\n return None;\n }\n\n if length.is_nearly_zero_within_tolerance(DEGENERATE_THRESHOLD) {\n // If the center positions are the same, then the gradient\n // is the radial variant of a 2 pt conical gradient,\n // an actual radial gradient (startRadius == 0),\n // or it is fully degenerate (startRadius == endRadius).\n\n let inv = radius.invert();\n let mut ts = Transform::from_translate(-start.x, -start.y);\n ts = ts.post_scale(inv, inv);\n\n // We can treat this gradient as radial, which is faster. If we got here, we know\n // that endRadius is not equal to 0, so this produces a meaningful gradient\n Some(Shader::RadialGradient(RadialGradient {\n base: Gradient::new(stops, mode, transform, ts),\n focal_data: None,\n }))\n } else {\n // From SkTwoPointConicalGradient::Create\n let mut ts = ts_from_poly_to_poly(\n start,\n end,\n Point::from_xy(0.0, 0.0),\n Point::from_xy(1.0, 0.0),\n )?;\n\n let d_center = (start - end).length();\n let r1 = radius / d_center;\n let focal_data = FocalData { r1 };\n\n // The following transformations are just to accelerate the shader computation by saving\n // some arithmetic operations.\n if focal_data.is_focal_on_circle() {\n ts = ts.post_scale(0.5, 0.5);\n } else {\n ts = ts.post_scale(r1 / (r1 * r1 - 1.0), 1.0 / ((r1 * r1 - 1.0).abs()).sqrt());\n }\n\n Some(Shader::RadialGradient(RadialGradient {\n base: Gradient::new(stops, mode, transform, ts),\n focal_data: Some(focal_data),\n }))\n }\n }\n\n pub(crate) fn push_stages(&self, p: &mut RasterPipelineBuilder) -> bool {\n let p0 = if let Some(focal_data) = self.focal_data {\n 1.0 / focal_data.r1\n } else {\n 1.0\n };\n\n p.ctx.two_point_conical_gradient = pipeline::TwoPointConicalGradientCtx {\n mask: u32x8::default(),\n p0,\n };\n\n self.base.push_stages(\n p,\n &|p| {\n if let Some(focal_data) = self.focal_data {\n // Unlike Skia, we have only the Focal radial gradient type.\n\n if focal_data.is_focal_on_circle() {\n p.push(pipeline::Stage::XYTo2PtConicalFocalOnCircle);\n } else if focal_data.is_well_behaved() {\n p.push(pipeline::Stage::XYTo2PtConicalWellBehaved);\n } else {\n p.push(pipeline::Stage::XYTo2PtConicalGreater);\n }\n\n if !focal_data.is_well_behaved() {\n p.push(pipeline::Stage::Mask2PtConicalDegenerates);\n }\n } else {\n p.push(pipeline::Stage::XYToRadius);\n }\n },\n &|p| {\n if let Some(focal_data) = self.focal_data {\n if !focal_data.is_well_behaved() {\n p.push(pipeline::Stage::ApplyVectorMask);\n }\n }\n },\n )\n }\n}\n\nfn ts_from_poly_to_poly(src1: Point, src2: Point, dst1: Point, dst2: Point) -> Option {\n let tmp = from_poly2(src1, src2);\n let res = tmp.invert()?;\n let tmp = from_poly2(dst1, dst2);\n Some(tmp.pre_concat(res))\n}\n\nfn from_poly2(p0: Point, p1: Point) -> Transform {\n Transform::from_row(\n p1.y - p0.y,\n p0.x - p1.x,\n p1.x - p0.x,\n p1.y - p0.y,\n p0.x,\n p0.y,\n )\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/wide/f32x16_t.rs", "sha256": "a4907134acfaa60fbc06c324c8595259795fae7aecd740ad268ba439155c38a9", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse super::{f32x8, u16x16};\n\n#[derive(Copy, Clone, Debug)]\n#[repr(C, align(32))]\npub struct f32x16(pub f32x8, pub f32x8);\n\nunsafe impl bytemuck::Zeroable for f32x16 {}\nunsafe impl bytemuck::Pod for f32x16 {}\n\nimpl Default for f32x16 {\n fn default() -> Self {\n Self::splat(0.0)\n }\n}\n\nimpl f32x16 {\n pub fn splat(n: f32) -> Self {\n Self(f32x8::splat(n), f32x8::splat(n))\n }\n\n #[inline]\n pub fn abs(&self) -> Self {\n // Yes, Skia does it in the same way.\n let abs = |x| bytemuck::cast::(bytemuck::cast::(x) & 0x7fffffff);\n\n let n0: [f32; 8] = self.0.into();\n let n1: [f32; 8] = self.1.into();\n Self(\n f32x8::from([\n abs(n0[0]),\n abs(n0[1]),\n abs(n0[2]),\n abs(n0[3]),\n abs(n0[4]),\n abs(n0[5]),\n abs(n0[6]),\n abs(n0[7]),\n ]),\n f32x8::from([\n abs(n1[0]),\n abs(n1[1]),\n abs(n1[2]),\n abs(n1[3]),\n abs(n1[4]),\n abs(n1[5]),\n abs(n1[6]),\n abs(n1[7]),\n ]),\n )\n }\n\n pub fn cmp_gt(self, rhs: &Self) -> Self {\n Self(self.0.cmp_gt(rhs.0), self.1.cmp_gt(rhs.1))\n }\n\n pub fn blend(self, t: Self, f: Self) -> Self {\n Self(self.0.blend(t.0, f.0), self.1.blend(t.1, f.1))\n }\n\n pub fn normalize(&self) -> Self {\n Self(self.0.normalize(), self.1.normalize())\n }\n\n pub fn floor(&self) -> Self {\n // Yes, Skia does it in the same way.\n let roundtrip = self.round();\n roundtrip\n - roundtrip\n .cmp_gt(self)\n .blend(f32x16::splat(1.0), f32x16::splat(0.0))\n }\n\n pub fn sqrt(&self) -> Self {\n Self(self.0.sqrt(), self.1.sqrt())\n }\n\n pub fn round(&self) -> Self {\n Self(self.0.round(), self.1.round())\n }\n\n // This method is too heavy and shouldn't be inlined.\n pub fn save_to_u16x16(&self, dst: &mut u16x16) {\n // Do not use to_i32x8, because it involves rounding,\n // and Skia cast's without it.\n\n let n0: [f32; 8] = self.0.into();\n let n1: [f32; 8] = self.1.into();\n\n dst.0[0] = n0[0] as u16;\n dst.0[1] = n0[1] as u16;\n dst.0[2] = n0[2] as u16;\n dst.0[3] = n0[3] as u16;\n\n dst.0[4] = n0[4] as u16;\n dst.0[5] = n0[5] as u16;\n dst.0[6] = n0[6] as u16;\n dst.0[7] = n0[7] as u16;\n\n dst.0[8] = n1[0] as u16;\n dst.0[9] = n1[1] as u16;\n dst.0[10] = n1[2] as u16;\n dst.0[11] = n1[3] as u16;\n\n dst.0[12] = n1[4] as u16;\n dst.0[13] = n1[5] as u16;\n dst.0[14] = n1[6] as u16;\n dst.0[15] = n1[7] as u16;\n }\n}\n\nimpl core::ops::Add for f32x16 {\n type Output = Self;\n\n fn add(self, rhs: Self) -> Self::Output {\n Self(self.0 + rhs.0, self.1 + rhs.1)\n }\n}\n\nimpl core::ops::Sub for f32x16 {\n type Output = Self;\n\n fn sub(self, rhs: Self) -> Self::Output {\n Self(self.0 - rhs.0, self.1 - rhs.1)\n }\n}\n\nimpl core::ops::Mul for f32x16 {\n type Output = Self;\n\n fn mul(self, rhs: Self) -> Self::Output {\n Self(self.0 * rhs.0, self.1 * rhs.1)\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/wide/f32x4_t.rs", "sha256": "7dc219c8dc68965994de82632d00f34480502096f9a8bafffd63f21f0cd76443", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// Based on https://github.com/Lokathor/wide (Zlib)\n\nuse bytemuck::cast;\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse tiny_skia_path::NoStdFloat;\n\nuse super::i32x4;\n\ncfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n #[cfg(target_arch = \"x86\")]\n use core::arch::x86::*;\n #[cfg(target_arch = \"x86_64\")]\n use core::arch::x86_64::*;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(16))]\n pub struct f32x4(__m128);\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n use core::arch::wasm32::*;\n\n // repr(transparent) allows for directly passing the v128 on the WASM stack.\n #[derive(Clone, Copy, Debug)]\n #[repr(transparent)]\n pub struct f32x4(v128);\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n use core::arch::aarch64::*;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(16))]\n pub struct f32x4(float32x4_t);\n } else {\n use super::FasterMinMax;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(16))]\n pub struct f32x4([f32; 4]);\n }\n}\n\nunsafe impl bytemuck::Zeroable for f32x4 {}\nunsafe impl bytemuck::Pod for f32x4 {}\n\nimpl Default for f32x4 {\n fn default() -> Self {\n Self::splat(0.0)\n }\n}\n\nimpl f32x4 {\n pub fn splat(n: f32) -> Self {\n Self::from([n, n, n, n])\n }\n\n pub fn floor(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_floor(self.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vrndmq_f32(self.0) })\n } else {\n let roundtrip: f32x4 = cast(self.trunc_int().to_f32x4());\n roundtrip - roundtrip.cmp_gt(self).blend(f32x4::splat(1.0), f32x4::default())\n }\n }\n }\n\n pub fn abs(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_abs(self.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vabsq_f32(self.0) })\n } else {\n let non_sign_bits = f32x4::splat(f32::from_bits(i32::MAX as u32));\n self & non_sign_bits\n }\n }\n }\n\n pub fn max(self, rhs: Self) -> Self {\n // These technically don't have the same semantics for NaN and 0, but it\n // doesn't seem to matter as Skia does it the same way.\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_max_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_pmax(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vmaxq_f32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0].faster_max(rhs.0[0]),\n self.0[1].faster_max(rhs.0[1]),\n self.0[2].faster_max(rhs.0[2]),\n self.0[3].faster_max(rhs.0[3]),\n ])\n }\n }\n }\n\n pub fn min(self, rhs: Self) -> Self {\n // These technically don't have the same semantics for NaN and 0, but it\n // doesn't seem to matter as Skia does it the same way.\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_min_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_pmin(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vminq_f32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0].faster_min(rhs.0[0]),\n self.0[1].faster_min(rhs.0[1]),\n self.0[2].faster_min(rhs.0[2]),\n self.0[3].faster_min(rhs.0[3]),\n ])\n }\n }\n }\n\n pub fn cmp_eq(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_cmpeq_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_eq(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(cast(unsafe { vceqq_f32(self.0, rhs.0) }))\n } else {\n Self([\n if self.0[0] == rhs.0[0] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[1] == rhs.0[1] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[2] == rhs.0[2] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[3] == rhs.0[3] { f32::from_bits(u32::MAX) } else { 0.0 },\n ])\n }\n }\n }\n\n pub fn cmp_ne(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_cmpneq_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_ne(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(cast(unsafe { vmvnq_u32(vceqq_f32(self.0, rhs.0)) }))\n } else {\n Self([\n if self.0[0] != rhs.0[0] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[1] != rhs.0[1] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[2] != rhs.0[2] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[3] != rhs.0[3] { f32::from_bits(u32::MAX) } else { 0.0 },\n ])\n }\n }\n }\n\n pub fn cmp_ge(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_cmpge_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_ge(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(cast(unsafe { vcgeq_f32(self.0, rhs.0) }))\n } else {\n Self([\n if self.0[0] >= rhs.0[0] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[1] >= rhs.0[1] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[2] >= rhs.0[2] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[3] >= rhs.0[3] { f32::from_bits(u32::MAX) } else { 0.0 },\n ])\n }\n }\n }\n\n pub fn cmp_gt(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_cmpgt_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_gt(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(cast(unsafe { vcgtq_f32(self.0, rhs.0) }))\n } else {\n Self([\n if self.0[0] > rhs.0[0] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[1] > rhs.0[1] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[2] > rhs.0[2] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[3] > rhs.0[3] { f32::from_bits(u32::MAX) } else { 0.0 },\n ])\n }\n }\n }\n\n pub fn cmp_le(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_cmple_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_le(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(cast(unsafe { vcleq_f32(self.0, rhs.0) }))\n } else {\n Self([\n if self.0[0] <= rhs.0[0] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[1] <= rhs.0[1] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[2] <= rhs.0[2] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[3] <= rhs.0[3] { f32::from_bits(u32::MAX) } else { 0.0 },\n ])\n }\n }\n }\n\n pub fn cmp_lt(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_cmplt_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_lt(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(cast(unsafe { vcltq_f32(self.0, rhs.0) }))\n } else {\n Self([\n if self.0[0] < rhs.0[0] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[1] < rhs.0[1] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[2] < rhs.0[2] { f32::from_bits(u32::MAX) } else { 0.0 },\n if self.0[3] < rhs.0[3] { f32::from_bits(u32::MAX) } else { 0.0 },\n ])\n }\n }\n }\n\n #[inline]\n pub fn blend(self, t: Self, f: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse4.1\"))] {\n Self(unsafe { _mm_blendv_ps(f.0, t.0, self.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(v128_bitselect(t.0, f.0, self.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { cast(vbslq_u32( cast(self.0), cast(t.0), cast(f.0))) })\n } else {\n super::generic_bit_blend(self, t, f)\n }\n }\n }\n\n pub fn round(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse4.1\"))] {\n Self(\n unsafe { _mm_round_ps(self.0, _MM_FROUND_NO_EXC | _MM_FROUND_TO_NEAREST_INT) },\n )\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_nearest(self.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vrndnq_f32(self.0) })\n } else {\n use super::u32x4;\n\n let to_int = f32x4::splat(1.0 / f32::EPSILON);\n let u: u32x4 = cast(self);\n let e: i32x4 = cast(u.shr::<23>() & u32x4::splat(0xff));\n let mut y: f32x4;\n\n let no_op_magic = i32x4::splat(0x7f + 23);\n let no_op_mask: f32x4 = cast(e.cmp_gt(no_op_magic) | e.cmp_eq(no_op_magic));\n let no_op_val: f32x4 = self;\n\n let zero_magic = i32x4::splat(0x7f - 1);\n let zero_mask: f32x4 = cast(e.cmp_lt(zero_magic));\n let zero_val: f32x4 = self * f32x4::splat(0.0);\n\n let neg_bit: f32x4 = cast(cast::(u).cmp_lt(i32x4::default()));\n let x: f32x4 = neg_bit.blend(-self, self);\n y = x + to_int - to_int - x;\n y = y.cmp_gt(f32x4::splat(0.5)).blend(\n y + x - f32x4::splat(-1.0),\n y.cmp_lt(f32x4::splat(-0.5)).blend(y + x + f32x4::splat(1.0), y + x),\n );\n y = neg_bit.blend(-y, y);\n\n no_op_mask.blend(no_op_val, zero_mask.blend(zero_val, y))\n }\n }\n }\n\n pub fn round_int(self) -> i32x4 {\n // These technically don't have the same semantics for NaN and out of\n // range values, but it doesn't seem to matter as Skia does it the same\n // way.\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n i32x4(unsafe { _mm_cvtps_epi32(self.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n i32x4(i32x4_trunc_sat_f32x4(self.round().0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n i32x4(unsafe { vcvtnq_s32_f32(self.0) } )\n } else {\n let rounded: [f32; 4] = cast(self.round());\n cast([\n rounded[0] as i32,\n rounded[1] as i32,\n rounded[2] as i32,\n rounded[3] as i32,\n ])\n }\n }\n }\n\n pub fn trunc_int(self) -> i32x4 {\n // These technically don't have the same semantics for NaN and out of\n // range values, but it doesn't seem to matter as Skia does it the same\n // way.\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n i32x4(unsafe { _mm_cvttps_epi32(self.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n i32x4(i32x4_trunc_sat_f32x4(self.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n i32x4(unsafe { vcvtq_s32_f32(self.0) })\n } else {\n cast([\n self.0[0] as i32,\n self.0[1] as i32,\n self.0[2] as i32,\n self.0[3] as i32,\n ])\n }\n }\n }\n\n pub fn recip_fast(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_rcp_ps(self.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_div(f32x4_splat(1.0), self.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n unsafe {\n let a = vrecpeq_f32(self.0);\n let a = vmulq_f32(vrecpsq_f32(self.0, a), a);\n Self(a)\n }\n } else {\n Self::from([\n 1.0 / self.0[0],\n 1.0 / self.0[1],\n 1.0 / self.0[2],\n 1.0 / self.0[3],\n ])\n }\n }\n }\n\n pub fn recip_sqrt(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_rsqrt_ps(self.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_div(f32x4_splat(1.0), f32x4_sqrt(self.0)))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n unsafe {\n let a = vrsqrteq_f32(self.0);\n let a = vmulq_f32(vrsqrtsq_f32(self.0, vmulq_f32(a, a)), a);\n Self(a)\n }\n } else {\n Self::from([\n 1.0 / self.0[0].sqrt(),\n 1.0 / self.0[1].sqrt(),\n 1.0 / self.0[2].sqrt(),\n 1.0 / self.0[3].sqrt(),\n ])\n }\n }\n }\n\n pub fn sqrt(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_sqrt_ps(self.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_sqrt(self.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vsqrtq_f32(self.0) })\n } else {\n Self::from([\n self.0[0].sqrt(),\n self.0[1].sqrt(),\n self.0[2].sqrt(),\n self.0[3].sqrt(),\n ])\n }\n }\n }\n}\n\nimpl From<[f32; 4]> for f32x4 {\n fn from(v: [f32; 4]) -> Self {\n cast(v)\n }\n}\n\nimpl From for [f32; 4] {\n fn from(v: f32x4) -> Self {\n cast(v)\n }\n}\n\nimpl core::ops::Add for f32x4 {\n type Output = Self;\n\n fn add(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_add_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_add(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vaddq_f32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0] + rhs.0[0],\n self.0[1] + rhs.0[1],\n self.0[2] + rhs.0[2],\n self.0[3] + rhs.0[3],\n ])\n }\n }\n }\n}\n\nimpl core::ops::AddAssign for f32x4 {\n fn add_assign(&mut self, rhs: f32x4) {\n *self = *self + rhs;\n }\n}\n\nimpl core::ops::Sub for f32x4 {\n type Output = Self;\n\n fn sub(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_sub_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_sub(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vsubq_f32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0] - rhs.0[0],\n self.0[1] - rhs.0[1],\n self.0[2] - rhs.0[2],\n self.0[3] - rhs.0[3],\n ])\n }\n }\n }\n}\n\nimpl core::ops::Mul for f32x4 {\n type Output = Self;\n\n fn mul(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_mul_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_mul(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vmulq_f32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0] * rhs.0[0],\n self.0[1] * rhs.0[1],\n self.0[2] * rhs.0[2],\n self.0[3] * rhs.0[3],\n ])\n }\n }\n }\n}\n\nimpl core::ops::MulAssign for f32x4 {\n fn mul_assign(&mut self, rhs: f32x4) {\n *self = *self * rhs;\n }\n}\n\nimpl core::ops::Div for f32x4 {\n type Output = Self;\n\n fn div(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_div_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(f32x4_div(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vdivq_f32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0] / rhs.0[0],\n self.0[1] / rhs.0[1],\n self.0[2] / rhs.0[2],\n self.0[3] / rhs.0[3],\n ])\n }\n }\n }\n}\n\nimpl core::ops::BitAnd for f32x4 {\n type Output = Self;\n\n #[inline(always)]\n fn bitand(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_and_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(v128_and(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(cast(unsafe { vandq_u32(cast(self.0), cast(rhs.0)) }))\n } else {\n Self([\n f32::from_bits(self.0[0].to_bits() & rhs.0[0].to_bits()),\n f32::from_bits(self.0[1].to_bits() & rhs.0[1].to_bits()),\n f32::from_bits(self.0[2].to_bits() & rhs.0[2].to_bits()),\n f32::from_bits(self.0[3].to_bits() & rhs.0[3].to_bits()),\n ])\n }\n }\n }\n}\n\nimpl core::ops::BitOr for f32x4 {\n type Output = Self;\n\n #[inline(always)]\n fn bitor(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_or_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(v128_or(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(cast(unsafe { vorrq_u32(cast(self.0), cast(rhs.0)) }))\n } else {\n Self([\n f32::from_bits(self.0[0].to_bits() | rhs.0[0].to_bits()),\n f32::from_bits(self.0[1].to_bits() | rhs.0[1].to_bits()),\n f32::from_bits(self.0[2].to_bits() | rhs.0[2].to_bits()),\n f32::from_bits(self.0[3].to_bits() | rhs.0[3].to_bits()),\n ])\n }\n }\n }\n}\n\nimpl core::ops::BitXor for f32x4 {\n type Output = Self;\n\n #[inline(always)]\n fn bitxor(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_xor_ps(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(v128_xor(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(cast(unsafe { veorq_u32(cast(self.0), cast(rhs.0)) }))\n } else {\n Self([\n f32::from_bits(self.0[0].to_bits() ^ rhs.0[0].to_bits()),\n f32::from_bits(self.0[1].to_bits() ^ rhs.0[1].to_bits()),\n f32::from_bits(self.0[2].to_bits() ^ rhs.0[2].to_bits()),\n f32::from_bits(self.0[3].to_bits() ^ rhs.0[3].to_bits()),\n ])\n }\n }\n }\n}\n\nimpl core::ops::Neg for f32x4 {\n type Output = Self;\n\n fn neg(self) -> Self {\n Self::default() - self\n }\n}\n\nimpl core::ops::Not for f32x4 {\n type Output = Self;\n\n fn not(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n unsafe {\n let all_bits = _mm_set1_ps(f32::from_bits(u32::MAX));\n Self(_mm_xor_ps(self.0, all_bits))\n }\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(v128_not(self.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(cast(unsafe { vmvnq_u32(cast(self.0)) }))\n } else {\n self ^ Self::splat(cast(u32::MAX))\n }\n }\n }\n}\n\nimpl core::cmp::PartialEq for f32x4 {\n fn eq(&self, rhs: &Self) -> bool {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n unsafe { _mm_movemask_ps(_mm_cmpeq_ps(self.0, rhs.0)) == 0b1111 }\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n unsafe { vminvq_u32(vceqq_f32(self.0, rhs.0)) != 0 }\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n u32x4_all_true(f32x4_eq(self.0, rhs.0))\n } else {\n self.0 == rhs.0\n }\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/wide/f32x8_t.rs", "sha256": "661777a07ae3bc2aad17228b067d75bb8674eebefae966bb37fff08819896104", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// Based on https://github.com/Lokathor/wide (Zlib)\n\nuse bytemuck::cast;\n\nuse super::{i32x8, u32x8};\n\ncfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n #[cfg(target_arch = \"x86\")]\n use core::arch::x86::*;\n #[cfg(target_arch = \"x86_64\")]\n use core::arch::x86_64::*;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(32))]\n pub struct f32x8(__m256);\n } else {\n use super::f32x4;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(32))]\n pub struct f32x8(pub f32x4, pub f32x4);\n }\n}\n\nunsafe impl bytemuck::Zeroable for f32x8 {}\nunsafe impl bytemuck::Pod for f32x8 {}\n\nimpl Default for f32x8 {\n fn default() -> Self {\n Self::splat(0.0)\n }\n}\n\nimpl f32x8 {\n pub fn splat(n: f32) -> Self {\n cast([n, n, n, n, n, n, n, n])\n }\n\n pub fn floor(self) -> Self {\n let roundtrip: f32x8 = cast(self.trunc_int().to_f32x8());\n roundtrip\n - roundtrip\n .cmp_gt(self)\n .blend(f32x8::splat(1.0), f32x8::default())\n }\n\n pub fn fract(self) -> Self {\n self - self.floor()\n }\n\n pub fn normalize(self) -> Self {\n self.max(f32x8::default()).min(f32x8::splat(1.0))\n }\n\n pub fn to_i32x8_bitcast(self) -> i32x8 {\n bytemuck::cast(self)\n }\n\n pub fn to_u32x8_bitcast(self) -> u32x8 {\n bytemuck::cast(self)\n }\n\n pub fn cmp_eq(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_cmp_ps(self.0, rhs.0, _CMP_EQ_OQ) })\n } else {\n Self(self.0.cmp_eq(rhs.0), self.1.cmp_eq(rhs.1))\n }\n }\n }\n\n pub fn cmp_ne(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_cmp_ps(self.0, rhs.0, _CMP_NEQ_OQ) })\n } else {\n Self(self.0.cmp_ne(rhs.0), self.1.cmp_ne(rhs.1))\n }\n }\n }\n\n pub fn cmp_ge(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_cmp_ps(self.0, rhs.0, _CMP_GE_OQ) })\n } else {\n Self(self.0.cmp_ge(rhs.0), self.1.cmp_ge(rhs.1))\n }\n }\n }\n\n pub fn cmp_gt(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_cmp_ps(self.0, rhs.0, _CMP_GT_OQ) })\n } else {\n Self(self.0.cmp_gt(rhs.0), self.1.cmp_gt(rhs.1))\n }\n }\n }\n\n pub fn cmp_le(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_cmp_ps(self.0, rhs.0, _CMP_LE_OQ) })\n } else {\n Self(self.0.cmp_le(rhs.0), self.1.cmp_le(rhs.1))\n }\n }\n }\n\n pub fn cmp_lt(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_cmp_ps(self.0, rhs.0, _CMP_LT_OQ) })\n } else {\n Self(self.0.cmp_lt(rhs.0), self.1.cmp_lt(rhs.1))\n }\n }\n }\n\n #[inline]\n pub fn blend(self, t: Self, f: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_blendv_ps(f.0, t.0, self.0) })\n } else {\n Self(self.0.blend(t.0, f.0), self.1.blend(t.1, f.1))\n }\n }\n }\n\n pub fn abs(self) -> Self {\n let non_sign_bits = f32x8::splat(f32::from_bits(i32::MAX as u32));\n self & non_sign_bits\n }\n\n pub fn max(self, rhs: Self) -> Self {\n // These technically don't have the same semantics for NaN and 0, but it\n // doesn't seem to matter as Skia does it the same way.\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_max_ps(self.0, rhs.0) })\n } else {\n Self(self.0.max(rhs.0), self.1.max(rhs.1))\n }\n }\n }\n\n pub fn min(self, rhs: Self) -> Self {\n // These technically don't have the same semantics for NaN and 0, but it\n // doesn't seem to matter as Skia does it the same way.\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_min_ps(self.0, rhs.0) })\n } else {\n Self(self.0.min(rhs.0), self.1.min(rhs.1))\n }\n }\n }\n\n pub fn is_finite(self) -> Self {\n let shifted_exp_mask = u32x8::splat(0xFF000000);\n let u: u32x8 = cast(self);\n let shift_u = u.shl::<1>();\n let out = !(shift_u & shifted_exp_mask).cmp_eq(shifted_exp_mask);\n cast(out)\n }\n\n pub fn round(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_round_ps(self.0, _MM_FROUND_NO_EXC | _MM_FROUND_TO_NEAREST_INT) })\n } else {\n Self(self.0.round(), self.1.round())\n }\n }\n }\n\n pub fn round_int(self) -> i32x8 {\n // These technically don't have the same semantics for NaN and out of\n // range values, but it doesn't seem to matter as Skia does it the same\n // way.\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n cast(unsafe { _mm256_cvtps_epi32(self.0) })\n } else {\n i32x8(self.0.round_int(), self.1.round_int())\n }\n }\n }\n\n pub fn trunc_int(self) -> i32x8 {\n // These technically don't have the same semantics for NaN and out of\n // range values, but it doesn't seem to matter as Skia does it the same\n // way.\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n cast(unsafe { _mm256_cvttps_epi32(self.0) })\n } else {\n i32x8(self.0.trunc_int(), self.1.trunc_int())\n }\n }\n }\n\n pub fn recip_fast(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_rcp_ps(self.0) })\n } else {\n Self(self.0.recip_fast(), self.1.recip_fast())\n }\n }\n }\n\n pub fn recip_sqrt(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_rsqrt_ps(self.0) })\n } else {\n Self(self.0.recip_sqrt(), self.1.recip_sqrt())\n }\n }\n }\n\n pub fn sqrt(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_sqrt_ps(self.0) })\n } else {\n Self(self.0.sqrt(), self.1.sqrt())\n }\n }\n }\n}\n\nimpl From<[f32; 8]> for f32x8 {\n fn from(v: [f32; 8]) -> Self {\n cast(v)\n }\n}\n\nimpl From for [f32; 8] {\n fn from(v: f32x8) -> Self {\n cast(v)\n }\n}\n\nimpl core::ops::Add for f32x8 {\n type Output = Self;\n\n fn add(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_add_ps(self.0, rhs.0) })\n } else {\n Self(self.0 + rhs.0, self.1 + rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::AddAssign for f32x8 {\n fn add_assign(&mut self, rhs: f32x8) {\n *self = *self + rhs;\n }\n}\n\nimpl core::ops::Sub for f32x8 {\n type Output = Self;\n\n fn sub(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_sub_ps(self.0, rhs.0) })\n } else {\n Self(self.0 - rhs.0, self.1 - rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::Mul for f32x8 {\n type Output = Self;\n\n fn mul(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_mul_ps(self.0, rhs.0) })\n } else {\n Self(self.0 * rhs.0, self.1 * rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::MulAssign for f32x8 {\n fn mul_assign(&mut self, rhs: f32x8) {\n *self = *self * rhs;\n }\n}\n\nimpl core::ops::Div for f32x8 {\n type Output = Self;\n\n fn div(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_div_ps(self.0, rhs.0) })\n } else {\n Self(self.0 / rhs.0, self.1 / rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::BitAnd for f32x8 {\n type Output = Self;\n\n #[inline(always)]\n fn bitand(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_and_ps(self.0, rhs.0) })\n } else {\n Self(self.0 & rhs.0, self.1 & rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::BitOr for f32x8 {\n type Output = Self;\n\n #[inline(always)]\n fn bitor(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_or_ps(self.0, rhs.0) })\n } else {\n Self(self.0 | rhs.0, self.1 | rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::BitXor for f32x8 {\n type Output = Self;\n\n #[inline(always)]\n fn bitxor(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n Self(unsafe { _mm256_xor_ps(self.0, rhs.0) })\n } else {\n Self(self.0 ^ rhs.0, self.1 ^ rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::Neg for f32x8 {\n type Output = Self;\n\n fn neg(self) -> Self {\n Self::default() - self\n }\n}\n\nimpl core::ops::Not for f32x8 {\n type Output = Self;\n\n fn not(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n let all_bits = unsafe { _mm256_set1_ps(f32::from_bits(u32::MAX)) };\n Self(unsafe { _mm256_xor_ps(self.0, all_bits) })\n } else {\n Self(!self.0, !self.1)\n }\n }\n }\n}\n\nimpl core::cmp::PartialEq for f32x8 {\n fn eq(&self, rhs: &Self) -> bool {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n let mask = unsafe { _mm256_cmp_ps(self.0, rhs.0, _CMP_EQ_OQ) };\n unsafe { _mm256_movemask_ps(mask) == 0b1111_1111 }\n } else {\n self.0 == rhs.0 && self.1 == rhs.1\n }\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/wide/i32x4_t.rs", "sha256": "2f4de262ed53ad6d429c140a6401b8267c8df8ebf7fefa5362865ef1ce007c05", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// Based on https://github.com/Lokathor/wide (Zlib)\n\nuse bytemuck::cast;\n\nuse super::f32x4;\n\ncfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n #[cfg(target_arch = \"x86\")]\n use core::arch::x86::*;\n #[cfg(target_arch = \"x86_64\")]\n use core::arch::x86_64::*;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(16))]\n pub struct i32x4(pub __m128i);\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n use core::arch::wasm32::*;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(16))]\n pub struct i32x4(pub v128);\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n use core::arch::aarch64::*;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(16))]\n pub struct i32x4(pub int32x4_t);\n } else {\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(16))]\n pub struct i32x4([i32; 4]);\n }\n}\n\nunsafe impl bytemuck::Zeroable for i32x4 {}\nunsafe impl bytemuck::Pod for i32x4 {}\n\nimpl Default for i32x4 {\n fn default() -> Self {\n Self::splat(0)\n }\n}\n\nimpl i32x4 {\n pub fn splat(n: i32) -> Self {\n cast([n, n, n, n])\n }\n\n pub fn blend(self, t: Self, f: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse4.1\"))] {\n Self(unsafe { _mm_blendv_epi8(f.0, t.0, self.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(v128_bitselect(t.0, f.0, self.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vbslq_s32(cast(self.0), t.0, f.0) })\n } else {\n super::generic_bit_blend(self, t, f)\n }\n }\n }\n\n pub fn cmp_eq(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n cast(Self(cast(unsafe { _mm_cmpeq_epi32(self.0, rhs.0) })))\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(i32x4_eq(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { cast(vceqq_s32(self.0, rhs.0)) })\n } else {\n Self([\n if self.0[0] == rhs.0[0] { -1 } else { 0 },\n if self.0[1] == rhs.0[1] { -1 } else { 0 },\n if self.0[2] == rhs.0[2] { -1 } else { 0 },\n if self.0[3] == rhs.0[3] { -1 } else { 0 },\n ])\n }\n }\n }\n\n pub fn cmp_gt(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n cast(Self(cast(unsafe { _mm_cmpgt_epi32(self.0, rhs.0) })))\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(i32x4_gt(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { cast(vcgtq_s32(self.0, rhs.0)) })\n } else {\n Self([\n if self.0[0] > rhs.0[0] { -1 } else { 0 },\n if self.0[1] > rhs.0[1] { -1 } else { 0 },\n if self.0[2] > rhs.0[2] { -1 } else { 0 },\n if self.0[3] > rhs.0[3] { -1 } else { 0 },\n ])\n }\n }\n }\n\n pub fn cmp_lt(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n cast(Self(cast(unsafe { _mm_cmplt_epi32(self.0, rhs.0) })))\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(i32x4_lt(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { cast(vcltq_s32(self.0, rhs.0)) })\n } else {\n Self([\n if self.0[0] < rhs.0[0] { -1 } else { 0 },\n if self.0[1] < rhs.0[1] { -1 } else { 0 },\n if self.0[2] < rhs.0[2] { -1 } else { 0 },\n if self.0[3] < rhs.0[3] { -1 } else { 0 },\n ])\n }\n }\n }\n\n pub fn to_f32x4(self) -> f32x4 {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n cast(Self(cast(unsafe { _mm_cvtepi32_ps(self.0) })))\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n cast(Self(f32x4_convert_i32x4(self.0)))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n cast(Self(unsafe { cast(vcvtq_f32_s32(self.0)) }))\n } else {\n let arr: [i32; 4] = cast(self);\n cast([\n arr[0] as f32,\n arr[1] as f32,\n arr[2] as f32,\n arr[3] as f32,\n ])\n }\n }\n }\n\n pub fn to_f32x4_bitcast(self) -> f32x4 {\n bytemuck::cast(self)\n }\n}\n\nimpl From<[i32; 4]> for i32x4 {\n fn from(v: [i32; 4]) -> Self {\n cast(v)\n }\n}\n\nimpl From for [i32; 4] {\n fn from(v: i32x4) -> Self {\n cast(v)\n }\n}\n\nimpl core::ops::Add for i32x4 {\n type Output = Self;\n\n fn add(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_add_epi32(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(i32x4_add(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vaddq_s32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0].wrapping_add(rhs.0[0]),\n self.0[1].wrapping_add(rhs.0[1]),\n self.0[2].wrapping_add(rhs.0[2]),\n self.0[3].wrapping_add(rhs.0[3]),\n ])\n }\n }\n }\n}\n\nimpl core::ops::BitAnd for i32x4 {\n type Output = Self;\n\n fn bitand(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_and_si128(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(v128_and(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vandq_s32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0] & rhs.0[0],\n self.0[1] & rhs.0[1],\n self.0[2] & rhs.0[2],\n self.0[3] & rhs.0[3],\n ])\n }\n }\n }\n}\n\nimpl core::ops::Mul for i32x4 {\n type Output = Self;\n\n fn mul(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse4.1\"))] {\n Self(unsafe { _mm_mullo_epi32(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(i32x4_mul(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vmulq_s32(self.0, rhs.0) })\n } else {\n // Cast is required, since we have to use scalar multiplication on SSE2.\n let a: [i32; 4] = cast(self);\n let b: [i32; 4] = cast(rhs);\n Self(cast([\n a[0].wrapping_mul(b[0]),\n a[1].wrapping_mul(b[1]),\n a[2].wrapping_mul(b[2]),\n a[3].wrapping_mul(b[3]),\n ]))\n }\n }\n }\n}\n\nimpl core::ops::BitOr for i32x4 {\n type Output = Self;\n\n #[inline]\n fn bitor(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_or_si128(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(v128_or(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vorrq_s32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0] | rhs.0[0],\n self.0[1] | rhs.0[1],\n self.0[2] | rhs.0[2],\n self.0[3] | rhs.0[3],\n ])\n }\n }\n }\n}\n\nimpl core::ops::BitXor for i32x4 {\n type Output = Self;\n\n #[inline]\n fn bitxor(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_xor_si128(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(v128_xor(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { veorq_s32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0] ^ rhs.0[0],\n self.0[1] ^ rhs.0[1],\n self.0[2] ^ rhs.0[2],\n self.0[3] ^ rhs.0[3],\n ])\n }\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/wide/i32x8_t.rs", "sha256": "7bfc17eacd1442e2008a3e213f04118d7c33150f5902091a59b50340e30cfe06", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// Based on https://github.com/Lokathor/wide (Zlib)\n\nuse bytemuck::cast;\n\nuse super::{f32x8, u32x8};\n\ncfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n #[cfg(target_arch = \"x86\")]\n use core::arch::x86::*;\n #[cfg(target_arch = \"x86_64\")]\n use core::arch::x86_64::*;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(32))]\n pub struct i32x8(__m256i);\n } else {\n use super::i32x4;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(32))]\n pub struct i32x8(pub i32x4, pub i32x4);\n }\n}\n\nunsafe impl bytemuck::Zeroable for i32x8 {}\nunsafe impl bytemuck::Pod for i32x8 {}\n\nimpl Default for i32x8 {\n fn default() -> Self {\n Self::splat(0)\n }\n}\n\nimpl i32x8 {\n pub fn splat(n: i32) -> Self {\n cast([n, n, n, n, n, n, n, n])\n }\n\n pub fn blend(self, t: Self, f: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n Self(unsafe { _mm256_blendv_epi8(f.0, t.0, self.0) })\n } else {\n Self(self.0.blend(t.0, f.0), self.1.blend(t.1, f.1))\n }\n }\n }\n\n pub fn cmp_eq(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n Self(unsafe { _mm256_cmpeq_epi32(self.0, rhs.0) })\n } else {\n Self(self.0.cmp_eq(rhs.0), self.1.cmp_eq(rhs.1))\n }\n }\n }\n\n pub fn cmp_gt(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n Self(unsafe { _mm256_cmpgt_epi32(self.0, rhs.0) })\n } else {\n Self(self.0.cmp_gt(rhs.0), self.1.cmp_gt(rhs.1))\n }\n }\n }\n\n pub fn cmp_lt(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n // There is no `_mm256_cmpLT_epi32`, therefore we have to use\n // `_mm256_cmpGT_epi32` and then invert the result.\n let v = unsafe { _mm256_cmpgt_epi32(self.0, rhs.0) };\n let all_bits = unsafe { _mm256_set1_epi16(-1) };\n Self(unsafe { _mm256_xor_si256(v, all_bits) })\n } else {\n Self(self.0.cmp_lt(rhs.0), self.1.cmp_lt(rhs.1))\n }\n }\n }\n\n pub fn to_f32x8(self) -> f32x8 {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n cast(unsafe { _mm256_cvtepi32_ps(self.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"avx\"))] {\n cast([self.0.to_f32x4(), self.1.to_f32x4()])\n } else {\n f32x8(self.0.to_f32x4(), self.1.to_f32x4())\n }\n }\n }\n\n pub fn to_u32x8_bitcast(self) -> u32x8 {\n bytemuck::cast(self)\n }\n\n pub fn to_f32x8_bitcast(self) -> f32x8 {\n bytemuck::cast(self)\n }\n}\n\nimpl From<[i32; 8]> for i32x8 {\n fn from(v: [i32; 8]) -> Self {\n cast(v)\n }\n}\n\nimpl From for [i32; 8] {\n fn from(v: i32x8) -> Self {\n cast(v)\n }\n}\n\nimpl core::ops::Add for i32x8 {\n type Output = Self;\n\n fn add(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n Self(unsafe { _mm256_add_epi32(self.0, rhs.0) })\n } else {\n Self(self.0 + rhs.0, self.1 + rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::BitAnd for i32x8 {\n type Output = Self;\n\n fn bitand(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n Self(unsafe { _mm256_and_si256(self.0, rhs.0) })\n } else {\n Self(self.0 & rhs.0, self.1 & rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::Mul for i32x8 {\n type Output = Self;\n\n fn mul(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n Self(unsafe { _mm256_mullo_epi32(self.0, rhs.0) })\n } else {\n Self(self.0 * rhs.0, self.1 * rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::BitOr for i32x8 {\n type Output = Self;\n\n #[inline]\n fn bitor(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n Self(unsafe { _mm256_or_si256(self.0, rhs.0) })\n } else {\n Self(self.0 | rhs.0, self.1 | rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::BitXor for i32x8 {\n type Output = Self;\n\n #[inline]\n fn bitxor(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n Self(unsafe { _mm256_xor_si256(self.0, rhs.0) })\n } else {\n Self(self.0 ^ rhs.0, self.1 ^ rhs.1)\n }\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/wide/mod.rs", "sha256": "653956086a14cbae739357bcddac8baacf56f88360100c07e3171ecccc313ee8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// This module was written from scratch, therefore there is no Google copyright.\n\n// f32x16, i32x16 and u32x16 are implemented as [Tx8; 2] and not as [T; 16].\n// This way we still can use some SIMD.\n//\n// We doesn't use #[inline] that much in this module.\n// The compiler will inline most of the methods automatically.\n// The only exception is U16x16, were we have to force inlining,\n// otherwise the performance will be horrible.\n\n#![allow(non_camel_case_types)]\n\nmod f32x16_t;\nmod f32x4_t;\nmod f32x8_t;\nmod i32x4_t;\nmod i32x8_t;\nmod u16x16_t;\nmod u32x4_t;\nmod u32x8_t;\n\npub use f32x16_t::f32x16;\npub use f32x4_t::f32x4;\npub use f32x8_t::f32x8;\npub use i32x4_t::i32x4;\npub use i32x8_t::i32x8;\npub use tiny_skia_path::f32x2;\npub use u16x16_t::u16x16;\npub use u32x4_t::u32x4;\npub use u32x8_t::u32x8;\n\n#[allow(dead_code)]\n#[inline]\npub fn generic_bit_blend(mask: T, y: T, n: T) -> T\nwhere\n T: Copy + core::ops::BitXor + core::ops::BitAnd,\n{\n n ^ ((n ^ y) & mask)\n}\n\n/// A faster and more forgiving f32 min/max implementation.\n///\n/// Unlike std one, we do not care about NaN.\n#[allow(dead_code)]\npub trait FasterMinMax {\n fn faster_min(self, rhs: f32) -> f32;\n fn faster_max(self, rhs: f32) -> f32;\n}\n\n#[allow(dead_code)]\nimpl FasterMinMax for f32 {\n fn faster_min(self, rhs: f32) -> f32 {\n if rhs < self {\n rhs\n } else {\n self\n }\n }\n\n fn faster_max(self, rhs: f32) -> f32 {\n if self < rhs {\n rhs\n } else {\n self\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/wide/u16x16_t.rs", "sha256": "fbbf313213582c8663471cb3739612840e9544d4a08ae78c1123fbbd2a6e8f54", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// No need to use explicit 256bit AVX2 SIMD.\n// `-C target-cpu=native` will autovectorize it better than us.\n// Not even sure why explicit instructions are so slow...\n//\n// On ARM AArch64 we can actually get up to 2x performance boost by using SIMD.\n//\n// We also have to inline all the methods. They are pretty large,\n// but without the inlining the performance is plummeting.\n\n#[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))]\nuse bytemuck::cast;\n#[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))]\nuse core::arch::aarch64::uint16x8_t;\n\n#[allow(non_camel_case_types)]\n#[derive(Copy, Clone, PartialEq, Default, Debug)]\npub struct u16x16(pub [u16; 16]);\n\nmacro_rules! impl_u16x16_op {\n ($a:expr, $op:ident, $b:expr) => {\n u16x16([\n $a.0[0].$op($b.0[0]),\n $a.0[1].$op($b.0[1]),\n $a.0[2].$op($b.0[2]),\n $a.0[3].$op($b.0[3]),\n $a.0[4].$op($b.0[4]),\n $a.0[5].$op($b.0[5]),\n $a.0[6].$op($b.0[6]),\n $a.0[7].$op($b.0[7]),\n $a.0[8].$op($b.0[8]),\n $a.0[9].$op($b.0[9]),\n $a.0[10].$op($b.0[10]),\n $a.0[11].$op($b.0[11]),\n $a.0[12].$op($b.0[12]),\n $a.0[13].$op($b.0[13]),\n $a.0[14].$op($b.0[14]),\n $a.0[15].$op($b.0[15]),\n ])\n };\n}\n\n#[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))]\nmacro_rules! impl_aarch64_call {\n ($f:ident, $a:expr, $b:expr) => {\n let a = $a.split();\n let b = $b.split();\n Self(bytemuck::cast([\n unsafe { core::arch::aarch64::$f(a.0, b.0) },\n unsafe { core::arch::aarch64::$f(a.1, b.1) },\n ]))\n };\n}\n\nimpl u16x16 {\n #[inline]\n pub fn splat(n: u16) -> Self {\n Self([n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n])\n }\n\n #[inline]\n pub fn as_slice(&self) -> &[u16; 16] {\n &self.0\n }\n\n #[inline]\n pub fn min(&self, rhs: &Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n impl_aarch64_call!(vminq_u16, self, rhs)\n } else {\n impl_u16x16_op!(self, min, rhs)\n }\n }\n }\n\n #[inline]\n pub fn max(&self, rhs: &Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n impl_aarch64_call!(vmaxq_u16, self, rhs)\n } else {\n impl_u16x16_op!(self, max, rhs)\n }\n }\n }\n\n #[inline]\n pub fn cmp_le(&self, rhs: &Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n impl_aarch64_call!(vcleq_u16, self, rhs)\n } else {\n Self([\n if self.0[ 0] <= rhs.0[ 0] { !0 } else { 0 },\n if self.0[ 1] <= rhs.0[ 1] { !0 } else { 0 },\n if self.0[ 2] <= rhs.0[ 2] { !0 } else { 0 },\n if self.0[ 3] <= rhs.0[ 3] { !0 } else { 0 },\n if self.0[ 4] <= rhs.0[ 4] { !0 } else { 0 },\n if self.0[ 5] <= rhs.0[ 5] { !0 } else { 0 },\n if self.0[ 6] <= rhs.0[ 6] { !0 } else { 0 },\n if self.0[ 7] <= rhs.0[ 7] { !0 } else { 0 },\n if self.0[ 8] <= rhs.0[ 8] { !0 } else { 0 },\n if self.0[ 9] <= rhs.0[ 9] { !0 } else { 0 },\n if self.0[10] <= rhs.0[10] { !0 } else { 0 },\n if self.0[11] <= rhs.0[11] { !0 } else { 0 },\n if self.0[12] <= rhs.0[12] { !0 } else { 0 },\n if self.0[13] <= rhs.0[13] { !0 } else { 0 },\n if self.0[14] <= rhs.0[14] { !0 } else { 0 },\n if self.0[15] <= rhs.0[15] { !0 } else { 0 },\n ])\n }\n }\n }\n\n #[inline]\n pub fn blend(self, t: Self, e: Self) -> Self {\n (t & self) | (e & !self)\n }\n\n #[inline]\n #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))]\n pub fn split(self) -> (uint16x8_t, uint16x8_t) {\n let pair: [uint16x8_t; 2] = cast(self.0);\n (pair[0], pair[1])\n }\n}\n\nimpl core::ops::Add for u16x16 {\n type Output = Self;\n\n #[inline]\n fn add(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n impl_aarch64_call!(vaddq_u16, self, rhs)\n } else {\n impl_u16x16_op!(self, add, rhs)\n }\n }\n }\n}\n\nimpl core::ops::Sub for u16x16 {\n type Output = Self;\n\n #[inline]\n fn sub(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n impl_aarch64_call!(vsubq_u16, self, rhs)\n } else {\n impl_u16x16_op!(self, sub, rhs)\n }\n }\n }\n}\n\nimpl core::ops::Mul for u16x16 {\n type Output = Self;\n\n #[inline]\n fn mul(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n impl_aarch64_call!(vmulq_u16, self, rhs)\n } else {\n impl_u16x16_op!(self, mul, rhs)\n }\n }\n }\n}\n\nimpl core::ops::Div for u16x16 {\n type Output = Self;\n\n #[inline]\n fn div(self, rhs: Self) -> Self::Output {\n impl_u16x16_op!(self, div, rhs)\n }\n}\n\nimpl core::ops::BitAnd for u16x16 {\n type Output = Self;\n\n #[inline]\n fn bitand(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n impl_aarch64_call!(vandq_u16, self, rhs)\n } else {\n impl_u16x16_op!(self, bitand, rhs)\n }\n }\n }\n}\n\nimpl core::ops::BitOr for u16x16 {\n type Output = Self;\n\n #[inline]\n fn bitor(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n impl_aarch64_call!(vorrq_u16, self, rhs)\n } else {\n impl_u16x16_op!(self, bitor, rhs)\n }\n }\n }\n}\n\nimpl core::ops::Not for u16x16 {\n type Output = Self;\n\n #[inline]\n fn not(self) -> Self::Output {\n u16x16([\n !self.0[0],\n !self.0[1],\n !self.0[2],\n !self.0[3],\n !self.0[4],\n !self.0[5],\n !self.0[6],\n !self.0[7],\n !self.0[8],\n !self.0[9],\n !self.0[10],\n !self.0[11],\n !self.0[12],\n !self.0[13],\n !self.0[14],\n !self.0[15],\n ])\n }\n}\n\nimpl core::ops::Shr for u16x16 {\n type Output = Self;\n\n #[inline]\n fn shr(self, rhs: Self) -> Self::Output {\n impl_u16x16_op!(self, shr, rhs)\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/wide/u32x4_t.rs", "sha256": "bdcdbbccec0171e5f743546ca89b6552182d2e9fd31c26863e9c7a1da68debae", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// Based on https://github.com/Lokathor/wide (Zlib)\n\ncfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n #[cfg(target_arch = \"x86\")]\n use core::arch::x86::*;\n #[cfg(target_arch = \"x86_64\")]\n use core::arch::x86_64::*;\n\n // unused when AVX is available\n #[cfg(not(target_feature = \"avx2\"))]\n use bytemuck::cast;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(16))]\n pub struct u32x4(__m128i);\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n use core::arch::wasm32::*;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(16))]\n pub struct u32x4(v128);\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n use core::arch::aarch64::*;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(16))]\n pub struct u32x4(uint32x4_t);\n } else {\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(16))]\n pub struct u32x4([u32; 4]);\n }\n}\n\nunsafe impl bytemuck::Zeroable for u32x4 {}\nunsafe impl bytemuck::Pod for u32x4 {}\n\nimpl Default for u32x4 {\n fn default() -> Self {\n Self::splat(0)\n }\n}\n\nimpl u32x4 {\n pub fn splat(n: u32) -> Self {\n bytemuck::cast([n, n, n, n])\n }\n\n // unused when AVX is available\n #[cfg(not(target_feature = \"avx2\"))]\n pub fn cmp_eq(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_cmpeq_epi32(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(u32x4_eq(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vceqq_u32(self.0, rhs.0) })\n } else {\n Self([\n if self.0[0] == rhs.0[0] { u32::MAX } else { 0 },\n if self.0[1] == rhs.0[1] { u32::MAX } else { 0 },\n if self.0[2] == rhs.0[2] { u32::MAX } else { 0 },\n if self.0[3] == rhs.0[3] { u32::MAX } else { 0 },\n ])\n }\n }\n }\n\n // unused when AVX is available\n #[cfg(not(target_feature = \"avx2\"))]\n pub fn shl(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n let shift = cast([RHS as u64, 0]);\n Self(unsafe { _mm_sll_epi32(self.0, shift) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(u32x4_shl(self.0, RHS as _))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vshlq_n_u32::(self.0) })\n } else {\n let u = RHS as u64;\n Self([\n self.0[0] << u,\n self.0[1] << u,\n self.0[2] << u,\n self.0[3] << u,\n ])\n }\n }\n }\n\n // unused when AVX is available\n #[cfg(not(target_feature = \"avx2\"))]\n pub fn shr(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n let shift: __m128i = cast([RHS as u64, 0]);\n Self(unsafe { _mm_srl_epi32(self.0, shift) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(u32x4_shr(self.0, RHS as _))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vshrq_n_u32::(self.0) })\n } else {\n let u = RHS as u64;\n Self([\n self.0[0] >> u,\n self.0[1] >> u,\n self.0[2] >> u,\n self.0[3] >> u,\n ])\n }\n }\n }\n}\n\nimpl core::ops::Not for u32x4 {\n type Output = Self;\n\n fn not(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n let all_bits = unsafe { _mm_set1_epi32(-1) };\n Self(unsafe { _mm_xor_si128(self.0, all_bits) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(v128_not(self.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vmvnq_u32(self.0) })\n } else {\n Self([\n !self.0[0],\n !self.0[1],\n !self.0[2],\n !self.0[3],\n ])\n }\n }\n }\n}\n\nimpl core::ops::Add for u32x4 {\n type Output = Self;\n\n fn add(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_add_epi32(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(u32x4_add(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vaddq_u32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0].wrapping_add(rhs.0[0]),\n self.0[1].wrapping_add(rhs.0[1]),\n self.0[2].wrapping_add(rhs.0[2]),\n self.0[3].wrapping_add(rhs.0[3]),\n ])\n }\n }\n }\n}\n\nimpl core::ops::BitAnd for u32x4 {\n type Output = Self;\n\n fn bitand(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"sse2\"))] {\n Self(unsafe { _mm_and_si128(self.0, rhs.0) })\n } else if #[cfg(all(feature = \"simd\", target_feature = \"simd128\"))] {\n Self(v128_and(self.0, rhs.0))\n } else if #[cfg(all(feature = \"simd\", target_arch = \"aarch64\", target_feature = \"neon\"))] {\n Self(unsafe { vandq_u32(self.0, rhs.0) })\n } else {\n Self([\n self.0[0] & rhs.0[0],\n self.0[1] & rhs.0[1],\n self.0[2] & rhs.0[2],\n self.0[3] & rhs.0[3],\n ])\n }\n }\n }\n}\n" }, { "package": "tiny-skia@0.10.0", "sourcePath": "src/wide/u32x8_t.rs", "sha256": "061feb0a51f6ca4dc715a53a9ac92f7bc163ca39366a73d9b6f2190ae30c8f6a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// Based on https://github.com/Lokathor/wide (Zlib)\n\nuse super::{f32x8, i32x8};\n\ncfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n #[cfg(target_arch = \"x86\")]\n use core::arch::x86::*;\n #[cfg(target_arch = \"x86_64\")]\n use core::arch::x86_64::*;\n\n use bytemuck::cast;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(32))]\n pub struct u32x8(__m256i);\n } else {\n use super::u32x4;\n\n #[derive(Clone, Copy, Debug)]\n #[repr(C, align(32))]\n pub struct u32x8(u32x4, u32x4);\n }\n}\n\nunsafe impl bytemuck::Zeroable for u32x8 {}\nunsafe impl bytemuck::Pod for u32x8 {}\n\nimpl Default for u32x8 {\n fn default() -> Self {\n Self::splat(0)\n }\n}\n\nimpl u32x8 {\n pub fn splat(n: u32) -> Self {\n bytemuck::cast([n, n, n, n, n, n, n, n])\n }\n\n pub fn to_i32x8_bitcast(self) -> i32x8 {\n bytemuck::cast(self)\n }\n\n pub fn to_f32x8_bitcast(self) -> f32x8 {\n bytemuck::cast(self)\n }\n\n pub fn cmp_eq(self, rhs: Self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n Self(unsafe { _mm256_cmpeq_epi32(self.0, rhs.0) })\n } else {\n Self(self.0.cmp_eq(rhs.0), self.1.cmp_eq(rhs.1))\n }\n }\n }\n\n pub fn shl(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n let shift: __m128i = cast([RHS as u64, 0]);\n Self(unsafe { _mm256_sll_epi32(self.0, shift) })\n } else {\n Self(self.0.shl::(), self.1.shl::())\n }\n }\n }\n\n pub fn shr(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n let shift: __m128i = cast([RHS as u64, 0]);\n Self(unsafe { _mm256_srl_epi32(self.0, shift) })\n } else {\n Self(self.0.shr::(), self.1.shr::())\n }\n }\n }\n}\n\nimpl core::ops::Not for u32x8 {\n type Output = Self;\n\n fn not(self) -> Self {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n let all_bits = unsafe { _mm256_set1_epi16(-1) };\n Self(unsafe { _mm256_xor_si256(self.0, all_bits) })\n } else {\n Self(!self.0, !self.1)\n }\n }\n }\n}\n\nimpl core::ops::Add for u32x8 {\n type Output = Self;\n\n fn add(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n Self(unsafe { _mm256_add_epi32(self.0, rhs.0) })\n } else {\n Self(self.0 + rhs.0, self.1 + rhs.1)\n }\n }\n }\n}\n\nimpl core::ops::BitAnd for u32x8 {\n type Output = Self;\n\n fn bitand(self, rhs: Self) -> Self::Output {\n cfg_if::cfg_if! {\n if #[cfg(all(feature = \"simd\", target_feature = \"avx2\"))] {\n Self(unsafe { _mm256_and_si256(self.0, rhs.0) })\n } else {\n Self(self.0 & rhs.0, self.1 & rhs.1)\n }\n }\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/dash.rs", "sha256": "3b7a2614cf3b72a6b48fdc348312c2562f268bc022f32f6437756aa08feae1a8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2014 Google Inc.\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// This module is a mix of SkDashPath, SkDashPathEffect, SkContourMeasure and SkPathMeasure.\n\nuse alloc::vec::Vec;\n\nuse arrayref::array_ref;\n\nuse crate::{Path, Point};\n\nuse crate::floating_point::{FiniteF32, NonZeroPositiveF32, NormalizedF32, NormalizedF32Exclusive};\nuse crate::path::{PathSegment, PathSegmentsIter, PathVerb};\nuse crate::path_builder::PathBuilder;\nuse crate::path_geometry;\nuse crate::scalar::Scalar;\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse crate::NoStdFloat;\n\n/// A stroke dashing properties.\n///\n/// Contains an array of pairs, where the first number indicates an \"on\" interval\n/// and the second one indicates an \"off\" interval;\n/// a dash offset value and internal properties.\n///\n/// # Guarantees\n///\n/// - The dash array always have an even number of values.\n/// - All dash array values are finite and >= 0.\n/// - There is at least two dash array values.\n/// - The sum of all dash array values is positive and finite.\n/// - Dash offset is finite.\n#[derive(Clone, PartialEq, Debug)]\npub struct StrokeDash {\n array: Vec,\n offset: f32,\n interval_len: NonZeroPositiveF32,\n first_len: f32, // TODO: PositiveF32\n first_index: usize,\n}\n\nimpl StrokeDash {\n /// Creates a new stroke dashing object.\n pub fn new(dash_array: Vec, dash_offset: f32) -> Option {\n let dash_offset = FiniteF32::new(dash_offset)?;\n\n if dash_array.len() < 2 || dash_array.len() % 2 != 0 {\n return None;\n }\n\n if dash_array.iter().any(|n| *n < 0.0) {\n return None;\n }\n\n let interval_len: f32 = dash_array.iter().sum();\n let interval_len = NonZeroPositiveF32::new(interval_len)?;\n\n let dash_offset = adjust_dash_offset(dash_offset.get(), interval_len.get());\n debug_assert!(dash_offset >= 0.0);\n debug_assert!(dash_offset < interval_len.get());\n\n let (first_len, first_index) = find_first_interval(&dash_array, dash_offset);\n debug_assert!(first_len >= 0.0);\n debug_assert!(first_index < dash_array.len());\n\n Some(StrokeDash {\n array: dash_array,\n offset: dash_offset,\n interval_len,\n first_len,\n first_index,\n })\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use alloc::vec;\n\n #[test]\n fn test() {\n assert_eq!(StrokeDash::new(vec![], 0.0), None);\n assert_eq!(StrokeDash::new(vec![1.0], 0.0), None);\n assert_eq!(StrokeDash::new(vec![1.0, 2.0, 3.0], 0.0), None);\n assert_eq!(StrokeDash::new(vec![1.0, -2.0], 0.0), None);\n assert_eq!(StrokeDash::new(vec![0.0, 0.0], 0.0), None);\n assert_eq!(StrokeDash::new(vec![1.0, -1.0], 0.0), None);\n assert_eq!(StrokeDash::new(vec![1.0, 1.0], f32::INFINITY), None);\n assert_eq!(StrokeDash::new(vec![1.0, f32::INFINITY], 0.0), None);\n }\n\n #[test]\n fn bug_26() {\n let mut pb = PathBuilder::new();\n pb.move_to(665.54, 287.3);\n pb.line_to(675.67, 273.04);\n pb.line_to(675.52, 271.32);\n pb.line_to(674.79, 269.61);\n pb.line_to(674.05, 268.04);\n pb.line_to(672.88, 266.47);\n pb.line_to(671.27, 264.9);\n let path = pb.finish().unwrap();\n\n let stroke_dash = StrokeDash::new(vec![6.0, 4.5], 0.0).unwrap();\n\n assert!(path.dash(&stroke_dash, 1.0).is_some());\n }\n}\n\n// Adjust phase to be between 0 and len, \"flipping\" phase if negative.\n// e.g., if len is 100, then phase of -20 (or -120) is equivalent to 80.\nfn adjust_dash_offset(mut offset: f32, len: f32) -> f32 {\n if offset < 0.0 {\n offset = -offset;\n if offset > len {\n offset %= len;\n }\n\n offset = len - offset;\n\n // Due to finite precision, it's possible that phase == len,\n // even after the subtract (if len >>> phase), so fix that here.\n debug_assert!(offset <= len);\n if offset == len {\n offset = 0.0;\n }\n\n offset\n } else if offset >= len {\n offset % len\n } else {\n offset\n }\n}\n\nfn find_first_interval(dash_array: &[f32], mut dash_offset: f32) -> (f32, usize) {\n for (i, gap) in dash_array.iter().copied().enumerate() {\n if dash_offset > gap || (dash_offset == gap && gap != 0.0) {\n dash_offset -= gap;\n } else {\n return (gap - dash_offset, i);\n }\n }\n\n // If we get here, phase \"appears\" to be larger than our length. This\n // shouldn't happen with perfect precision, but we can accumulate errors\n // during the initial length computation (rounding can make our sum be too\n // big or too small. In that event, we just have to eat the error here.\n (dash_array[0], 0)\n}\n\nimpl Path {\n /// Converts the current path into a dashed one.\n ///\n /// `resolution_scale` can be obtained via\n /// [`compute_resolution_scale`](crate::PathStroker::compute_resolution_scale).\n ///\n /// Returns `None` when more than 1_000_000 dashes had to be produced\n /// or when the final path has an invalid bounding box.\n pub fn dash(&self, dash: &StrokeDash, resolution_scale: f32) -> Option {\n dash_impl(self, dash, resolution_scale)\n }\n}\n\nfn dash_impl(src: &Path, dash: &StrokeDash, res_scale: f32) -> Option {\n // We do not support the `cull_path` branch here.\n // Skia has a lot of code for cases when a path contains only a single zero-length line\n // or when a path is a rect. Not sure why.\n // We simply ignoring it for the sake of simplicity.\n\n // We also doesn't support the `SpecialLineRec` case.\n // I have no idea what the point in it.\n\n fn is_even(x: usize) -> bool {\n x % 2 == 0\n }\n\n let mut pb = PathBuilder::new();\n let mut dash_count = 0.0;\n for contour in ContourMeasureIter::new(src, res_scale) {\n let mut skip_first_segment = contour.is_closed;\n let mut added_segment = false;\n let length = contour.length;\n let mut index = dash.first_index;\n\n // Since the path length / dash length ratio may be arbitrarily large, we can exert\n // significant memory pressure while attempting to build the filtered path. To avoid this,\n // we simply give up dashing beyond a certain threshold.\n //\n // The original bug report (http://crbug.com/165432) is based on a path yielding more than\n // 90 million dash segments and crashing the memory allocator. A limit of 1 million\n // segments seems reasonable: at 2 verbs per segment * 9 bytes per verb, this caps the\n // maximum dash memory overhead at roughly 17MB per path.\n const MAX_DASH_COUNT: usize = 1000000;\n dash_count += length * (dash.array.len() >> 1) as f32 / dash.interval_len.get();\n if dash_count > MAX_DASH_COUNT as f32 {\n return None;\n }\n\n // Using double precision to avoid looping indefinitely due to single precision rounding\n // (for extreme path_length/dash_length ratios). See test_infinite_dash() unittest.\n let mut distance = 0.0;\n let mut d_len = dash.first_len;\n\n while distance < length {\n debug_assert!(d_len >= 0.0);\n added_segment = false;\n if is_even(index) && !skip_first_segment {\n added_segment = true;\n contour.push_segment(distance as f32, (distance + d_len) as f32, true, &mut pb);\n }\n\n distance += d_len;\n\n // clear this so we only respect it the first time around\n skip_first_segment = false;\n\n // wrap around our intervals array if necessary\n index += 1;\n debug_assert!(index <= dash.array.len());\n if index == dash.array.len() {\n index = 0;\n }\n\n // fetch our next d_len\n d_len = dash.array[index];\n }\n\n // extend if we ended on a segment and we need to join up with the (skipped) initial segment\n if contour.is_closed && is_even(dash.first_index) && dash.first_len >= 0.0 {\n contour.push_segment(0.0, dash.first_len, !added_segment, &mut pb);\n }\n }\n\n pb.finish()\n}\n\nconst MAX_T_VALUE: u32 = 0x3FFFFFFF;\n\nstruct ContourMeasureIter<'a> {\n iter: PathSegmentsIter<'a>,\n tolerance: f32,\n}\n\nimpl<'a> ContourMeasureIter<'a> {\n fn new(path: &'a Path, res_scale: f32) -> Self {\n // can't use tangents, since we need [0..1..................2] to be seen\n // as definitely not a line (it is when drawn, but not parametrically)\n // so we compare midpoints\n const CHEAP_DIST_LIMIT: f32 = 0.5; // just made this value up\n\n ContourMeasureIter {\n iter: path.segments(),\n tolerance: CHEAP_DIST_LIMIT * res_scale.invert(),\n }\n }\n}\n\nimpl Iterator for ContourMeasureIter<'_> {\n type Item = ContourMeasure;\n\n // If it encounters a zero-length contour, it is skipped.\n fn next(&mut self) -> Option {\n // Note:\n // as we accumulate distance, we have to check that the result of +=\n // actually made it larger, since a very small delta might be > 0, but\n // still have no effect on distance (if distance >>> delta).\n //\n // We do this check below, and in compute_quad_segs and compute_cubic_segs\n\n let mut contour = ContourMeasure::default();\n\n let mut point_index = 0;\n let mut distance = 0.0;\n let mut have_seen_close = false;\n let mut prev_p = Point::zero();\n while let Some(seg) = self.iter.next() {\n match seg {\n PathSegment::MoveTo(p0) => {\n contour.points.push(p0);\n prev_p = p0;\n }\n PathSegment::LineTo(p0) => {\n let prev_d = distance;\n distance = contour.compute_line_seg(prev_p, p0, distance, point_index);\n\n if distance > prev_d {\n contour.points.push(p0);\n point_index += 1;\n }\n\n prev_p = p0;\n }\n PathSegment::QuadTo(p0, p1) => {\n let prev_d = distance;\n distance = contour.compute_quad_segs(\n prev_p,\n p0,\n p1,\n distance,\n 0,\n MAX_T_VALUE,\n point_index,\n self.tolerance,\n );\n\n if distance > prev_d {\n contour.points.push(p0);\n contour.points.push(p1);\n point_index += 2;\n }\n\n prev_p = p1;\n }\n PathSegment::CubicTo(p0, p1, p2) => {\n let prev_d = distance;\n distance = contour.compute_cubic_segs(\n prev_p,\n p0,\n p1,\n p2,\n distance,\n 0,\n MAX_T_VALUE,\n point_index,\n self.tolerance,\n );\n\n if distance > prev_d {\n contour.points.push(p0);\n contour.points.push(p1);\n contour.points.push(p2);\n point_index += 3;\n }\n\n prev_p = p2;\n }\n PathSegment::Close => {\n have_seen_close = true;\n }\n }\n\n // TODO: to contour iter?\n if self.iter.next_verb() == Some(PathVerb::Move) {\n break;\n }\n }\n\n if !distance.is_finite() {\n return None;\n }\n\n if have_seen_close {\n let prev_d = distance;\n let first_pt = contour.points[0];\n distance = contour.compute_line_seg(\n contour.points[point_index],\n first_pt,\n distance,\n point_index,\n );\n\n if distance > prev_d {\n contour.points.push(first_pt);\n }\n }\n\n contour.length = distance;\n contour.is_closed = have_seen_close;\n\n if contour.points.is_empty() {\n None\n } else {\n Some(contour)\n }\n }\n}\n\n#[derive(Copy, Clone, PartialEq, Debug)]\nenum SegmentType {\n Line,\n Quad,\n Cubic,\n}\n\n#[derive(Copy, Clone, Debug)]\nstruct Segment {\n distance: f32, // total distance up to this point\n point_index: usize, // index into the ContourMeasure::points array\n t_value: u32,\n kind: SegmentType,\n}\n\nimpl Segment {\n fn scalar_t(&self) -> f32 {\n debug_assert!(self.t_value <= MAX_T_VALUE);\n // 1/kMaxTValue can't be represented as a float, but it's close and the limits work fine.\n const MAX_T_RECIPROCAL: f32 = 1.0 / MAX_T_VALUE as f32;\n self.t_value as f32 * MAX_T_RECIPROCAL\n }\n}\n\n#[derive(Default, Debug)]\nstruct ContourMeasure {\n segments: Vec,\n points: Vec,\n length: f32,\n is_closed: bool,\n}\n\nimpl ContourMeasure {\n fn push_segment(\n &self,\n mut start_d: f32,\n mut stop_d: f32,\n start_with_move_to: bool,\n pb: &mut PathBuilder,\n ) {\n if start_d < 0.0 {\n start_d = 0.0;\n }\n\n if stop_d > self.length {\n stop_d = self.length;\n }\n\n if !(start_d <= stop_d) {\n // catch NaN values as well\n return;\n }\n\n if self.segments.is_empty() {\n return;\n }\n\n let (seg_index, mut start_t) = match self.distance_to_segment(start_d) {\n Some(v) => v,\n None => return,\n };\n let mut seg = self.segments[seg_index];\n\n let (stop_seg_index, stop_t) = match self.distance_to_segment(stop_d) {\n Some(v) => v,\n None => return,\n };\n let stop_seg = self.segments[stop_seg_index];\n\n debug_assert!(stop_seg_index <= stop_seg_index);\n let mut p = Point::zero();\n if start_with_move_to {\n compute_pos_tan(\n &self.points[seg.point_index..],\n seg.kind,\n start_t,\n Some(&mut p),\n None,\n );\n pb.move_to(p.x, p.y);\n }\n\n if seg.point_index == stop_seg.point_index {\n segment_to(\n &self.points[seg.point_index..],\n seg.kind,\n start_t,\n stop_t,\n pb,\n );\n } else {\n let mut new_seg_index = seg_index;\n loop {\n segment_to(\n &self.points[seg.point_index..],\n seg.kind,\n start_t,\n NormalizedF32::ONE,\n pb,\n );\n\n let old_point_index = seg.point_index;\n loop {\n new_seg_index += 1;\n if self.segments[new_seg_index].point_index != old_point_index {\n break;\n }\n }\n seg = self.segments[new_seg_index];\n\n start_t = NormalizedF32::ZERO;\n\n if seg.point_index >= stop_seg.point_index {\n break;\n }\n }\n\n segment_to(\n &self.points[seg.point_index..],\n seg.kind,\n NormalizedF32::ZERO,\n stop_t,\n pb,\n );\n }\n }\n\n fn distance_to_segment(&self, distance: f32) -> Option<(usize, NormalizedF32)> {\n debug_assert!(distance >= 0.0 && distance <= self.length);\n\n let mut index = find_segment(&self.segments, distance);\n // don't care if we hit an exact match or not, so we xor index if it is negative\n index ^= index >> 31;\n let index = index as usize;\n let seg = self.segments[index];\n\n // now interpolate t-values with the prev segment (if possible)\n let mut start_t = 0.0;\n let mut start_d = 0.0;\n // check if the prev segment is legal, and references the same set of points\n if index > 0 {\n start_d = self.segments[index - 1].distance;\n if self.segments[index - 1].point_index == seg.point_index {\n debug_assert!(self.segments[index - 1].kind == seg.kind);\n start_t = self.segments[index - 1].scalar_t();\n }\n }\n\n debug_assert!(seg.scalar_t() > start_t);\n debug_assert!(distance >= start_d);\n debug_assert!(seg.distance > start_d);\n\n let t =\n start_t + (seg.scalar_t() - start_t) * (distance - start_d) / (seg.distance - start_d);\n let t = NormalizedF32::new(t)?;\n Some((index, t))\n }\n\n fn compute_line_seg(\n &mut self,\n p0: Point,\n p1: Point,\n mut distance: f32,\n point_index: usize,\n ) -> f32 {\n let d = p0.distance(p1);\n debug_assert!(d >= 0.0);\n let prev_d = distance;\n distance += d;\n if distance > prev_d {\n debug_assert!(point_index < self.points.len());\n self.segments.push(Segment {\n distance,\n point_index,\n t_value: MAX_T_VALUE,\n kind: SegmentType::Line,\n });\n }\n\n distance\n }\n\n fn compute_quad_segs(\n &mut self,\n p0: Point,\n p1: Point,\n p2: Point,\n mut distance: f32,\n min_t: u32,\n max_t: u32,\n point_index: usize,\n tolerance: f32,\n ) -> f32 {\n if t_span_big_enough(max_t - min_t) != 0 && quad_too_curvy(p0, p1, p2, tolerance) {\n let mut tmp = [Point::zero(); 5];\n let half_t = (min_t + max_t) >> 1;\n\n path_geometry::chop_quad_at(&[p0, p1, p2], NormalizedF32Exclusive::HALF, &mut tmp);\n distance = self.compute_quad_segs(\n tmp[0],\n tmp[1],\n tmp[2],\n distance,\n min_t,\n half_t,\n point_index,\n tolerance,\n );\n distance = self.compute_quad_segs(\n tmp[2],\n tmp[3],\n tmp[4],\n distance,\n half_t,\n max_t,\n point_index,\n tolerance,\n );\n } else {\n let d = p0.distance(p2);\n let prev_d = distance;\n distance += d;\n if distance > prev_d {\n debug_assert!(point_index < self.points.len());\n self.segments.push(Segment {\n distance,\n point_index,\n t_value: max_t,\n kind: SegmentType::Quad,\n });\n }\n }\n\n distance\n }\n\n fn compute_cubic_segs(\n &mut self,\n p0: Point,\n p1: Point,\n p2: Point,\n p3: Point,\n mut distance: f32,\n min_t: u32,\n max_t: u32,\n point_index: usize,\n tolerance: f32,\n ) -> f32 {\n if t_span_big_enough(max_t - min_t) != 0 && cubic_too_curvy(p0, p1, p2, p3, tolerance) {\n let mut tmp = [Point::zero(); 7];\n let half_t = (min_t + max_t) >> 1;\n\n path_geometry::chop_cubic_at2(\n &[p0, p1, p2, p3],\n NormalizedF32Exclusive::HALF,\n &mut tmp,\n );\n distance = self.compute_cubic_segs(\n tmp[0],\n tmp[1],\n tmp[2],\n tmp[3],\n distance,\n min_t,\n half_t,\n point_index,\n tolerance,\n );\n distance = self.compute_cubic_segs(\n tmp[3],\n tmp[4],\n tmp[5],\n tmp[6],\n distance,\n half_t,\n max_t,\n point_index,\n tolerance,\n );\n } else {\n let d = p0.distance(p3);\n let prev_d = distance;\n distance += d;\n if distance > prev_d {\n debug_assert!(point_index < self.points.len());\n self.segments.push(Segment {\n distance,\n point_index,\n t_value: max_t,\n kind: SegmentType::Cubic,\n });\n }\n }\n\n distance\n }\n}\n\nfn find_segment(base: &[Segment], key: f32) -> i32 {\n let mut lo = 0u32;\n let mut hi = (base.len() - 1) as u32;\n\n while lo < hi {\n let mid = (hi + lo) >> 1;\n if base[mid as usize].distance < key {\n lo = mid + 1;\n } else {\n hi = mid;\n }\n }\n\n if base[hi as usize].distance < key {\n hi += 1;\n hi = !hi;\n } else if key < base[hi as usize].distance {\n hi = !hi;\n }\n\n hi as i32\n}\n\nfn compute_pos_tan(\n points: &[Point],\n seg_kind: SegmentType,\n t: NormalizedF32,\n pos: Option<&mut Point>,\n tangent: Option<&mut Point>,\n) {\n match seg_kind {\n SegmentType::Line => {\n if let Some(pos) = pos {\n *pos = Point::from_xy(\n interp(points[0].x, points[1].x, t),\n interp(points[0].y, points[1].y, t),\n );\n }\n\n if let Some(tangent) = tangent {\n tangent.set_normalize(points[1].x - points[0].x, points[1].y - points[0].y);\n }\n }\n SegmentType::Quad => {\n let src = array_ref![points, 0, 3];\n if let Some(pos) = pos {\n *pos = path_geometry::eval_quad_at(src, t);\n }\n\n if let Some(tangent) = tangent {\n *tangent = path_geometry::eval_quad_tangent_at(src, t);\n tangent.normalize();\n }\n }\n SegmentType::Cubic => {\n let src = array_ref![points, 0, 4];\n if let Some(pos) = pos {\n *pos = path_geometry::eval_cubic_pos_at(src, t);\n }\n\n if let Some(tangent) = tangent {\n *tangent = path_geometry::eval_cubic_tangent_at(src, t);\n tangent.normalize();\n }\n }\n }\n}\n\nfn segment_to(\n points: &[Point],\n seg_kind: SegmentType,\n start_t: NormalizedF32,\n stop_t: NormalizedF32,\n pb: &mut PathBuilder,\n) {\n debug_assert!(start_t <= stop_t);\n\n if start_t == stop_t {\n if let Some(pt) = pb.last_point() {\n // If the dash as a zero-length on segment, add a corresponding zero-length line.\n // The stroke code will add end caps to zero length lines as appropriate.\n pb.line_to(pt.x, pt.y);\n }\n\n return;\n }\n\n match seg_kind {\n SegmentType::Line => {\n if stop_t == NormalizedF32::ONE {\n pb.line_to(points[1].x, points[1].y);\n } else {\n pb.line_to(\n interp(points[0].x, points[1].x, stop_t),\n interp(points[0].y, points[1].y, stop_t),\n );\n }\n }\n SegmentType::Quad => {\n let mut tmp0 = [Point::zero(); 5];\n let mut tmp1 = [Point::zero(); 5];\n if start_t == NormalizedF32::ZERO {\n if stop_t == NormalizedF32::ONE {\n pb.quad_to_pt(points[1], points[2]);\n } else {\n let stop_t = NormalizedF32Exclusive::new_bounded(stop_t.get());\n path_geometry::chop_quad_at(points, stop_t, &mut tmp0);\n pb.quad_to_pt(tmp0[1], tmp0[2]);\n }\n } else {\n let start_tt = NormalizedF32Exclusive::new_bounded(start_t.get());\n path_geometry::chop_quad_at(points, start_tt, &mut tmp0);\n if stop_t == NormalizedF32::ONE {\n pb.quad_to_pt(tmp0[3], tmp0[4]);\n } else {\n let new_t = (stop_t.get() - start_t.get()) / (1.0 - start_t.get());\n let new_t = NormalizedF32Exclusive::new_bounded(new_t);\n path_geometry::chop_quad_at(&tmp0[2..], new_t, &mut tmp1);\n pb.quad_to_pt(tmp1[1], tmp1[2]);\n }\n }\n }\n SegmentType::Cubic => {\n let mut tmp0 = [Point::zero(); 7];\n let mut tmp1 = [Point::zero(); 7];\n if start_t == NormalizedF32::ZERO {\n if stop_t == NormalizedF32::ONE {\n pb.cubic_to_pt(points[1], points[2], points[3]);\n } else {\n let stop_t = NormalizedF32Exclusive::new_bounded(stop_t.get());\n path_geometry::chop_cubic_at2(array_ref![points, 0, 4], stop_t, &mut tmp0);\n pb.cubic_to_pt(tmp0[1], tmp0[2], tmp0[3]);\n }\n } else {\n let start_tt = NormalizedF32Exclusive::new_bounded(start_t.get());\n path_geometry::chop_cubic_at2(array_ref![points, 0, 4], start_tt, &mut tmp0);\n if stop_t == NormalizedF32::ONE {\n pb.cubic_to_pt(tmp0[4], tmp0[5], tmp0[6]);\n } else {\n let new_t = (stop_t.get() - start_t.get()) / (1.0 - start_t.get());\n let new_t = NormalizedF32Exclusive::new_bounded(new_t);\n path_geometry::chop_cubic_at2(array_ref![tmp0, 3, 4], new_t, &mut tmp1);\n pb.cubic_to_pt(tmp1[1], tmp1[2], tmp1[3]);\n }\n }\n }\n }\n}\n\nfn t_span_big_enough(t_span: u32) -> u32 {\n debug_assert!(t_span <= MAX_T_VALUE);\n t_span >> 10\n}\n\nfn quad_too_curvy(p0: Point, p1: Point, p2: Point, tolerance: f32) -> bool {\n // diff = (a/4 + b/2 + c/4) - (a/2 + c/2)\n // diff = -a/4 + b/2 - c/4\n let dx = (p1.x).half() - (p0.x + p2.x).half().half();\n let dy = (p1.y).half() - (p0.y + p2.y).half().half();\n\n let dist = dx.abs().max(dy.abs());\n dist > tolerance\n}\n\nfn cubic_too_curvy(p0: Point, p1: Point, p2: Point, p3: Point, tolerance: f32) -> bool {\n let n0 = cheap_dist_exceeds_limit(\n p1,\n interp_safe(p0.x, p3.x, 1.0 / 3.0),\n interp_safe(p0.y, p3.y, 1.0 / 3.0),\n tolerance,\n );\n\n let n1 = cheap_dist_exceeds_limit(\n p2,\n interp_safe(p0.x, p3.x, 2.0 / 3.0),\n interp_safe(p0.y, p3.y, 2.0 / 3.0),\n tolerance,\n );\n\n n0 || n1\n}\n\nfn cheap_dist_exceeds_limit(pt: Point, x: f32, y: f32, tolerance: f32) -> bool {\n let dist = (x - pt.x).abs().max((y - pt.y).abs());\n // just made up the 1/2\n dist > tolerance\n}\n\n/// Linearly interpolate between A and B, based on t.\n///\n/// If t is 0, return A. If t is 1, return B else interpolate.\nfn interp(a: f32, b: f32, t: NormalizedF32) -> f32 {\n a + (b - a) * t.get()\n}\n\nfn interp_safe(a: f32, b: f32, t: f32) -> f32 {\n debug_assert!(t >= 0.0 && t <= 1.0);\n a + (b - a) * t\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/f32x2_t.rs", "sha256": "cd99d6e1fa00ed9c544af718b3f01b68a6eb500d183d3eeced804effb96878f8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse crate::NoStdFloat;\n\n// Right now, there are no visible benefits of using SIMD for f32x2. So we don't.\n/// A pair of f32 numbers.\n///\n/// Mainly for internal use. Do not rely on it!\n#[allow(non_camel_case_types)]\n#[derive(Copy, Clone, Default, PartialEq, Debug)]\npub struct f32x2(pub [f32; 2]);\n\nimpl f32x2 {\n /// Creates a new pair.\n pub fn new(a: f32, b: f32) -> f32x2 {\n f32x2([a, b])\n }\n\n /// Creates a new pair from a single value.\n pub fn splat(x: f32) -> f32x2 {\n f32x2([x, x])\n }\n\n /// Returns an absolute value.\n pub fn abs(self) -> f32x2 {\n f32x2([self.x().abs(), self.y().abs()])\n }\n\n /// Returns a minimum value.\n pub fn min(self, other: f32x2) -> f32x2 {\n f32x2([pmin(self.x(), other.x()), pmin(self.y(), other.y())])\n }\n\n /// Returns a maximum value.\n pub fn max(self, other: f32x2) -> f32x2 {\n f32x2([pmax(self.x(), other.x()), pmax(self.y(), other.y())])\n }\n\n /// Returns a maximum of both values.\n pub fn max_component(self) -> f32 {\n pmax(self.x(), self.y())\n }\n\n /// Returns the first value.\n pub fn x(&self) -> f32 {\n self.0[0]\n }\n\n /// Returns the second value.\n pub fn y(&self) -> f32 {\n self.0[1]\n }\n}\n\nimpl core::ops::Add for f32x2 {\n type Output = f32x2;\n\n fn add(self, other: f32x2) -> f32x2 {\n f32x2([self.x() + other.x(), self.y() + other.y()])\n }\n}\n\nimpl core::ops::Sub for f32x2 {\n type Output = f32x2;\n\n fn sub(self, other: f32x2) -> f32x2 {\n f32x2([self.x() - other.x(), self.y() - other.y()])\n }\n}\n\nimpl core::ops::Mul for f32x2 {\n type Output = f32x2;\n\n fn mul(self, other: f32x2) -> f32x2 {\n f32x2([self.x() * other.x(), self.y() * other.y()])\n }\n}\n\nimpl core::ops::Div for f32x2 {\n type Output = f32x2;\n\n fn div(self, other: f32x2) -> f32x2 {\n f32x2([self.x() / other.x(), self.y() / other.y()])\n }\n}\n\n// A faster and more forgiving f32 min/max implementation.\n//\n// Unlike std one, we do not care about NaN.\n\nfn pmax(a: f32, b: f32) -> f32 {\n if a < b {\n b\n } else {\n a\n }\n}\n\nfn pmin(a: f32, b: f32) -> f32 {\n if b < a {\n b\n } else {\n a\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/f32x4_t.rs", "sha256": "6328e5670e9dffa3443b4ab2d7c7f1ab36e2ac14511c54c824e782c89a955607", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// Right now, there are no visible benefits of using SIMD for f32x4. So we don't.\n#[derive(Default, Clone, Copy, PartialEq, Debug)]\n#[repr(C, align(16))]\npub struct f32x4(pub [f32; 4]);\n\nimpl f32x4 {\n pub fn max(self, rhs: Self) -> Self {\n Self([\n self.0[0].max(rhs.0[0]),\n self.0[1].max(rhs.0[1]),\n self.0[2].max(rhs.0[2]),\n self.0[3].max(rhs.0[3]),\n ])\n }\n\n pub fn min(self, rhs: Self) -> Self {\n Self([\n self.0[0].min(rhs.0[0]),\n self.0[1].min(rhs.0[1]),\n self.0[2].min(rhs.0[2]),\n self.0[3].min(rhs.0[3]),\n ])\n }\n}\n\nimpl core::ops::Add for f32x4 {\n type Output = Self;\n\n fn add(self, rhs: Self) -> Self::Output {\n Self([\n self.0[0] + rhs.0[0],\n self.0[1] + rhs.0[1],\n self.0[2] + rhs.0[2],\n self.0[3] + rhs.0[3],\n ])\n }\n}\n\nimpl core::ops::AddAssign for f32x4 {\n fn add_assign(&mut self, rhs: f32x4) {\n *self = *self + rhs;\n }\n}\n\nimpl core::ops::Sub for f32x4 {\n type Output = Self;\n\n fn sub(self, rhs: Self) -> Self::Output {\n Self([\n self.0[0] - rhs.0[0],\n self.0[1] - rhs.0[1],\n self.0[2] - rhs.0[2],\n self.0[3] - rhs.0[3],\n ])\n }\n}\n\nimpl core::ops::Mul for f32x4 {\n type Output = Self;\n\n fn mul(self, rhs: Self) -> Self::Output {\n Self([\n self.0[0] * rhs.0[0],\n self.0[1] * rhs.0[1],\n self.0[2] * rhs.0[2],\n self.0[3] * rhs.0[3],\n ])\n }\n}\n\nimpl core::ops::MulAssign for f32x4 {\n fn mul_assign(&mut self, rhs: f32x4) {\n *self = *self * rhs;\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/floating_point.rs", "sha256": "ef655a1eb3146fd0a77c57b584f164af73b51e56d36c485ef67fe139bbde11af", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse crate::scalar::Scalar;\n\npub use strict_num::{FiniteF32, NonZeroPositiveF32, NormalizedF32};\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse crate::NoStdFloat;\n\npub(crate) const FLOAT_PI: f32 = 3.14159265;\n\nconst MAX_I32_FITS_IN_F32: f32 = 2147483520.0;\nconst MIN_I32_FITS_IN_F32: f32 = -MAX_I32_FITS_IN_F32;\n\n// TODO: is there an std alternative?\n/// Custom float to integer conversion routines.\npub trait SaturateCast: Sized {\n /// Return the closest integer for the given float.\n fn saturate_from(n: T) -> Self;\n}\n\nimpl SaturateCast for i32 {\n /// Return the closest integer for the given float.\n ///\n /// Returns MAX_I32_FITS_IN_F32 for NaN.\n fn saturate_from(mut x: f32) -> Self {\n x = if x < MAX_I32_FITS_IN_F32 {\n x\n } else {\n MAX_I32_FITS_IN_F32\n };\n x = if x > MIN_I32_FITS_IN_F32 {\n x\n } else {\n MIN_I32_FITS_IN_F32\n };\n x as i32\n }\n}\n\nimpl SaturateCast for i32 {\n /// Return the closest integer for the given double.\n ///\n /// Returns i32::MAX for NaN.\n fn saturate_from(mut x: f64) -> Self {\n x = if x < i32::MAX as f64 {\n x\n } else {\n i32::MAX as f64\n };\n x = if x > i32::MIN as f64 {\n x\n } else {\n i32::MIN as f64\n };\n x as i32\n }\n}\n\n/// Custom float to integer rounding routines.\n#[allow(missing_docs)]\npub trait SaturateRound: SaturateCast {\n fn saturate_floor(n: T) -> Self;\n fn saturate_ceil(n: T) -> Self;\n fn saturate_round(n: T) -> Self;\n}\n\nimpl SaturateRound for i32 {\n fn saturate_floor(x: f32) -> Self {\n Self::saturate_from(x.floor())\n }\n\n fn saturate_ceil(x: f32) -> Self {\n Self::saturate_from(x.ceil())\n }\n\n fn saturate_round(x: f32) -> Self {\n Self::saturate_from(x.floor() + 0.5)\n }\n}\n\n/// Return the float as a 2s compliment int. Just to be used to compare floats\n/// to each other or against positive float-bit-constants (like 0). This does\n/// not return the int equivalent of the float, just something cheaper for\n/// compares-only.\npub(crate) fn f32_as_2s_compliment(x: f32) -> i32 {\n sign_bit_to_2s_compliment(bytemuck::cast(x))\n}\n\n/// Convert a sign-bit int (i.e. float interpreted as int) into a 2s compliement\n/// int. This also converts -0 (0x80000000) to 0. Doing this to a float allows\n/// it to be compared using normal C operators (<, <=, etc.)\nfn sign_bit_to_2s_compliment(mut x: i32) -> i32 {\n if x < 0 {\n x &= 0x7FFFFFFF;\n x = -x;\n }\n\n x\n}\n\n/// An immutable `f32` that is larger than 0 but less then 1.\n#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Default, Debug)]\n#[repr(transparent)]\npub struct NormalizedF32Exclusive(FiniteF32);\n\nimpl NormalizedF32Exclusive {\n /// Just a random, valid number.\n pub const ANY: Self = Self::HALF;\n\n /// A predefined 0.5 value.\n pub const HALF: Self = NormalizedF32Exclusive(unsafe { FiniteF32::new_unchecked(0.5) });\n\n /// Creates a `NormalizedF32Exclusive`.\n pub fn new(n: f32) -> Option {\n if n > 0.0 && n < 1.0 {\n // `n` is guarantee to be finite after the bounds check.\n FiniteF32::new(n).map(NormalizedF32Exclusive)\n } else {\n None\n }\n }\n\n /// Creates a `NormalizedF32Exclusive` clamping the given value.\n ///\n /// Returns zero in case of NaN or infinity.\n pub fn new_bounded(n: f32) -> Self {\n let n = n.bound(core::f32::EPSILON, 1.0 - core::f32::EPSILON);\n // `n` is guarantee to be finite after clamping.\n debug_assert!(n.is_finite());\n NormalizedF32Exclusive(unsafe { FiniteF32::new_unchecked(n) })\n }\n\n /// Returns the value as a primitive type.\n pub fn get(self) -> f32 {\n self.0.get()\n }\n\n /// Returns the value as a `FiniteF32`.\n pub fn to_normalized(self) -> NormalizedF32 {\n // NormalizedF32 is (0,1), while NormalizedF32 is [0,1], so it will always fit.\n unsafe { NormalizedF32::new_unchecked(self.0.get()) }\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/lib.rs", "sha256": "497bff2609b15972184c9730facc3d08082d7371c6e23f40065f45b5acf66135", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n//! A [tiny-skia](https://github.com/RazrFalcon/tiny-skia) Bezier path implementation.\n//!\n//! Provides a memory-efficient Bezier path container, path builder, path stroker and path dasher.\n//!\n//! Also provides some basic geometry types, but they will be moved to an external crate eventually.\n//!\n//! Note that all types use single precision floats (`f32`), just like [Skia](https://skia.org/).\n\n#![no_std]\n#![warn(missing_docs)]\n#![warn(missing_copy_implementations)]\n#![warn(missing_debug_implementations)]\n#![allow(clippy::approx_constant)]\n#![allow(clippy::collapsible_if)]\n#![allow(clippy::eq_op)]\n#![allow(clippy::excessive_precision)]\n#![allow(clippy::identity_op)]\n#![allow(clippy::manual_range_contains)]\n#![allow(clippy::neg_cmp_op_on_partial_ord)]\n#![allow(clippy::too_many_arguments)]\n#![allow(clippy::upper_case_acronyms)]\n#![allow(clippy::wrong_self_convention)]\n\n#[cfg(not(any(feature = \"std\", feature = \"no-std-float\")))]\ncompile_error!(\"You have to activate either the `std` or the `no-std-float` feature.\");\n\n#[cfg(feature = \"std\")]\nextern crate std;\n\nextern crate alloc;\n\nmod dash;\nmod f32x2_t;\nmod f32x4_t;\nmod floating_point;\nmod path;\nmod path_builder;\npub mod path_geometry;\nmod rect;\nmod scalar;\nmod size;\nmod stroker;\nmod transform;\n\npub use dash::StrokeDash;\npub use f32x2_t::f32x2;\npub use floating_point::*;\npub use path::*;\npub use path_builder::*;\npub use rect::*;\npub use scalar::*;\npub use size::*;\npub use stroker::*;\npub use transform::*;\n\n/// An integer length that is guarantee to be > 0\ntype LengthU32 = core::num::NonZeroU32;\n\n/// A point.\n///\n/// Doesn't guarantee to be finite.\n#[allow(missing_docs)]\n#[repr(C)]\n#[derive(Copy, Clone, PartialEq, Default, Debug)]\npub struct Point {\n pub x: f32,\n pub y: f32,\n}\n\nimpl From<(f32, f32)> for Point {\n #[inline]\n fn from(v: (f32, f32)) -> Self {\n Point { x: v.0, y: v.1 }\n }\n}\n\nimpl Point {\n /// Creates a new `Point`.\n pub fn from_xy(x: f32, y: f32) -> Self {\n Point { x, y }\n }\n\n /// Creates a new `Point` from `f32x2`.\n pub fn from_f32x2(r: f32x2) -> Self {\n Point::from_xy(r.x(), r.y())\n }\n\n /// Converts a `Point` into a `f32x2`.\n pub fn to_f32x2(&self) -> f32x2 {\n f32x2::new(self.x, self.y)\n }\n\n /// Creates a point at 0x0 position.\n pub fn zero() -> Self {\n Point { x: 0.0, y: 0.0 }\n }\n\n /// Returns true if x and y are both zero.\n pub fn is_zero(&self) -> bool {\n self.x == 0.0 && self.y == 0.0\n }\n\n /// Returns true if both x and y are measurable values.\n ///\n /// Both values are other than infinities and NaN.\n pub fn is_finite(&self) -> bool {\n (self.x * self.y).is_finite()\n }\n\n /// Checks that two `Point`s are almost equal.\n pub(crate) fn almost_equal(&self, other: Point) -> bool {\n !(*self - other).can_normalize()\n }\n\n /// Checks that two `Point`s are almost equal using the specified tolerance.\n pub(crate) fn equals_within_tolerance(&self, other: Point, tolerance: f32) -> bool {\n (self.x - other.x).is_nearly_zero_within_tolerance(tolerance)\n && (self.y - other.y).is_nearly_zero_within_tolerance(tolerance)\n }\n\n /// Scales (fX, fY) so that length() returns one, while preserving ratio of fX to fY,\n /// if possible.\n ///\n /// If prior length is nearly zero, sets vector to (0, 0) and returns\n /// false; otherwise returns true.\n pub fn normalize(&mut self) -> bool {\n self.set_length_from(self.x, self.y, 1.0)\n }\n\n /// Sets vector to (x, y) scaled so length() returns one, and so that (x, y)\n /// is proportional to (x, y).\n ///\n /// If (x, y) length is nearly zero, sets vector to (0, 0) and returns false;\n /// otherwise returns true.\n pub fn set_normalize(&mut self, x: f32, y: f32) -> bool {\n self.set_length_from(x, y, 1.0)\n }\n\n pub(crate) fn can_normalize(&self) -> bool {\n self.x.is_finite() && self.y.is_finite() && (self.x != 0.0 || self.y != 0.0)\n }\n\n /// Returns the Euclidean distance from origin.\n pub fn length(&self) -> f32 {\n let mag2 = self.x * self.x + self.y * self.y;\n if mag2.is_finite() {\n mag2.sqrt()\n } else {\n let xx = f64::from(self.x);\n let yy = f64::from(self.y);\n (xx * xx + yy * yy).sqrt() as f32\n }\n }\n\n /// Scales vector so that distanceToOrigin() returns length, if possible.\n ///\n /// If former length is nearly zero, sets vector to (0, 0) and return false;\n /// otherwise returns true.\n pub fn set_length(&mut self, length: f32) -> bool {\n self.set_length_from(self.x, self.y, length)\n }\n\n /// Sets vector to (x, y) scaled to length, if possible.\n ///\n /// If former length is nearly zero, sets vector to (0, 0) and return false;\n /// otherwise returns true.\n pub fn set_length_from(&mut self, x: f32, y: f32, length: f32) -> bool {\n set_point_length(self, x, y, length, &mut None)\n }\n\n /// Returns the Euclidean distance from origin.\n pub fn distance(&self, other: Point) -> f32 {\n (*self - other).length()\n }\n\n /// Returns the dot product of two points.\n pub fn dot(&self, other: Point) -> f32 {\n self.x * other.x + self.y * other.y\n }\n\n /// Returns the cross product of vector and vec.\n ///\n /// Vector and vec form three-dimensional vectors with z-axis value equal to zero.\n /// The cross product is a three-dimensional vector with x-axis and y-axis values\n /// equal to zero. The cross product z-axis component is returned.\n pub fn cross(&self, other: Point) -> f32 {\n self.x * other.y - self.y * other.x\n }\n\n pub(crate) fn distance_to_sqd(&self, pt: Point) -> f32 {\n let dx = self.x - pt.x;\n let dy = self.y - pt.y;\n dx * dx + dy * dy\n }\n\n pub(crate) fn length_sqd(&self) -> f32 {\n self.dot(*self)\n }\n\n /// Scales Point in-place by scale.\n pub fn scale(&mut self, scale: f32) {\n self.x *= scale;\n self.y *= scale;\n }\n\n pub(crate) fn scaled(&self, scale: f32) -> Self {\n Point::from_xy(self.x * scale, self.y * scale)\n }\n\n pub(crate) fn swap_coords(&mut self) {\n core::mem::swap(&mut self.x, &mut self.y);\n }\n\n pub(crate) fn rotate_cw(&mut self) {\n self.swap_coords();\n self.x = -self.x;\n }\n\n pub(crate) fn rotate_ccw(&mut self) {\n self.swap_coords();\n self.y = -self.y;\n }\n}\n\n// We have to worry about 2 tricky conditions:\n// 1. underflow of mag2 (compared against nearlyzero^2)\n// 2. overflow of mag2 (compared w/ isfinite)\n//\n// If we underflow, we return false. If we overflow, we compute again using\n// doubles, which is much slower (3x in a desktop test) but will not overflow.\nfn set_point_length(\n pt: &mut Point,\n mut x: f32,\n mut y: f32,\n length: f32,\n orig_length: &mut Option,\n) -> bool {\n // our mag2 step overflowed to infinity, so use doubles instead.\n // much slower, but needed when x or y are very large, other wise we\n // divide by inf. and return (0,0) vector.\n let xx = x as f64;\n let yy = y as f64;\n let dmag = (xx * xx + yy * yy).sqrt();\n let dscale = length as f64 / dmag;\n x *= dscale as f32;\n y *= dscale as f32;\n\n // check if we're not finite, or we're zero-length\n if !x.is_finite() || !y.is_finite() || (x == 0.0 && y == 0.0) {\n *pt = Point::zero();\n return false;\n }\n\n let mut mag = 0.0;\n if orig_length.is_some() {\n mag = dmag as f32;\n }\n\n *pt = Point::from_xy(x, y);\n\n if orig_length.is_some() {\n *orig_length = Some(mag);\n }\n\n true\n}\n\nimpl core::ops::Neg for Point {\n type Output = Point;\n\n fn neg(self) -> Self::Output {\n Point {\n x: -self.x,\n y: -self.y,\n }\n }\n}\n\nimpl core::ops::Add for Point {\n type Output = Point;\n\n fn add(self, other: Point) -> Self::Output {\n Point::from_xy(self.x + other.x, self.y + other.y)\n }\n}\n\nimpl core::ops::AddAssign for Point {\n fn add_assign(&mut self, other: Point) {\n self.x += other.x;\n self.y += other.y;\n }\n}\n\nimpl core::ops::Sub for Point {\n type Output = Point;\n\n fn sub(self, other: Point) -> Self::Output {\n Point::from_xy(self.x - other.x, self.y - other.y)\n }\n}\n\nimpl core::ops::SubAssign for Point {\n fn sub_assign(&mut self, other: Point) {\n self.x -= other.x;\n self.y -= other.y;\n }\n}\n\nimpl core::ops::Mul for Point {\n type Output = Point;\n\n fn mul(self, other: Point) -> Self::Output {\n Point::from_xy(self.x * other.x, self.y * other.y)\n }\n}\n\nimpl core::ops::MulAssign for Point {\n fn mul_assign(&mut self, other: Point) {\n self.x *= other.x;\n self.y *= other.y;\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/path.rs", "sha256": "41d0090cfd4a5d9240801f946304f575dee70809e9f97938acb654e8fd435c1c", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse alloc::vec::Vec;\n\nuse crate::path_builder::PathBuilder;\nuse crate::transform::Transform;\nuse crate::{Point, Rect};\n\n/// A path verb.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug)]\npub enum PathVerb {\n Move,\n Line,\n Quad,\n Cubic,\n Close,\n}\n\n/// A Bezier path.\n///\n/// Can be created via [`PathBuilder`].\n/// Where [`PathBuilder`] can be created from the [`Path`] using [`clear`] to reuse the allocation.\n///\n/// Path is immutable and uses compact storage, where segment types and numbers are stored\n/// separately. Use can access path segments via [`Path::verbs`] and [`Path::points`],\n/// or via [`Path::segments`]\n///\n/// # Guarantees\n///\n/// - Has a valid, precomputed bounds.\n/// - All points are finite.\n/// - Has at least two segments.\n/// - Each contour starts with a MoveTo.\n/// - No duplicated Move.\n/// - No duplicated Close.\n/// - Zero-length contours are allowed.\n///\n/// [`PathBuilder`]: struct.PathBuilder.html\n/// [`clear`]: struct.Path.html#method.clear\n#[derive(Clone, PartialEq)]\npub struct Path {\n pub(crate) verbs: Vec,\n pub(crate) points: Vec,\n pub(crate) bounds: Rect,\n}\n\nimpl Path {\n /// Returns the number of segments in the path.\n pub fn len(&self) -> usize {\n self.verbs.len()\n }\n\n /// Returns the bounds of the path's points.\n ///\n /// The value is already calculated.\n pub fn bounds(&self) -> Rect {\n self.bounds\n }\n\n /// Returns an internal vector of verbs.\n pub fn verbs(&self) -> &[PathVerb] {\n &self.verbs\n }\n\n /// Returns an internal vector of points.\n pub fn points(&self) -> &[Point] {\n &self.points\n }\n\n /// Returns a transformed in-place path.\n ///\n /// Some points may become NaN/inf therefore this method can fail.\n pub fn transform(mut self, ts: Transform) -> Option {\n if ts.is_identity() {\n return Some(self);\n }\n\n ts.map_points(&mut self.points);\n\n // Update bounds.\n self.bounds = Rect::from_points(&self.points)?;\n\n Some(self)\n }\n\n /// Returns an iterator over path's segments.\n pub fn segments(&self) -> PathSegmentsIter {\n PathSegmentsIter {\n path: self,\n verb_index: 0,\n points_index: 0,\n is_auto_close: false,\n last_move_to: Point::zero(),\n last_point: Point::zero(),\n }\n }\n\n /// Clears the path and returns a `PathBuilder` that will reuse an allocated memory.\n pub fn clear(mut self) -> PathBuilder {\n self.verbs.clear();\n self.points.clear();\n\n PathBuilder {\n verbs: self.verbs,\n points: self.points,\n last_move_to_index: 0,\n move_to_required: true,\n }\n }\n}\n\nimpl core::fmt::Debug for Path {\n fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n use core::fmt::Write;\n\n let mut s = alloc::string::String::new();\n for segment in self.segments() {\n match segment {\n PathSegment::MoveTo(p) => s.write_fmt(format_args!(\"M {} {} \", p.x, p.y))?,\n PathSegment::LineTo(p) => s.write_fmt(format_args!(\"L {} {} \", p.x, p.y))?,\n PathSegment::QuadTo(p0, p1) => {\n s.write_fmt(format_args!(\"Q {} {} {} {} \", p0.x, p0.y, p1.x, p1.y))?\n }\n PathSegment::CubicTo(p0, p1, p2) => s.write_fmt(format_args!(\n \"C {} {} {} {} {} {} \",\n p0.x, p0.y, p1.x, p1.y, p2.x, p2.y\n ))?,\n PathSegment::Close => s.write_fmt(format_args!(\"Z \"))?,\n }\n }\n\n s.pop(); // ' '\n\n f.debug_struct(\"Path\")\n .field(\"segments\", &s)\n .field(\"bounds\", &self.bounds)\n .finish()\n }\n}\n\n/// A path segment.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, PartialEq, Debug)]\npub enum PathSegment {\n MoveTo(Point),\n LineTo(Point),\n QuadTo(Point, Point),\n CubicTo(Point, Point, Point),\n Close,\n}\n\n/// A path segments iterator.\n#[allow(missing_debug_implementations)]\n#[derive(Clone)]\npub struct PathSegmentsIter<'a> {\n path: &'a Path,\n verb_index: usize,\n points_index: usize,\n\n is_auto_close: bool,\n last_move_to: Point,\n last_point: Point,\n}\n\nimpl<'a> PathSegmentsIter<'a> {\n /// Sets the auto closing mode. Off by default.\n ///\n /// When enabled, emits an additional `PathSegment::Line` from the current position\n /// to the previous `PathSegment::Move`. And only then emits `PathSegment::Close`.\n pub fn set_auto_close(&mut self, flag: bool) {\n self.is_auto_close = flag;\n }\n\n pub(crate) fn auto_close(&mut self) -> PathSegment {\n if self.is_auto_close && self.last_point != self.last_move_to {\n self.verb_index -= 1;\n PathSegment::LineTo(self.last_move_to)\n } else {\n PathSegment::Close\n }\n }\n\n pub(crate) fn has_valid_tangent(&self) -> bool {\n let mut iter = self.clone();\n while let Some(segment) = iter.next() {\n match segment {\n PathSegment::MoveTo(_) => {\n return false;\n }\n PathSegment::LineTo(p) => {\n if iter.last_point == p {\n continue;\n }\n\n return true;\n }\n PathSegment::QuadTo(p1, p2) => {\n if iter.last_point == p1 && iter.last_point == p2 {\n continue;\n }\n\n return true;\n }\n PathSegment::CubicTo(p1, p2, p3) => {\n if iter.last_point == p1 && iter.last_point == p2 && iter.last_point == p3 {\n continue;\n }\n\n return true;\n }\n PathSegment::Close => {\n return false;\n }\n }\n }\n\n false\n }\n\n /// Returns the current verb.\n pub fn curr_verb(&self) -> PathVerb {\n self.path.verbs[self.verb_index - 1]\n }\n\n /// Returns the next verb.\n pub fn next_verb(&self) -> Option {\n self.path.verbs.get(self.verb_index).cloned()\n }\n}\n\nimpl<'a> Iterator for PathSegmentsIter<'a> {\n type Item = PathSegment;\n\n fn next(&mut self) -> Option {\n if self.verb_index < self.path.verbs.len() {\n let verb = self.path.verbs[self.verb_index];\n self.verb_index += 1;\n\n match verb {\n PathVerb::Move => {\n self.points_index += 1;\n self.last_move_to = self.path.points[self.points_index - 1];\n self.last_point = self.last_move_to;\n Some(PathSegment::MoveTo(self.last_move_to))\n }\n PathVerb::Line => {\n self.points_index += 1;\n self.last_point = self.path.points[self.points_index - 1];\n Some(PathSegment::LineTo(self.last_point))\n }\n PathVerb::Quad => {\n self.points_index += 2;\n self.last_point = self.path.points[self.points_index - 1];\n Some(PathSegment::QuadTo(\n self.path.points[self.points_index - 2],\n self.last_point,\n ))\n }\n PathVerb::Cubic => {\n self.points_index += 3;\n self.last_point = self.path.points[self.points_index - 1];\n Some(PathSegment::CubicTo(\n self.path.points[self.points_index - 3],\n self.path.points[self.points_index - 2],\n self.last_point,\n ))\n }\n PathVerb::Close => {\n let seg = self.auto_close();\n self.last_point = self.last_move_to;\n Some(seg)\n }\n }\n } else {\n None\n }\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/path_builder.rs", "sha256": "28c18ba8da6cdb85a08bc423175248c9a4d8d6736a2a1e4e389dce5fab2435c5", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// NOTE: this is not SkPathBuilder, but rather a reimplementation of SkPath.\n\nuse alloc::vec;\nuse alloc::vec::Vec;\n\nuse crate::{Path, Point, Rect};\n\nuse crate::path::PathVerb;\nuse crate::path_geometry;\nuse crate::scalar::{Scalar, SCALAR_ROOT_2_OVER_2};\n\n#[derive(Copy, Clone, PartialEq, Debug)]\npub(crate) enum PathDirection {\n /// Clockwise direction for adding closed contours.\n CW,\n /// Counter-clockwise direction for adding closed contours.\n CCW,\n}\n\n/// A path builder.\n#[derive(Clone, Default, Debug)]\npub struct PathBuilder {\n pub(crate) verbs: Vec,\n pub(crate) points: Vec,\n pub(crate) last_move_to_index: usize,\n pub(crate) move_to_required: bool,\n}\n\nimpl PathBuilder {\n /// Creates a new builder.\n pub fn new() -> Self {\n PathBuilder {\n verbs: Vec::new(),\n points: Vec::new(),\n last_move_to_index: 0,\n move_to_required: true,\n }\n }\n\n /// Creates a new builder with a specified capacity.\n ///\n /// Number of points depends on a verb type:\n ///\n /// - Move - 1\n /// - Line - 1\n /// - Quad - 2\n /// - Cubic - 3\n /// - Close - 0\n pub fn with_capacity(verbs_capacity: usize, points_capacity: usize) -> Self {\n PathBuilder {\n verbs: Vec::with_capacity(verbs_capacity),\n points: Vec::with_capacity(points_capacity),\n last_move_to_index: 0,\n move_to_required: true,\n }\n }\n\n /// Creates a new `Path` from `Rect`.\n ///\n /// Never fails since `Rect` is always valid.\n ///\n /// Segments are created clockwise: TopLeft -> TopRight -> BottomRight -> BottomLeft\n ///\n /// The contour is closed.\n pub fn from_rect(rect: Rect) -> Path {\n let verbs = vec![\n PathVerb::Move,\n PathVerb::Line,\n PathVerb::Line,\n PathVerb::Line,\n PathVerb::Close,\n ];\n\n let points = vec![\n Point::from_xy(rect.left(), rect.top()),\n Point::from_xy(rect.right(), rect.top()),\n Point::from_xy(rect.right(), rect.bottom()),\n Point::from_xy(rect.left(), rect.bottom()),\n ];\n\n Path {\n bounds: rect,\n verbs,\n points,\n }\n }\n\n /// Creates a new `Path` from a circle.\n ///\n /// See [`PathBuilder::push_circle`] for details.\n pub fn from_circle(cx: f32, cy: f32, radius: f32) -> Option {\n let mut b = PathBuilder::new();\n b.push_circle(cx, cy, radius);\n b.finish()\n }\n\n /// Creates a new `Path` from an oval.\n ///\n /// See [`PathBuilder::push_oval`] for details.\n pub fn from_oval(oval: Rect) -> Option {\n let mut b = PathBuilder::new();\n b.push_oval(oval);\n b.finish()\n }\n\n pub(crate) fn reserve(&mut self, additional_verbs: usize, additional_points: usize) {\n self.verbs.reserve(additional_verbs);\n self.points.reserve(additional_points);\n }\n\n /// Returns the current number of segments in the builder.\n pub fn len(&self) -> usize {\n self.verbs.len()\n }\n\n /// Checks if the builder has any segments added.\n pub fn is_empty(&self) -> bool {\n self.verbs.is_empty()\n }\n\n /// Adds beginning of a contour.\n ///\n /// Multiple continuous MoveTo segments are not allowed.\n /// If the previous segment was also MoveTo, it will be overwritten with the current one.\n pub fn move_to(&mut self, x: f32, y: f32) {\n if let Some(PathVerb::Move) = self.verbs.last() {\n let last_idx = self.points.len() - 1;\n self.points[last_idx] = Point::from_xy(x, y);\n } else {\n self.last_move_to_index = self.points.len();\n self.move_to_required = false;\n\n self.verbs.push(PathVerb::Move);\n self.points.push(Point::from_xy(x, y));\n }\n }\n\n fn inject_move_to_if_needed(&mut self) {\n if self.move_to_required {\n match self.points.get(self.last_move_to_index).cloned() {\n Some(p) => self.move_to(p.x, p.y),\n None => self.move_to(0.0, 0.0),\n }\n }\n }\n\n /// Adds a line from the last point.\n ///\n /// - If `Path` is empty - adds Move(0, 0) first.\n /// - If `Path` ends with Close - adds Move(last_x, last_y) first.\n pub fn line_to(&mut self, x: f32, y: f32) {\n self.inject_move_to_if_needed();\n\n self.verbs.push(PathVerb::Line);\n self.points.push(Point::from_xy(x, y));\n }\n\n /// Adds a quad curve from the last point to `x`, `y`.\n ///\n /// - If `Path` is empty - adds Move(0, 0) first.\n /// - If `Path` ends with Close - adds Move(last_x, last_y) first.\n pub fn quad_to(&mut self, x1: f32, y1: f32, x: f32, y: f32) {\n self.inject_move_to_if_needed();\n\n self.verbs.push(PathVerb::Quad);\n self.points.push(Point::from_xy(x1, y1));\n self.points.push(Point::from_xy(x, y));\n }\n\n pub(crate) fn quad_to_pt(&mut self, p1: Point, p: Point) {\n self.quad_to(p1.x, p1.y, p.x, p.y);\n }\n\n // We do not support conic segments, but Skia still relies on them from time to time.\n // This method will simply convert the input data into quad segments.\n pub(crate) fn conic_to(&mut self, x1: f32, y1: f32, x: f32, y: f32, weight: f32) {\n // check for <= 0 or NaN with this test\n if !(weight > 0.0) {\n self.line_to(x, y);\n } else if !weight.is_finite() {\n self.line_to(x1, y1);\n self.line_to(x, y);\n } else if weight == 1.0 {\n self.quad_to(x1, y1, x, y);\n } else {\n self.inject_move_to_if_needed();\n\n let last = self.last_point().unwrap();\n let quadder = path_geometry::AutoConicToQuads::compute(\n last,\n Point::from_xy(x1, y1),\n Point::from_xy(x, y),\n weight,\n );\n if let Some(quadder) = quadder {\n // Points are ordered as: 0 - 1 2 - 3 4 - 5 6 - ..\n // `count` is a number of pairs +1\n let mut offset = 1;\n for _ in 0..quadder.len {\n let pt1 = quadder.points[offset + 0];\n let pt2 = quadder.points[offset + 1];\n self.quad_to(pt1.x, pt1.y, pt2.x, pt2.y);\n offset += 2;\n }\n }\n }\n }\n\n pub(crate) fn conic_points_to(&mut self, pt1: Point, pt2: Point, weight: f32) {\n self.conic_to(pt1.x, pt1.y, pt2.x, pt2.y, weight);\n }\n\n /// Adds a cubic curve from the last point to `x`, `y`.\n ///\n /// - If `Path` is empty - adds Move(0, 0) first.\n /// - If `Path` ends with Close - adds Move(last_x, last_y) first.\n pub fn cubic_to(&mut self, x1: f32, y1: f32, x2: f32, y2: f32, x: f32, y: f32) {\n self.inject_move_to_if_needed();\n\n self.verbs.push(PathVerb::Cubic);\n self.points.push(Point::from_xy(x1, y1));\n self.points.push(Point::from_xy(x2, y2));\n self.points.push(Point::from_xy(x, y));\n }\n\n pub(crate) fn cubic_to_pt(&mut self, p1: Point, p2: Point, p: Point) {\n self.cubic_to(p1.x, p1.y, p2.x, p2.y, p.x, p.y);\n }\n\n /// Closes the current contour.\n ///\n /// A closed contour connects the first and the last Point\n /// with a line, forming a continuous loop.\n ///\n /// Does nothing when `Path` is empty or already closed.\n ///\n /// Open and closed contour will be filled the same way.\n /// Stroking an open contour will add LineCap at contour's start and end.\n /// Stroking an closed contour will add LineJoin at contour's start and end.\n pub fn close(&mut self) {\n // don't add a close if it's the first verb or a repeat\n if !self.verbs.is_empty() {\n if self.verbs.last().cloned() != Some(PathVerb::Close) {\n self.verbs.push(PathVerb::Close);\n }\n }\n\n self.move_to_required = true;\n }\n\n /// Returns the last point if any.\n pub fn last_point(&self) -> Option {\n self.points.last().cloned()\n }\n\n pub(crate) fn set_last_point(&mut self, pt: Point) {\n match self.points.last_mut() {\n Some(last) => *last = pt,\n None => self.move_to(pt.x, pt.y),\n }\n }\n\n pub(crate) fn is_zero_length_since_point(&self, start_pt_index: usize) -> bool {\n let count = self.points.len() - start_pt_index;\n if count < 2 {\n return true;\n }\n\n let first = self.points[start_pt_index];\n for i in 1..count {\n if first != self.points[start_pt_index + i] {\n return false;\n }\n }\n\n true\n }\n\n /// Adds a rectangle contour.\n ///\n /// The contour is closed and has a clock-wise direction.\n pub fn push_rect(&mut self, rect: Rect) {\n self.move_to(rect.left(), rect.top());\n self.line_to(rect.right(), rect.top());\n self.line_to(rect.right(), rect.bottom());\n self.line_to(rect.left(), rect.bottom());\n self.close();\n }\n\n /// Adds an oval contour bounded by the provided rectangle.\n ///\n /// The contour is closed and has a clock-wise direction.\n pub fn push_oval(&mut self, oval: Rect) {\n let cx = oval.left().half() + oval.right().half();\n let cy = oval.top().half() + oval.bottom().half();\n\n let oval_points = [\n Point::from_xy(cx, oval.bottom()),\n Point::from_xy(oval.left(), cy),\n Point::from_xy(cx, oval.top()),\n Point::from_xy(oval.right(), cy),\n ];\n\n let rect_points = [\n Point::from_xy(oval.right(), oval.bottom()),\n Point::from_xy(oval.left(), oval.bottom()),\n Point::from_xy(oval.left(), oval.top()),\n Point::from_xy(oval.right(), oval.top()),\n ];\n\n let weight = SCALAR_ROOT_2_OVER_2;\n self.move_to(oval_points[3].x, oval_points[3].y);\n for (p1, p2) in rect_points.iter().zip(oval_points.iter()) {\n self.conic_points_to(*p1, *p2, weight);\n }\n self.close();\n }\n\n /// Adds a circle contour.\n ///\n /// The contour is closed and has a clock-wise direction.\n ///\n /// Does nothing when:\n /// - `radius` <= 0\n /// - any value is not finite or really large\n pub fn push_circle(&mut self, x: f32, y: f32, r: f32) {\n if let Some(r) = Rect::from_xywh(x - r, y - r, r + r, r + r) {\n self.push_oval(r);\n }\n }\n\n /// Adds a path.\n pub fn push_path(&mut self, other: &Path) {\n self.last_move_to_index = self.points.len();\n\n self.verbs.extend_from_slice(&other.verbs);\n self.points.extend_from_slice(&other.points);\n }\n\n pub(crate) fn push_path_builder(&mut self, other: &PathBuilder) {\n if other.is_empty() {\n return;\n }\n\n if self.last_move_to_index != 0 {\n self.last_move_to_index = self.points.len() + other.last_move_to_index;\n }\n\n self.verbs.extend_from_slice(&other.verbs);\n self.points.extend_from_slice(&other.points);\n }\n\n /// Appends, in a reverse order, the first contour of path ignoring path's last point.\n pub(crate) fn reverse_path_to(&mut self, other: &PathBuilder) {\n if other.is_empty() {\n return;\n }\n\n debug_assert_eq!(other.verbs[0], PathVerb::Move);\n\n let mut points_offset = other.points.len() - 1;\n for verb in other.verbs.iter().rev() {\n match verb {\n PathVerb::Move => {\n // if the path has multiple contours, stop after reversing the last\n break;\n }\n PathVerb::Line => {\n // We're moving one point back manually, to prevent points_offset overflow.\n let pt = other.points[points_offset - 1];\n points_offset -= 1;\n self.line_to(pt.x, pt.y);\n }\n PathVerb::Quad => {\n let pt1 = other.points[points_offset - 1];\n let pt2 = other.points[points_offset - 2];\n points_offset -= 2;\n self.quad_to(pt1.x, pt1.y, pt2.x, pt2.y);\n }\n PathVerb::Cubic => {\n let pt1 = other.points[points_offset - 1];\n let pt2 = other.points[points_offset - 2];\n let pt3 = other.points[points_offset - 3];\n points_offset -= 3;\n self.cubic_to(pt1.x, pt1.y, pt2.x, pt2.y, pt3.x, pt3.y);\n }\n PathVerb::Close => {}\n }\n }\n }\n\n /// Reset the builder.\n ///\n /// Memory is not deallocated.\n pub fn clear(&mut self) {\n self.verbs.clear();\n self.points.clear();\n self.last_move_to_index = 0;\n self.move_to_required = true;\n }\n\n /// Finishes the builder and returns a `Path`.\n ///\n /// Returns `None` when `Path` is empty or has invalid bounds.\n pub fn finish(self) -> Option {\n if self.is_empty() {\n return None;\n }\n\n // Just a move to? Bail.\n if self.verbs.len() == 1 {\n return None;\n }\n\n let bounds = Rect::from_points(&self.points)?;\n\n Some(Path {\n bounds,\n verbs: self.verbs,\n points: self.points,\n })\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/path_geometry.rs", "sha256": "3203df234be68484f7b4fd2e0ace6a4a868b234010e146b9b2a9305188ee19cb", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n//! A collection of functions to work with Bezier paths.\n//!\n//! Mainly for internal use. Do not rely on it!\n\n#![allow(missing_docs)]\n\nuse crate::{Point, Transform};\n\nuse crate::f32x2_t::f32x2;\nuse crate::floating_point::FLOAT_PI;\nuse crate::scalar::{Scalar, SCALAR_NEARLY_ZERO, SCALAR_ROOT_2_OVER_2};\n\nuse crate::floating_point::{NormalizedF32, NormalizedF32Exclusive};\nuse crate::path_builder::PathDirection;\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse crate::NoStdFloat;\n\n// use for : eval(t) == A * t^2 + B * t + C\n#[derive(Clone, Copy, Default, Debug)]\npub struct QuadCoeff {\n pub a: f32x2,\n pub b: f32x2,\n pub c: f32x2,\n}\n\nimpl QuadCoeff {\n pub fn from_points(points: &[Point; 3]) -> Self {\n let c = points[0].to_f32x2();\n let p1 = points[1].to_f32x2();\n let p2 = points[2].to_f32x2();\n let b = times_2(p1 - c);\n let a = p2 - times_2(p1) + c;\n\n QuadCoeff { a, b, c }\n }\n\n pub fn eval(&self, t: f32x2) -> f32x2 {\n (self.a * t + self.b) * t + self.c\n }\n}\n\n#[derive(Clone, Copy, Default, Debug)]\npub struct CubicCoeff {\n pub a: f32x2,\n pub b: f32x2,\n pub c: f32x2,\n pub d: f32x2,\n}\n\nimpl CubicCoeff {\n pub fn from_points(points: &[Point; 4]) -> Self {\n let p0 = points[0].to_f32x2();\n let p1 = points[1].to_f32x2();\n let p2 = points[2].to_f32x2();\n let p3 = points[3].to_f32x2();\n let three = f32x2::splat(3.0);\n\n CubicCoeff {\n a: p3 + three * (p1 - p2) - p0,\n b: three * (p2 - times_2(p1) + p0),\n c: three * (p1 - p0),\n d: p0,\n }\n }\n\n pub fn eval(&self, t: f32x2) -> f32x2 {\n ((self.a * t + self.b) * t + self.c) * t + self.d\n }\n}\n\n// TODO: to a custom type?\npub fn new_t_values() -> [NormalizedF32Exclusive; 3] {\n [\n NormalizedF32Exclusive::ANY,\n NormalizedF32Exclusive::ANY,\n NormalizedF32Exclusive::ANY,\n ]\n}\n\npub fn chop_quad_at(src: &[Point], t: NormalizedF32Exclusive, dst: &mut [Point; 5]) {\n let p0 = src[0].to_f32x2();\n let p1 = src[1].to_f32x2();\n let p2 = src[2].to_f32x2();\n let tt = f32x2::splat(t.get());\n\n let p01 = interp(p0, p1, tt);\n let p12 = interp(p1, p2, tt);\n\n dst[0] = Point::from_f32x2(p0);\n dst[1] = Point::from_f32x2(p01);\n dst[2] = Point::from_f32x2(interp(p01, p12, tt));\n dst[3] = Point::from_f32x2(p12);\n dst[4] = Point::from_f32x2(p2);\n}\n\n// From Numerical Recipes in C.\n//\n// Q = -1/2 (B + sign(B) sqrt[B*B - 4*A*C])\n// x1 = Q / A\n// x2 = C / Q\npub fn find_unit_quad_roots(\n a: f32,\n b: f32,\n c: f32,\n roots: &mut [NormalizedF32Exclusive; 3],\n) -> usize {\n if a == 0.0 {\n if let Some(r) = valid_unit_divide(-c, b) {\n roots[0] = r;\n return 1;\n } else {\n return 0;\n }\n }\n\n // use doubles so we don't overflow temporarily trying to compute R\n let mut dr = f64::from(b) * f64::from(b) - 4.0 * f64::from(a) * f64::from(c);\n if dr < 0.0 {\n return 0;\n }\n dr = dr.sqrt();\n let r = dr as f32;\n if !r.is_finite() {\n return 0;\n }\n\n let q = if b < 0.0 {\n -(b - r) / 2.0\n } else {\n -(b + r) / 2.0\n };\n\n let mut roots_offset = 0;\n if let Some(r) = valid_unit_divide(q, a) {\n roots[roots_offset] = r;\n roots_offset += 1;\n }\n\n if let Some(r) = valid_unit_divide(c, q) {\n roots[roots_offset] = r;\n roots_offset += 1;\n }\n\n if roots_offset == 2 {\n if roots[0].get() > roots[1].get() {\n roots.swap(0, 1);\n } else if roots[0] == roots[1] {\n // nearly-equal?\n roots_offset -= 1; // skip the double root\n }\n }\n\n roots_offset\n}\n\npub fn chop_cubic_at2(src: &[Point; 4], t: NormalizedF32Exclusive, dst: &mut [Point]) {\n let p0 = src[0].to_f32x2();\n let p1 = src[1].to_f32x2();\n let p2 = src[2].to_f32x2();\n let p3 = src[3].to_f32x2();\n let tt = f32x2::splat(t.get());\n\n let ab = interp(p0, p1, tt);\n let bc = interp(p1, p2, tt);\n let cd = interp(p2, p3, tt);\n let abc = interp(ab, bc, tt);\n let bcd = interp(bc, cd, tt);\n let abcd = interp(abc, bcd, tt);\n\n dst[0] = Point::from_f32x2(p0);\n dst[1] = Point::from_f32x2(ab);\n dst[2] = Point::from_f32x2(abc);\n dst[3] = Point::from_f32x2(abcd);\n dst[4] = Point::from_f32x2(bcd);\n dst[5] = Point::from_f32x2(cd);\n dst[6] = Point::from_f32x2(p3);\n}\n\npub fn valid_unit_divide(mut numer: f32, mut denom: f32) -> Option {\n if numer < 0.0 {\n numer = -numer;\n denom = -denom;\n }\n\n if denom == 0.0 || numer == 0.0 || numer >= denom {\n return None;\n }\n\n let r = numer / denom;\n NormalizedF32Exclusive::new(r)\n}\n\nfn interp(v0: f32x2, v1: f32x2, t: f32x2) -> f32x2 {\n v0 + (v1 - v0) * t\n}\n\nfn times_2(value: f32x2) -> f32x2 {\n value + value\n}\n\n// F(t) = a (1 - t) ^ 2 + 2 b t (1 - t) + c t ^ 2\n// F'(t) = 2 (b - a) + 2 (a - 2b + c) t\n// F''(t) = 2 (a - 2b + c)\n//\n// A = 2 (b - a)\n// B = 2 (a - 2b + c)\n//\n// Maximum curvature for a quadratic means solving\n// Fx' Fx'' + Fy' Fy'' = 0\n//\n// t = - (Ax Bx + Ay By) / (Bx ^ 2 + By ^ 2)\npub(crate) fn find_quad_max_curvature(src: &[Point; 3]) -> NormalizedF32 {\n let ax = src[1].x - src[0].x;\n let ay = src[1].y - src[0].y;\n let bx = src[0].x - src[1].x - src[1].x + src[2].x;\n let by = src[0].y - src[1].y - src[1].y + src[2].y;\n\n let mut numer = -(ax * bx + ay * by);\n let mut denom = bx * bx + by * by;\n if denom < 0.0 {\n numer = -numer;\n denom = -denom;\n }\n\n if numer <= 0.0 {\n return NormalizedF32::ZERO;\n }\n\n if numer >= denom {\n // Also catches denom=0\n return NormalizedF32::ONE;\n }\n\n let t = numer / denom;\n NormalizedF32::new(t).unwrap()\n}\n\npub(crate) fn eval_quad_at(src: &[Point; 3], t: NormalizedF32) -> Point {\n Point::from_f32x2(QuadCoeff::from_points(src).eval(f32x2::splat(t.get())))\n}\n\npub(crate) fn eval_quad_tangent_at(src: &[Point; 3], tol: NormalizedF32) -> Point {\n // The derivative equation is 2(b - a +(a - 2b +c)t). This returns a\n // zero tangent vector when t is 0 or 1, and the control point is equal\n // to the end point. In this case, use the quad end points to compute the tangent.\n if (tol == NormalizedF32::ZERO && src[0] == src[1])\n || (tol == NormalizedF32::ONE && src[1] == src[2])\n {\n return src[2] - src[0];\n }\n\n let p0 = src[0].to_f32x2();\n let p1 = src[1].to_f32x2();\n let p2 = src[2].to_f32x2();\n\n let b = p1 - p0;\n let a = p2 - p1 - b;\n let t = a * f32x2::splat(tol.get()) + b;\n\n Point::from_f32x2(t + t)\n}\n\n// Looking for F' dot F'' == 0\n//\n// A = b - a\n// B = c - 2b + a\n// C = d - 3c + 3b - a\n//\n// F' = 3Ct^2 + 6Bt + 3A\n// F'' = 6Ct + 6B\n//\n// F' dot F'' -> CCt^3 + 3BCt^2 + (2BB + CA)t + AB\npub fn find_cubic_max_curvature<'a>(\n src: &[Point; 4],\n t_values: &'a mut [NormalizedF32; 3],\n) -> &'a [NormalizedF32] {\n let mut coeff_x = formulate_f1_dot_f2(&[src[0].x, src[1].x, src[2].x, src[3].x]);\n let coeff_y = formulate_f1_dot_f2(&[src[0].y, src[1].y, src[2].y, src[3].y]);\n\n for i in 0..4 {\n coeff_x[i] += coeff_y[i];\n }\n\n let len = solve_cubic_poly(&coeff_x, t_values);\n &t_values[0..len]\n}\n\n// Looking for F' dot F'' == 0\n//\n// A = b - a\n// B = c - 2b + a\n// C = d - 3c + 3b - a\n//\n// F' = 3Ct^2 + 6Bt + 3A\n// F'' = 6Ct + 6B\n//\n// F' dot F'' -> CCt^3 + 3BCt^2 + (2BB + CA)t + AB\nfn formulate_f1_dot_f2(src: &[f32; 4]) -> [f32; 4] {\n let a = src[1] - src[0];\n let b = src[2] - 2.0 * src[1] + src[0];\n let c = src[3] + 3.0 * (src[1] - src[2]) - src[0];\n\n [c * c, 3.0 * b * c, 2.0 * b * b + c * a, a * b]\n}\n\n/// Solve coeff(t) == 0, returning the number of roots that lie withing 0 < t < 1.\n/// coeff[0]t^3 + coeff[1]t^2 + coeff[2]t + coeff[3]\n///\n/// Eliminates repeated roots (so that all t_values are distinct, and are always\n/// in increasing order.\nfn solve_cubic_poly(coeff: &[f32; 4], t_values: &mut [NormalizedF32; 3]) -> usize {\n if coeff[0].is_nearly_zero() {\n // we're just a quadratic\n let mut tmp_t = new_t_values();\n let count = find_unit_quad_roots(coeff[1], coeff[2], coeff[3], &mut tmp_t);\n for i in 0..count {\n t_values[i] = tmp_t[i].to_normalized();\n }\n\n return count;\n }\n\n debug_assert!(coeff[0] != 0.0);\n\n let inva = coeff[0].invert();\n let a = coeff[1] * inva;\n let b = coeff[2] * inva;\n let c = coeff[3] * inva;\n\n let q = (a * a - b * 3.0) / 9.0;\n let r = (2.0 * a * a * a - 9.0 * a * b + 27.0 * c) / 54.0;\n\n let q3 = q * q * q;\n let r2_minus_q3 = r * r - q3;\n let adiv3 = a / 3.0;\n\n if r2_minus_q3 < 0.0 {\n // we have 3 real roots\n // the divide/root can, due to finite precisions, be slightly outside of -1...1\n let theta = (r / q3.sqrt()).bound(-1.0, 1.0).acos();\n let neg2_root_q = -2.0 * q.sqrt();\n\n t_values[0] = NormalizedF32::new_clamped(neg2_root_q * (theta / 3.0).cos() - adiv3);\n t_values[1] = NormalizedF32::new_clamped(\n neg2_root_q * ((theta + 2.0 * FLOAT_PI) / 3.0).cos() - adiv3,\n );\n t_values[2] = NormalizedF32::new_clamped(\n neg2_root_q * ((theta - 2.0 * FLOAT_PI) / 3.0).cos() - adiv3,\n );\n\n // now sort the roots\n sort_array3(t_values);\n collapse_duplicates3(t_values)\n } else {\n // we have 1 real root\n let mut a = r.abs() + r2_minus_q3.sqrt();\n a = scalar_cube_root(a);\n if r > 0.0 {\n a = -a;\n }\n\n if a != 0.0 {\n a += q / a;\n }\n\n t_values[0] = NormalizedF32::new_clamped(a - adiv3);\n 1\n }\n}\n\nfn sort_array3(array: &mut [NormalizedF32; 3]) {\n if array[0] > array[1] {\n array.swap(0, 1);\n }\n\n if array[1] > array[2] {\n array.swap(1, 2);\n }\n\n if array[0] > array[1] {\n array.swap(0, 1);\n }\n}\n\nfn collapse_duplicates3(array: &mut [NormalizedF32; 3]) -> usize {\n let mut len = 3;\n\n if array[1] == array[2] {\n len = 2;\n }\n\n if array[0] == array[1] {\n len = 1;\n }\n\n len\n}\n\nfn scalar_cube_root(x: f32) -> f32 {\n x.powf(0.3333333)\n}\n\n// This is SkEvalCubicAt split into three functions.\npub(crate) fn eval_cubic_pos_at(src: &[Point; 4], t: NormalizedF32) -> Point {\n Point::from_f32x2(CubicCoeff::from_points(src).eval(f32x2::splat(t.get())))\n}\n\n// This is SkEvalCubicAt split into three functions.\npub(crate) fn eval_cubic_tangent_at(src: &[Point; 4], t: NormalizedF32) -> Point {\n // The derivative equation returns a zero tangent vector when t is 0 or 1, and the\n // adjacent control point is equal to the end point. In this case, use the\n // next control point or the end points to compute the tangent.\n if (t.get() == 0.0 && src[0] == src[1]) || (t.get() == 1.0 && src[2] == src[3]) {\n let mut tangent = if t.get() == 0.0 {\n src[2] - src[0]\n } else {\n src[3] - src[1]\n };\n\n if tangent.x == 0.0 && tangent.y == 0.0 {\n tangent = src[3] - src[0];\n }\n\n tangent\n } else {\n eval_cubic_derivative(src, t)\n }\n}\n\nfn eval_cubic_derivative(src: &[Point; 4], t: NormalizedF32) -> Point {\n let p0 = src[0].to_f32x2();\n let p1 = src[1].to_f32x2();\n let p2 = src[2].to_f32x2();\n let p3 = src[3].to_f32x2();\n\n let coeff = QuadCoeff {\n a: p3 + f32x2::splat(3.0) * (p1 - p2) - p0,\n b: times_2(p2 - times_2(p1) + p0),\n c: p1 - p0,\n };\n\n Point::from_f32x2(coeff.eval(f32x2::splat(t.get())))\n}\n\n// http://www.faculty.idc.ac.il/arik/quality/appendixA.html\n//\n// Inflection means that curvature is zero.\n// Curvature is [F' x F''] / [F'^3]\n// So we solve F'x X F''y - F'y X F''y == 0\n// After some canceling of the cubic term, we get\n// A = b - a\n// B = c - 2b + a\n// C = d - 3c + 3b - a\n// (BxCy - ByCx)t^2 + (AxCy - AyCx)t + AxBy - AyBx == 0\npub(crate) fn find_cubic_inflections<'a>(\n src: &[Point; 4],\n t_values: &'a mut [NormalizedF32Exclusive; 3],\n) -> &'a [NormalizedF32Exclusive] {\n let ax = src[1].x - src[0].x;\n let ay = src[1].y - src[0].y;\n let bx = src[2].x - 2.0 * src[1].x + src[0].x;\n let by = src[2].y - 2.0 * src[1].y + src[0].y;\n let cx = src[3].x + 3.0 * (src[1].x - src[2].x) - src[0].x;\n let cy = src[3].y + 3.0 * (src[1].y - src[2].y) - src[0].y;\n\n let len = find_unit_quad_roots(\n bx * cy - by * cx,\n ax * cy - ay * cx,\n ax * by - ay * bx,\n t_values,\n );\n\n &t_values[0..len]\n}\n\n// Return location (in t) of cubic cusp, if there is one.\n// Note that classify cubic code does not reliably return all cusp'd cubics, so\n// it is not called here.\npub(crate) fn find_cubic_cusp(src: &[Point; 4]) -> Option {\n // When the adjacent control point matches the end point, it behaves as if\n // the cubic has a cusp: there's a point of max curvature where the derivative\n // goes to zero. Ideally, this would be where t is zero or one, but math\n // error makes not so. It is not uncommon to create cubics this way; skip them.\n if src[0] == src[1] {\n return None;\n }\n\n if src[2] == src[3] {\n return None;\n }\n\n // Cubics only have a cusp if the line segments formed by the control and end points cross.\n // Detect crossing if line ends are on opposite sides of plane formed by the other line.\n if on_same_side(src, 0, 2) || on_same_side(src, 2, 0) {\n return None;\n }\n\n // Cubics may have multiple points of maximum curvature, although at most only\n // one is a cusp.\n let mut t_values = [NormalizedF32::ZERO; 3];\n let max_curvature = find_cubic_max_curvature(src, &mut t_values);\n for test_t in max_curvature {\n if 0.0 >= test_t.get() || test_t.get() >= 1.0 {\n // no need to consider max curvature on the end\n continue;\n }\n\n // A cusp is at the max curvature, and also has a derivative close to zero.\n // Choose the 'close to zero' meaning by comparing the derivative length\n // with the overall cubic size.\n let d_pt = eval_cubic_derivative(src, *test_t);\n let d_pt_magnitude = d_pt.length_sqd();\n let precision = calc_cubic_precision(src);\n if d_pt_magnitude < precision {\n // All three max curvature t values may be close to the cusp;\n // return the first one.\n return Some(NormalizedF32Exclusive::new_bounded(test_t.get()));\n }\n }\n\n None\n}\n\n// Returns true if both points src[testIndex], src[testIndex+1] are in the same half plane defined\n// by the line segment src[lineIndex], src[lineIndex+1].\nfn on_same_side(src: &[Point; 4], test_index: usize, line_index: usize) -> bool {\n let origin = src[line_index];\n let line = src[line_index + 1] - origin;\n let mut crosses = [0.0, 0.0];\n for index in 0..2 {\n let test_line = src[test_index + index] - origin;\n crosses[index] = line.cross(test_line);\n }\n\n crosses[0] * crosses[1] >= 0.0\n}\n\n// Returns a constant proportional to the dimensions of the cubic.\n// Constant found through experimentation -- maybe there's a better way....\nfn calc_cubic_precision(src: &[Point; 4]) -> f32 {\n (src[1].distance_to_sqd(src[0])\n + src[2].distance_to_sqd(src[1])\n + src[3].distance_to_sqd(src[2]))\n * 1e-8\n}\n\n#[derive(Copy, Clone, Default, Debug)]\npub(crate) struct Conic {\n pub points: [Point; 3],\n pub weight: f32,\n}\n\nimpl Conic {\n pub fn new(pt0: Point, pt1: Point, pt2: Point, weight: f32) -> Self {\n Conic {\n points: [pt0, pt1, pt2],\n weight,\n }\n }\n\n pub fn from_points(points: &[Point], weight: f32) -> Self {\n Conic {\n points: [points[0], points[1], points[2]],\n weight,\n }\n }\n\n fn compute_quad_pow2(&self, tolerance: f32) -> Option {\n if tolerance < 0.0 || !tolerance.is_finite() {\n return None;\n }\n\n if !self.points[0].is_finite() || !self.points[1].is_finite() || !self.points[2].is_finite()\n {\n return None;\n }\n\n // Limit the number of suggested quads to approximate a conic\n const MAX_CONIC_TO_QUAD_POW2: usize = 4;\n\n // \"High order approximation of conic sections by quadratic splines\"\n // by Michael Floater, 1993\n let a = self.weight - 1.0;\n let k = a / (4.0 * (2.0 + a));\n let x = k * (self.points[0].x - 2.0 * self.points[1].x + self.points[2].x);\n let y = k * (self.points[0].y - 2.0 * self.points[1].y + self.points[2].y);\n\n let mut error = (x * x + y * y).sqrt();\n let mut pow2 = 0;\n for _ in 0..MAX_CONIC_TO_QUAD_POW2 {\n if error <= tolerance {\n break;\n }\n\n error *= 0.25;\n pow2 += 1;\n }\n\n // Unlike Skia, we always expect `pow2` to be at least 1.\n // Otherwise it produces ugly results.\n Some(pow2.max(1))\n }\n\n // Chop this conic into N quads, stored continuously in pts[], where\n // N = 1 << pow2. The amount of storage needed is (1 + 2 * N)\n pub fn chop_into_quads_pow2(&self, pow2: u8, points: &mut [Point]) -> u8 {\n debug_assert!(pow2 < 5);\n\n points[0] = self.points[0];\n subdivide(self, &mut points[1..], pow2);\n\n let quad_count = 1 << pow2;\n let pt_count = 2 * quad_count + 1;\n if points.iter().take(pt_count).any(|n| !n.is_finite()) {\n // if we generated a non-finite, pin ourselves to the middle of the hull,\n // as our first and last are already on the first/last pts of the hull.\n for p in points.iter_mut().take(pt_count - 1).skip(1) {\n *p = self.points[1];\n }\n }\n\n 1 << pow2\n }\n\n fn chop(&self) -> (Conic, Conic) {\n let scale = f32x2::splat((1.0 + self.weight).invert());\n let new_w = subdivide_weight_value(self.weight);\n\n let p0 = self.points[0].to_f32x2();\n let p1 = self.points[1].to_f32x2();\n let p2 = self.points[2].to_f32x2();\n let ww = f32x2::splat(self.weight);\n\n let wp1 = ww * p1;\n let m = (p0 + times_2(wp1) + p2) * scale * f32x2::splat(0.5);\n let mut m_pt = Point::from_f32x2(m);\n if !m_pt.is_finite() {\n let w_d = self.weight as f64;\n let w_2 = w_d * 2.0;\n let scale_half = 1.0 / (1.0 + w_d) * 0.5;\n m_pt.x = ((self.points[0].x as f64\n + w_2 * self.points[1].x as f64\n + self.points[2].x as f64)\n * scale_half) as f32;\n\n m_pt.y = ((self.points[0].y as f64\n + w_2 * self.points[1].y as f64\n + self.points[2].y as f64)\n * scale_half) as f32;\n }\n\n (\n Conic {\n points: [self.points[0], Point::from_f32x2((p0 + wp1) * scale), m_pt],\n weight: new_w,\n },\n Conic {\n points: [m_pt, Point::from_f32x2((wp1 + p2) * scale), self.points[2]],\n weight: new_w,\n },\n )\n }\n\n pub fn build_unit_arc(\n u_start: Point,\n u_stop: Point,\n dir: PathDirection,\n user_transform: Transform,\n dst: &mut [Conic; 5],\n ) -> Option<&[Conic]> {\n // rotate by x,y so that u_start is (1.0)\n let x = u_start.dot(u_stop);\n let mut y = u_start.cross(u_stop);\n\n let abs_y = y.abs();\n\n // check for (effectively) coincident vectors\n // this can happen if our angle is nearly 0 or nearly 180 (y == 0)\n // ... we use the dot-prod to distinguish between 0 and 180 (x > 0)\n if abs_y <= SCALAR_NEARLY_ZERO\n && x > 0.0\n && ((y >= 0.0 && dir == PathDirection::CW) || (y <= 0.0 && dir == PathDirection::CCW))\n {\n return None;\n }\n\n if dir == PathDirection::CCW {\n y = -y;\n }\n\n // We decide to use 1-conic per quadrant of a circle. What quadrant does [xy] lie in?\n // 0 == [0 .. 90)\n // 1 == [90 ..180)\n // 2 == [180..270)\n // 3 == [270..360)\n //\n let mut quadrant = 0;\n if y == 0.0 {\n quadrant = 2; // 180\n debug_assert!((x + 1.0) <= SCALAR_NEARLY_ZERO);\n } else if x == 0.0 {\n debug_assert!(abs_y - 1.0 <= SCALAR_NEARLY_ZERO);\n quadrant = if y > 0.0 { 1 } else { 3 }; // 90 / 270\n } else {\n if y < 0.0 {\n quadrant += 2;\n }\n\n if (x < 0.0) != (y < 0.0) {\n quadrant += 1;\n }\n }\n\n let quadrant_points = [\n Point::from_xy(1.0, 0.0),\n Point::from_xy(1.0, 1.0),\n Point::from_xy(0.0, 1.0),\n Point::from_xy(-1.0, 1.0),\n Point::from_xy(-1.0, 0.0),\n Point::from_xy(-1.0, -1.0),\n Point::from_xy(0.0, -1.0),\n Point::from_xy(1.0, -1.0),\n ];\n\n const QUADRANT_WEIGHT: f32 = SCALAR_ROOT_2_OVER_2;\n\n let mut conic_count = quadrant;\n for i in 0..conic_count {\n dst[i] = Conic::from_points(&quadrant_points[i * 2..], QUADRANT_WEIGHT);\n }\n\n // Now compute any remaing (sub-90-degree) arc for the last conic\n let final_pt = Point::from_xy(x, y);\n let last_q = quadrant_points[quadrant * 2]; // will already be a unit-vector\n let dot = last_q.dot(final_pt);\n debug_assert!(0.0 <= dot && dot <= 1.0 + SCALAR_NEARLY_ZERO);\n\n if dot < 1.0 {\n let mut off_curve = Point::from_xy(last_q.x + x, last_q.y + y);\n // compute the bisector vector, and then rescale to be the off-curve point.\n // we compute its length from cos(theta/2) = length / 1, using half-angle identity we get\n // length = sqrt(2 / (1 + cos(theta)). We already have cos() when to computed the dot.\n // This is nice, since our computed weight is cos(theta/2) as well!\n let cos_theta_over_2 = ((1.0 + dot) / 2.0).sqrt();\n off_curve.set_length(cos_theta_over_2.invert());\n if !last_q.almost_equal(off_curve) {\n dst[conic_count] = Conic::new(last_q, off_curve, final_pt, cos_theta_over_2);\n conic_count += 1;\n }\n }\n\n // now handle counter-clockwise and the initial unitStart rotation\n let mut transform = Transform::from_sin_cos(u_start.y, u_start.x);\n if dir == PathDirection::CCW {\n transform = transform.pre_scale(1.0, -1.0);\n }\n\n transform = transform.post_concat(user_transform);\n\n for conic in dst.iter_mut().take(conic_count) {\n transform.map_points(&mut conic.points);\n }\n\n if conic_count == 0 {\n None\n } else {\n Some(&dst[0..conic_count])\n }\n }\n}\n\nfn subdivide_weight_value(w: f32) -> f32 {\n (0.5 + w * 0.5).sqrt()\n}\n\nfn subdivide<'a>(src: &Conic, mut points: &'a mut [Point], mut level: u8) -> &'a mut [Point] {\n if level == 0 {\n points[0] = src.points[1];\n points[1] = src.points[2];\n &mut points[2..]\n } else {\n let mut dst = src.chop();\n\n let start_y = src.points[0].y;\n let end_y = src.points[2].y;\n if between(start_y, src.points[1].y, end_y) {\n // If the input is monotonic and the output is not, the scan converter hangs.\n // Ensure that the chopped conics maintain their y-order.\n let mid_y = dst.0.points[2].y;\n if !between(start_y, mid_y, end_y) {\n // If the computed midpoint is outside the ends, move it to the closer one.\n let closer_y = if (mid_y - start_y).abs() < (mid_y - end_y).abs() {\n start_y\n } else {\n end_y\n };\n dst.0.points[2].y = closer_y;\n dst.1.points[0].y = closer_y;\n }\n\n if !between(start_y, dst.0.points[1].y, dst.0.points[2].y) {\n // If the 1st control is not between the start and end, put it at the start.\n // This also reduces the quad to a line.\n dst.0.points[1].y = start_y;\n }\n\n if !between(dst.1.points[0].y, dst.1.points[1].y, end_y) {\n // If the 2nd control is not between the start and end, put it at the end.\n // This also reduces the quad to a line.\n dst.1.points[1].y = end_y;\n }\n\n // Verify that all five points are in order.\n debug_assert!(between(start_y, dst.0.points[1].y, dst.0.points[2].y));\n debug_assert!(between(\n dst.0.points[1].y,\n dst.0.points[2].y,\n dst.1.points[1].y\n ));\n debug_assert!(between(dst.0.points[2].y, dst.1.points[1].y, end_y));\n }\n\n level -= 1;\n points = subdivide(&dst.0, points, level);\n subdivide(&dst.1, points, level)\n }\n}\n\n// This was originally developed and tested for pathops: see SkOpTypes.h\n// returns true if (a <= b <= c) || (a >= b >= c)\nfn between(a: f32, b: f32, c: f32) -> bool {\n (a - b) * (c - b) <= 0.0\n}\n\npub(crate) struct AutoConicToQuads {\n pub points: [Point; 64],\n pub len: u8, // the number of quads\n}\n\nimpl AutoConicToQuads {\n pub fn compute(pt0: Point, pt1: Point, pt2: Point, weight: f32) -> Option {\n let conic = Conic::new(pt0, pt1, pt2, weight);\n let pow2 = conic.compute_quad_pow2(0.25)?;\n let mut points = [Point::zero(); 64];\n let len = conic.chop_into_quads_pow2(pow2, &mut points);\n Some(AutoConicToQuads { points, len })\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn eval_cubic_at_1() {\n let src = [\n Point::from_xy(30.0, 40.0),\n Point::from_xy(30.0, 40.0),\n Point::from_xy(171.0, 45.0),\n Point::from_xy(180.0, 155.0),\n ];\n\n assert_eq!(\n eval_cubic_pos_at(&src, NormalizedF32::ZERO),\n Point::from_xy(30.0, 40.0)\n );\n assert_eq!(\n eval_cubic_tangent_at(&src, NormalizedF32::ZERO),\n Point::from_xy(141.0, 5.0)\n );\n }\n\n #[test]\n fn find_cubic_max_curvature_1() {\n let src = [\n Point::from_xy(20.0, 160.0),\n Point::from_xy(20.0001, 160.0),\n Point::from_xy(160.0, 20.0),\n Point::from_xy(160.0001, 20.0),\n ];\n\n let mut t_values = [NormalizedF32::ZERO; 3];\n let t_values = find_cubic_max_curvature(&src, &mut t_values);\n\n assert_eq!(\n &t_values,\n &[\n NormalizedF32::ZERO,\n NormalizedF32::new_clamped(0.5),\n NormalizedF32::ONE,\n ]\n );\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/rect.rs", "sha256": "d8da6ff60134d50ad36f52f05c097eaefe5c31d08eff30af68c21cbc42da72dd", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse core::convert::TryFrom;\n\nuse crate::{FiniteF32, IntSize, LengthU32, PathBuilder, Point, SaturateRound, Size, Transform};\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse crate::NoStdFloat;\n\n/// An integer rectangle.\n///\n/// # Guarantees\n///\n/// - Width and height are in 1..=i32::MAX range.\n/// - x+width and y+height does not overflow.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, PartialEq, Debug)]\npub struct IntRect {\n x: i32,\n y: i32,\n width: LengthU32,\n height: LengthU32,\n}\n\nimpl IntRect {\n /// Creates a new `IntRect`.\n pub fn from_xywh(x: i32, y: i32, width: u32, height: u32) -> Option {\n x.checked_add(i32::try_from(width).ok()?)?;\n y.checked_add(i32::try_from(height).ok()?)?;\n\n Some(IntRect {\n x,\n y,\n width: LengthU32::new(width)?,\n height: LengthU32::new(height)?,\n })\n }\n\n /// Creates a new `IntRect`.\n pub fn from_ltrb(left: i32, top: i32, right: i32, bottom: i32) -> Option {\n let width = u32::try_from(right.checked_sub(left)?).ok()?;\n let height = u32::try_from(bottom.checked_sub(top)?).ok()?;\n IntRect::from_xywh(left, top, width, height)\n }\n\n /// Returns rect's X position.\n pub fn x(&self) -> i32 {\n self.x\n }\n\n /// Returns rect's Y position.\n pub fn y(&self) -> i32 {\n self.y\n }\n\n /// Returns rect's width.\n pub fn width(&self) -> u32 {\n self.width.get()\n }\n\n /// Returns rect's height.\n pub fn height(&self) -> u32 {\n self.height.get()\n }\n\n /// Returns rect's left edge.\n pub fn left(&self) -> i32 {\n self.x\n }\n\n /// Returns rect's top edge.\n pub fn top(&self) -> i32 {\n self.y\n }\n\n /// Returns rect's right edge.\n pub fn right(&self) -> i32 {\n // No overflow is guaranteed by constructors.\n self.x + self.width.get() as i32\n }\n\n /// Returns rect's bottom edge.\n pub fn bottom(&self) -> i32 {\n // No overflow is guaranteed by constructors.\n self.y + self.height.get() as i32\n }\n\n /// Returns rect's size.\n pub fn size(&self) -> IntSize {\n IntSize::from_wh_safe(self.width, self.height)\n }\n\n /// Checks that the rect is completely includes `other` Rect.\n pub fn contains(&self, other: &Self) -> bool {\n self.x <= other.x\n && self.y <= other.y\n && self.right() >= other.right()\n && self.bottom() >= other.bottom()\n }\n\n /// Returns an intersection of two rectangles.\n ///\n /// Returns `None` otherwise.\n pub fn intersect(&self, other: &Self) -> Option {\n let left = self.x.max(other.x);\n let top = self.y.max(other.y);\n\n let right = self.right().min(other.right());\n let bottom = self.bottom().min(other.bottom());\n\n let w = u32::try_from(right.checked_sub(left)?).ok()?;\n let h = u32::try_from(bottom.checked_sub(top)?).ok()?;\n\n IntRect::from_xywh(left, top, w, h)\n }\n\n /// Insets the rectangle.\n pub fn inset(&self, dx: i32, dy: i32) -> Option {\n IntRect::from_ltrb(\n self.left() + dx,\n self.top() + dy,\n self.right() - dx,\n self.bottom() - dy,\n )\n }\n\n /// Outsets the rectangle.\n pub fn make_outset(&self, dx: i32, dy: i32) -> Option {\n IntRect::from_ltrb(\n self.left().saturating_sub(dx),\n self.top().saturating_sub(dy),\n self.right().saturating_add(dx),\n self.bottom().saturating_add(dy),\n )\n }\n\n /// Translates the rect by the specified offset.\n pub fn translate(&self, tx: i32, ty: i32) -> Option {\n IntRect::from_xywh(self.x() + tx, self.y() + ty, self.width(), self.height())\n }\n\n /// Translates the rect to the specified position.\n pub fn translate_to(&self, x: i32, y: i32) -> Option {\n IntRect::from_xywh(x, y, self.width(), self.height())\n }\n\n /// Converts into `Rect`.\n pub fn to_rect(&self) -> Rect {\n // Can't fail, because `IntRect` is always valid.\n Rect::from_ltrb(\n self.x as f32,\n self.y as f32,\n self.x as f32 + self.width.get() as f32,\n self.y as f32 + self.height.get() as f32,\n )\n .unwrap()\n }\n}\n\n#[cfg(test)]\nmod int_rect_tests {\n use super::*;\n\n #[test]\n fn tests() {\n assert_eq!(IntRect::from_xywh(0, 0, 0, 0), None);\n assert_eq!(IntRect::from_xywh(0, 0, 1, 0), None);\n assert_eq!(IntRect::from_xywh(0, 0, 0, 1), None);\n\n assert_eq!(\n IntRect::from_xywh(0, 0, core::u32::MAX, core::u32::MAX),\n None\n );\n assert_eq!(IntRect::from_xywh(0, 0, 1, core::u32::MAX), None);\n assert_eq!(IntRect::from_xywh(0, 0, core::u32::MAX, 1), None);\n\n assert_eq!(IntRect::from_xywh(core::i32::MAX, 0, 1, 1), None);\n assert_eq!(IntRect::from_xywh(0, core::i32::MAX, 1, 1), None);\n\n {\n // No intersection.\n let r1 = IntRect::from_xywh(1, 2, 3, 4).unwrap();\n let r2 = IntRect::from_xywh(11, 12, 13, 14).unwrap();\n assert_eq!(r1.intersect(&r2), None);\n }\n\n {\n // Second inside the first one.\n let r1 = IntRect::from_xywh(1, 2, 30, 40).unwrap();\n let r2 = IntRect::from_xywh(11, 12, 13, 14).unwrap();\n assert_eq!(r1.intersect(&r2), IntRect::from_xywh(11, 12, 13, 14));\n }\n\n {\n // Partial overlap.\n let r1 = IntRect::from_xywh(1, 2, 30, 40).unwrap();\n let r2 = IntRect::from_xywh(11, 12, 50, 60).unwrap();\n assert_eq!(r1.intersect(&r2), IntRect::from_xywh(11, 12, 20, 30));\n }\n }\n}\n\n/// A rectangle defined by left, top, right and bottom edges.\n///\n/// Can have zero width and/or height. But not a negative one.\n///\n/// # Guarantees\n///\n/// - All values are finite.\n/// - Left edge is <= right.\n/// - Top edge is <= bottom.\n/// - Width and height are <= f32::MAX.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, PartialEq)]\npub struct Rect {\n left: FiniteF32,\n top: FiniteF32,\n right: FiniteF32,\n bottom: FiniteF32,\n}\n\nimpl core::fmt::Debug for Rect {\n fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n f.debug_struct(\"Rect\")\n .field(\"left\", &self.left.get())\n .field(\"top\", &self.top.get())\n .field(\"right\", &self.right.get())\n .field(\"bottom\", &self.bottom.get())\n .finish()\n }\n}\n\nimpl Rect {\n /// Creates new `Rect`.\n pub fn from_ltrb(left: f32, top: f32, right: f32, bottom: f32) -> Option {\n let left = FiniteF32::new(left)?;\n let top = FiniteF32::new(top)?;\n let right = FiniteF32::new(right)?;\n let bottom = FiniteF32::new(bottom)?;\n\n if left.get() <= right.get() && top.get() <= bottom.get() {\n // Width and height must not overflow.\n checked_f32_sub(right.get(), left.get())?;\n checked_f32_sub(bottom.get(), top.get())?;\n\n Some(Rect {\n left,\n top,\n right,\n bottom,\n })\n } else {\n None\n }\n }\n\n /// Creates new `Rect`.\n pub fn from_xywh(x: f32, y: f32, w: f32, h: f32) -> Option {\n Rect::from_ltrb(x, y, w + x, h + y)\n }\n\n /// Returns the left edge.\n pub fn left(&self) -> f32 {\n self.left.get()\n }\n\n /// Returns the top edge.\n pub fn top(&self) -> f32 {\n self.top.get()\n }\n\n /// Returns the right edge.\n pub fn right(&self) -> f32 {\n self.right.get()\n }\n\n /// Returns the bottom edge.\n pub fn bottom(&self) -> f32 {\n self.bottom.get()\n }\n\n /// Returns rect's X position.\n pub fn x(&self) -> f32 {\n self.left.get()\n }\n\n /// Returns rect's Y position.\n pub fn y(&self) -> f32 {\n self.top.get()\n }\n\n /// Returns rect's width.\n #[inline]\n pub fn width(&self) -> f32 {\n self.right.get() - self.left.get()\n }\n\n /// Returns rect's height.\n #[inline]\n pub fn height(&self) -> f32 {\n self.bottom.get() - self.top.get()\n }\n\n /// Converts into an `IntRect` by adding 0.5 and discarding the fractional portion.\n ///\n /// Width and height are guarantee to be >= 1.\n pub fn round(&self) -> Option {\n IntRect::from_xywh(\n i32::saturate_round(self.x()),\n i32::saturate_round(self.y()),\n core::cmp::max(1, i32::saturate_round(self.width()) as u32),\n core::cmp::max(1, i32::saturate_round(self.height()) as u32),\n )\n }\n\n /// Converts into an `IntRect` rounding outwards.\n ///\n /// Width and height are guarantee to be >= 1.\n pub fn round_out(&self) -> Option {\n IntRect::from_xywh(\n i32::saturate_floor(self.x()),\n i32::saturate_floor(self.y()),\n core::cmp::max(1, i32::saturate_ceil(self.width()) as u32),\n core::cmp::max(1, i32::saturate_ceil(self.height()) as u32),\n )\n }\n\n /// Returns an intersection of two rectangles.\n ///\n /// Returns `None` otherwise.\n pub fn intersect(&self, other: &Self) -> Option {\n let left = self.x().max(other.x());\n let top = self.y().max(other.y());\n\n let right = self.right().min(other.right());\n let bottom = self.bottom().min(other.bottom());\n\n Rect::from_ltrb(left, top, right, bottom)\n }\n\n /// Creates a Rect from Point array.\n ///\n /// Returns None if count is zero or if Point array contains an infinity or NaN.\n pub fn from_points(points: &[Point]) -> Option {\n use crate::f32x4_t::f32x4;\n\n if points.is_empty() {\n return None;\n }\n\n let mut offset = 0;\n let mut min;\n let mut max;\n if points.len() & 1 != 0 {\n let pt = points[0];\n min = f32x4([pt.x, pt.y, pt.x, pt.y]);\n max = min;\n offset += 1;\n } else {\n let pt0 = points[0];\n let pt1 = points[1];\n min = f32x4([pt0.x, pt0.y, pt1.x, pt1.y]);\n max = min;\n offset += 2;\n }\n\n let mut accum = f32x4::default();\n while offset != points.len() {\n let pt0 = points[offset + 0];\n let pt1 = points[offset + 1];\n let xy = f32x4([pt0.x, pt0.y, pt1.x, pt1.y]);\n\n accum *= xy;\n min = min.min(xy);\n max = max.max(xy);\n offset += 2;\n }\n\n let all_finite = accum * f32x4::default() == f32x4::default();\n let min: [f32; 4] = min.0;\n let max: [f32; 4] = max.0;\n if all_finite {\n Rect::from_ltrb(\n min[0].min(min[2]),\n min[1].min(min[3]),\n max[0].max(max[2]),\n max[1].max(max[3]),\n )\n } else {\n None\n }\n }\n\n /// Insets the rectangle by the specified offset.\n pub fn inset(&mut self, dx: f32, dy: f32) -> Option {\n Rect::from_ltrb(\n self.left() + dx,\n self.top() + dy,\n self.right() - dx,\n self.bottom() - dy,\n )\n }\n\n /// Outsets the rectangle by the specified offset.\n pub fn outset(&mut self, dx: f32, dy: f32) -> Option {\n self.inset(-dx, -dy)\n }\n\n /// Transforms the rect using the provided `Transform`.\n ///\n /// This method is expensive.\n pub fn transform(&self, ts: Transform) -> Option {\n if !ts.is_identity() {\n // TODO: remove allocation\n let mut path = PathBuilder::from_rect(*self);\n path = path.transform(ts)?;\n Some(path.bounds())\n } else {\n Some(*self)\n }\n }\n\n /// Applies a bounding box transform.\n pub fn bbox_transform(&self, bbox: NonZeroRect) -> Self {\n let x = self.x() * bbox.width() + bbox.x();\n let y = self.y() * bbox.height() + bbox.y();\n let w = self.width() * bbox.width();\n let h = self.height() * bbox.height();\n Self::from_xywh(x, y, w, h).unwrap()\n }\n\n /// Converts into [`NonZeroRect`].\n pub fn to_non_zero_rect(&self) -> Option {\n NonZeroRect::from_xywh(self.x(), self.y(), self.width(), self.height())\n }\n}\n\nfn checked_f32_sub(a: f32, b: f32) -> Option {\n debug_assert!(a.is_finite());\n debug_assert!(b.is_finite());\n\n let n = a as f64 - b as f64;\n // Not sure if this is perfectly correct.\n if n > core::f32::MIN as f64 && n < core::f32::MAX as f64 {\n Some(n as f32)\n } else {\n None\n }\n}\n\n#[cfg(test)]\nmod rect_tests {\n use super::*;\n\n #[test]\n fn tests() {\n assert_eq!(Rect::from_ltrb(10.0, 10.0, 5.0, 10.0), None);\n assert_eq!(Rect::from_ltrb(10.0, 10.0, 10.0, 5.0), None);\n assert_eq!(Rect::from_ltrb(core::f32::NAN, 10.0, 10.0, 10.0), None);\n assert_eq!(Rect::from_ltrb(10.0, core::f32::NAN, 10.0, 10.0), None);\n assert_eq!(Rect::from_ltrb(10.0, 10.0, core::f32::NAN, 10.0), None);\n assert_eq!(Rect::from_ltrb(10.0, 10.0, 10.0, core::f32::NAN), None);\n assert_eq!(Rect::from_ltrb(10.0, 10.0, 10.0, core::f32::INFINITY), None);\n\n let rect = Rect::from_ltrb(10.0, 20.0, 30.0, 40.0).unwrap();\n assert_eq!(rect.left(), 10.0);\n assert_eq!(rect.top(), 20.0);\n assert_eq!(rect.right(), 30.0);\n assert_eq!(rect.bottom(), 40.0);\n assert_eq!(rect.width(), 20.0);\n assert_eq!(rect.height(), 20.0);\n\n let rect = Rect::from_ltrb(-30.0, 20.0, -10.0, 40.0).unwrap();\n assert_eq!(rect.width(), 20.0);\n assert_eq!(rect.height(), 20.0);\n }\n\n #[test]\n fn round_overflow() {\n // minimum value that cause overflow\n // because i32::MAX has no exact conversion to f32\n let x = 128.0;\n // maximum width\n let width = i32::MAX as f32;\n\n let rect = Rect::from_xywh(x, 0.0, width, 1.0).unwrap();\n assert_eq!(rect.round(), None);\n assert_eq!(rect.round_out(), None);\n }\n}\n\n/// A rectangle defined by left, top, right and bottom edges.\n///\n/// Similar to [`Rect`], but width and height guarantee to be non-zero and positive.\n///\n/// # Guarantees\n///\n/// - All values are finite.\n/// - Left edge is < right.\n/// - Top edge is < bottom.\n/// - Width and height are <= f32::MAX.\n/// - Width and height are > 0.0\n#[allow(missing_docs)]\n#[derive(Copy, Clone, PartialEq)]\npub struct NonZeroRect {\n left: FiniteF32,\n top: FiniteF32,\n right: FiniteF32,\n bottom: FiniteF32,\n}\n\nimpl core::fmt::Debug for NonZeroRect {\n fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n f.debug_struct(\"NonZeroRect\")\n .field(\"left\", &self.left.get())\n .field(\"top\", &self.top.get())\n .field(\"right\", &self.right.get())\n .field(\"bottom\", &self.bottom.get())\n .finish()\n }\n}\n\nimpl NonZeroRect {\n /// Creates new `NonZeroRect`.\n pub fn from_ltrb(left: f32, top: f32, right: f32, bottom: f32) -> Option {\n let left = FiniteF32::new(left)?;\n let top = FiniteF32::new(top)?;\n let right = FiniteF32::new(right)?;\n let bottom = FiniteF32::new(bottom)?;\n\n if left.get() < right.get() && top.get() < bottom.get() {\n // Width and height must not overflow.\n checked_f32_sub(right.get(), left.get())?;\n checked_f32_sub(bottom.get(), top.get())?;\n\n Some(Self {\n left,\n top,\n right,\n bottom,\n })\n } else {\n None\n }\n }\n\n /// Creates new `NonZeroRect`.\n pub fn from_xywh(x: f32, y: f32, w: f32, h: f32) -> Option {\n Self::from_ltrb(x, y, w + x, h + y)\n }\n\n /// Returns the left edge.\n pub fn left(&self) -> f32 {\n self.left.get()\n }\n\n /// Returns the top edge.\n pub fn top(&self) -> f32 {\n self.top.get()\n }\n\n /// Returns the right edge.\n pub fn right(&self) -> f32 {\n self.right.get()\n }\n\n /// Returns the bottom edge.\n pub fn bottom(&self) -> f32 {\n self.bottom.get()\n }\n\n /// Returns rect's X position.\n pub fn x(&self) -> f32 {\n self.left.get()\n }\n\n /// Returns rect's Y position.\n pub fn y(&self) -> f32 {\n self.top.get()\n }\n\n /// Returns rect's width.\n pub fn width(&self) -> f32 {\n self.right.get() - self.left.get()\n }\n\n /// Returns rect's height.\n pub fn height(&self) -> f32 {\n self.bottom.get() - self.top.get()\n }\n\n /// Returns rect's size.\n pub fn size(&self) -> Size {\n Size::from_wh(self.width(), self.height()).unwrap()\n }\n\n /// Translates the rect to the specified position.\n pub fn translate_to(&self, x: f32, y: f32) -> Option {\n Self::from_xywh(x, y, self.width(), self.height())\n }\n\n /// Transforms the rect using the provided `Transform`.\n ///\n /// This method is expensive.\n pub fn transform(&self, ts: Transform) -> Option {\n if !ts.is_identity() {\n // TODO: remove allocation\n let mut path = PathBuilder::from_rect(self.to_rect());\n path = path.transform(ts)?;\n path.bounds().to_non_zero_rect()\n } else {\n Some(*self)\n }\n }\n\n /// Applies a bounding box transform.\n pub fn bbox_transform(&self, bbox: NonZeroRect) -> Self {\n let x = self.x() * bbox.width() + bbox.x();\n let y = self.y() * bbox.height() + bbox.y();\n let w = self.width() * bbox.width();\n let h = self.height() * bbox.height();\n Self::from_xywh(x, y, w, h).unwrap()\n }\n\n /// Converts into [`Rect`].\n pub fn to_rect(&self) -> Rect {\n Rect::from_xywh(self.x(), self.y(), self.width(), self.height()).unwrap()\n }\n\n /// Converts into [`IntRect`].\n pub fn to_int_rect(&self) -> IntRect {\n IntRect::from_xywh(\n self.x().floor() as i32,\n self.y().floor() as i32,\n core::cmp::max(1, self.width().ceil() as u32),\n core::cmp::max(1, self.height().ceil() as u32),\n )\n .unwrap()\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/scalar.rs", "sha256": "f6ece3dfbc76a30a3b51a2bafe43addbd59840a84cb94a4c58fb9e718e362706", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse crate::floating_point::f32_as_2s_compliment;\n\n#[allow(missing_docs)]\npub const SCALAR_MAX: f32 = 3.402823466e+38;\n#[allow(missing_docs)]\npub const SCALAR_NEARLY_ZERO: f32 = 1.0 / (1 << 12) as f32;\n#[allow(missing_docs)]\npub const SCALAR_ROOT_2_OVER_2: f32 = 0.707106781;\n\n/// Float number extension methods.\n///\n/// Mainly for internal use. Do not rely on it!\n#[allow(missing_docs)]\npub trait Scalar {\n fn half(self) -> Self;\n fn ave(self, other: Self) -> Self;\n fn sqr(self) -> Self;\n fn invert(self) -> Self;\n fn bound(self, min: Self, max: Self) -> Self;\n fn is_nearly_equal(self, other: Self) -> bool;\n fn is_nearly_zero(self) -> bool;\n fn is_nearly_zero_within_tolerance(self, tolerance: Self) -> bool;\n fn almost_dequal_ulps(self, other: Self) -> bool;\n}\n\nimpl Scalar for f32 {\n fn half(self) -> f32 {\n self * 0.5\n }\n\n fn ave(self, other: Self) -> f32 {\n (self + other) * 0.5\n }\n\n fn sqr(self) -> f32 {\n self * self\n }\n\n fn invert(self) -> f32 {\n 1.0 / self\n }\n\n // Works just like SkTPin, returning `max` for NaN/inf\n /// A non-panicking clamp.\n fn bound(self, min: Self, max: Self) -> Self {\n max.min(self).max(min)\n }\n\n fn is_nearly_equal(self, other: Self) -> bool {\n (self - other).abs() <= SCALAR_NEARLY_ZERO\n }\n\n fn is_nearly_zero(self) -> bool {\n self.is_nearly_zero_within_tolerance(SCALAR_NEARLY_ZERO)\n }\n\n fn is_nearly_zero_within_tolerance(self, tolerance: Self) -> bool {\n debug_assert!(tolerance >= 0.0);\n self.abs() <= tolerance\n }\n\n // From SkPathOpsTypes.\n fn almost_dequal_ulps(self, other: Self) -> bool {\n const ULPS_EPSILON: i32 = 16;\n let a_bits = f32_as_2s_compliment(self);\n let b_bits = f32_as_2s_compliment(other);\n // Find the difference in ULPs.\n a_bits < b_bits + ULPS_EPSILON && b_bits < a_bits + ULPS_EPSILON\n }\n}\n\n#[allow(missing_docs)]\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\npub trait NoStdFloat {\n fn trunc(self) -> Self;\n fn sqrt(self) -> Self;\n fn abs(self) -> Self;\n fn sin(self) -> Self;\n fn cos(self) -> Self;\n fn ceil(self) -> Self;\n fn floor(self) -> Self;\n fn round(self) -> Self;\n fn powf(self, y: Self) -> Self;\n fn acos(self) -> Self;\n}\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nimpl NoStdFloat for f32 {\n fn trunc(self) -> Self {\n libm::truncf(self)\n }\n fn sqrt(self) -> Self {\n libm::sqrtf(self)\n }\n fn abs(self) -> Self {\n libm::fabsf(self)\n }\n fn sin(self) -> Self {\n libm::sinf(self)\n }\n fn cos(self) -> Self {\n libm::cosf(self)\n }\n fn ceil(self) -> Self {\n libm::ceilf(self)\n }\n fn floor(self) -> Self {\n libm::floorf(self)\n }\n fn round(self) -> Self {\n libm::roundf(self)\n }\n fn powf(self, y: Self) -> Self {\n libm::powf(self, y)\n }\n fn acos(self) -> Self {\n libm::acosf(self)\n }\n}\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nimpl NoStdFloat for f64 {\n fn trunc(self) -> Self {\n libm::trunc(self)\n }\n fn sqrt(self) -> Self {\n libm::sqrt(self)\n }\n fn abs(self) -> Self {\n libm::fabs(self)\n }\n fn sin(self) -> Self {\n libm::sin(self)\n }\n fn cos(self) -> Self {\n libm::cos(self)\n }\n fn ceil(self) -> Self {\n libm::ceil(self)\n }\n fn floor(self) -> Self {\n libm::floor(self)\n }\n fn round(self) -> Self {\n libm::round(self)\n }\n fn powf(self, y: Self) -> Self {\n libm::pow(self, y)\n }\n fn acos(self) -> Self {\n libm::acos(self)\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn bound() {\n assert_eq!(core::f32::NAN.bound(0.0, 1.0), 1.0);\n assert_eq!(core::f32::INFINITY.bound(0.0, 1.0), 1.0);\n assert_eq!(core::f32::NEG_INFINITY.bound(0.0, 1.0), 0.0);\n assert_eq!(core::f32::EPSILON.bound(0.0, 1.0), core::f32::EPSILON);\n assert_eq!(0.5.bound(0.0, 1.0), 0.5);\n assert_eq!((-1.0).bound(0.0, 1.0), 0.0);\n assert_eq!(2.0.bound(0.0, 1.0), 1.0);\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/size.rs", "sha256": "2a887b25bee4194b730283f3e09ea8647af1f4584c2ba5d415024b0485022107", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse strict_num::NonZeroPositiveF32;\n\nuse crate::{IntRect, LengthU32, NonZeroRect, Rect};\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse crate::NoStdFloat;\n\n/// An integer size.\n///\n/// # Guarantees\n///\n/// - Width and height are positive and non-zero.\n#[derive(Copy, Clone, PartialEq, Debug)]\npub struct IntSize {\n width: LengthU32,\n height: LengthU32,\n}\n\nimpl IntSize {\n /// Creates a new `IntSize` from width and height.\n pub fn from_wh(width: u32, height: u32) -> Option {\n Some(IntSize {\n width: LengthU32::new(width)?,\n height: LengthU32::new(height)?,\n })\n }\n\n pub(crate) fn from_wh_safe(width: LengthU32, height: LengthU32) -> Self {\n IntSize { width, height }\n }\n\n /// Returns width.\n pub fn width(&self) -> u32 {\n self.width.get()\n }\n\n /// Returns height.\n pub fn height(&self) -> u32 {\n self.height.get()\n }\n\n /// Returns width and height as a tuple.\n pub fn dimensions(&self) -> (u32, u32) {\n (self.width(), self.height())\n }\n\n /// Scales current size by the specified factor.\n #[inline]\n pub fn scale_by(&self, factor: f32) -> Option {\n Self::from_wh(\n (self.width() as f32 * factor).round() as u32,\n (self.height() as f32 * factor).round() as u32,\n )\n }\n\n /// Scales current size to the specified size.\n #[inline]\n pub fn scale_to(&self, to: Self) -> Self {\n size_scale(*self, to, false)\n }\n\n /// Scales current size to the specified width.\n #[inline]\n pub fn scale_to_width(&self, new_width: u32) -> Option {\n let new_height = (new_width as f32 * self.height() as f32 / self.width() as f32).ceil();\n Self::from_wh(new_width, new_height as u32)\n }\n\n /// Scales current size to the specified height.\n #[inline]\n pub fn scale_to_height(&self, new_height: u32) -> Option {\n let new_width = (new_height as f32 * self.width() as f32 / self.height() as f32).ceil();\n Self::from_wh(new_width as u32, new_height)\n }\n\n /// Converts into [`Size`].\n pub fn to_size(&self) -> Size {\n Size::from_wh(self.width() as f32, self.height() as f32).unwrap()\n }\n\n /// Converts into [`IntRect`] at the provided position.\n pub fn to_int_rect(&self, x: i32, y: i32) -> IntRect {\n IntRect::from_xywh(x, y, self.width(), self.height()).unwrap()\n }\n}\n\nfn size_scale(s1: IntSize, s2: IntSize, expand: bool) -> IntSize {\n let rw = (s2.height() as f32 * s1.width() as f32 / s1.height() as f32).ceil() as u32;\n let with_h = if expand {\n rw <= s2.width()\n } else {\n rw >= s2.width()\n };\n\n if !with_h {\n IntSize::from_wh(rw, s2.height()).unwrap()\n } else {\n let h = (s2.width() as f32 * s1.height() as f32 / s1.width() as f32).ceil() as u32;\n IntSize::from_wh(s2.width(), h).unwrap()\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn int_size_tests() {\n assert_eq!(IntSize::from_wh(0, 0), None);\n assert_eq!(IntSize::from_wh(1, 0), None);\n assert_eq!(IntSize::from_wh(0, 1), None);\n\n let size = IntSize::from_wh(3, 4).unwrap();\n assert_eq!(\n size.to_int_rect(1, 2),\n IntRect::from_xywh(1, 2, 3, 4).unwrap()\n );\n }\n}\n\n/// A size.\n///\n/// # Guarantees\n///\n/// - Width and height are positive, non-zero and finite.\n#[derive(Copy, Clone, PartialEq, Debug)]\npub struct Size {\n width: NonZeroPositiveF32,\n height: NonZeroPositiveF32,\n}\n\nimpl Size {\n /// Creates a new `Size` from width and height.\n pub fn from_wh(width: f32, height: f32) -> Option {\n Some(Size {\n width: NonZeroPositiveF32::new(width)?,\n height: NonZeroPositiveF32::new(height)?,\n })\n }\n\n /// Returns width.\n pub fn width(&self) -> f32 {\n self.width.get()\n }\n\n /// Returns height.\n pub fn height(&self) -> f32 {\n self.height.get()\n }\n\n /// Scales current size to specified size.\n pub fn scale_to(&self, to: Self) -> Self {\n size_scale_f64(*self, to, false)\n }\n\n /// Expands current size to specified size.\n pub fn expand_to(&self, to: Self) -> Self {\n size_scale_f64(*self, to, true)\n }\n\n /// Converts into [`IntSize`].\n pub fn to_int_size(&self) -> IntSize {\n IntSize::from_wh(\n core::cmp::max(1, self.width().round() as u32),\n core::cmp::max(1, self.height().round() as u32),\n )\n .unwrap()\n }\n\n /// Converts the current size to `Rect` at provided position.\n pub fn to_rect(&self, x: f32, y: f32) -> Option {\n Rect::from_xywh(x, y, self.width.get(), self.height.get())\n }\n\n /// Converts the current size to `NonZeroRect` at provided position.\n pub fn to_non_zero_rect(&self, x: f32, y: f32) -> NonZeroRect {\n NonZeroRect::from_xywh(x, y, self.width.get(), self.height.get()).unwrap()\n }\n}\n\nfn size_scale_f64(s1: Size, s2: Size, expand: bool) -> Size {\n let rw = s2.height.get() * s1.width.get() / s1.height.get();\n let with_h = if expand {\n rw <= s2.width.get()\n } else {\n rw >= s2.width.get()\n };\n if !with_h {\n Size::from_wh(rw, s2.height.get()).unwrap()\n } else {\n let h = s2.width.get() * s1.height.get() / s1.width.get();\n Size::from_wh(s2.width.get(), h).unwrap()\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/stroker.rs", "sha256": "e3a53080763ec4dca637b33aea663f17c697450ff0080f37219fb53b634ab94d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2008 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\n// Based on SkStroke.cpp\n\nuse crate::{Path, Point, Transform};\n\nuse crate::dash::StrokeDash;\nuse crate::floating_point::{NonZeroPositiveF32, NormalizedF32, NormalizedF32Exclusive};\nuse crate::path::{PathSegment, PathSegmentsIter};\nuse crate::path_builder::{PathBuilder, PathDirection};\nuse crate::path_geometry;\nuse crate::scalar::{Scalar, SCALAR_NEARLY_ZERO, SCALAR_ROOT_2_OVER_2};\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse crate::NoStdFloat;\n\nstruct SwappableBuilders<'a> {\n inner: &'a mut PathBuilder,\n outer: &'a mut PathBuilder,\n}\n\nimpl<'a> SwappableBuilders<'a> {\n fn swap(&mut self) {\n // Skia swaps pointers to inner and outer builders during joining,\n // but not builders itself. So a simple `core::mem::swap` will produce invalid results.\n // And if we try to use use `core::mem::swap` on references, like below,\n // borrow checker will be unhappy.\n // That's why we need this wrapper. Maybe there is a better solution.\n core::mem::swap(&mut self.inner, &mut self.outer);\n }\n}\n\n/// Stroke properties.\n#[derive(Clone, PartialEq, Debug)]\npub struct Stroke {\n /// A stroke thickness.\n ///\n /// Must be >= 0.\n ///\n /// When set to 0, a hairline stroking will be used.\n ///\n /// Default: 1.0\n pub width: f32,\n\n /// The limit at which a sharp corner is drawn beveled.\n ///\n /// Default: 4.0\n pub miter_limit: f32,\n\n /// A stroke line cap.\n ///\n /// Default: Butt\n pub line_cap: LineCap,\n\n /// A stroke line join.\n ///\n /// Default: Miter\n pub line_join: LineJoin,\n\n /// A stroke dashing properties.\n ///\n /// Default: None\n pub dash: Option,\n}\n\nimpl Default for Stroke {\n fn default() -> Self {\n Stroke {\n width: 1.0,\n miter_limit: 4.0,\n line_cap: LineCap::default(),\n line_join: LineJoin::default(),\n dash: None,\n }\n }\n}\n\n/// Draws at the beginning and end of an open path contour.\n#[derive(Copy, Clone, PartialEq, Debug)]\npub enum LineCap {\n /// No stroke extension.\n Butt,\n /// Adds circle.\n Round,\n /// Adds square.\n Square,\n}\n\nimpl Default for LineCap {\n fn default() -> Self {\n LineCap::Butt\n }\n}\n\n/// Specifies how corners are drawn when a shape is stroked.\n///\n/// Join affects the four corners of a stroked rectangle, and the connected segments in a\n/// stroked path.\n///\n/// Choose miter join to draw sharp corners. Choose round join to draw a circle with a\n/// radius equal to the stroke width on top of the corner. Choose bevel join to minimally\n/// connect the thick strokes.\n///\n/// The fill path constructed to describe the stroked path respects the join setting but may\n/// not contain the actual join. For instance, a fill path constructed with round joins does\n/// not necessarily include circles at each connected segment.\n#[derive(Copy, Clone, PartialEq, Debug)]\npub enum LineJoin {\n /// Extends to miter limit.\n Miter,\n /// Adds circle.\n Round,\n /// Connects outside edges.\n Bevel,\n}\n\nimpl Default for LineJoin {\n fn default() -> Self {\n LineJoin::Miter\n }\n}\n\nconst QUAD_RECURSIVE_LIMIT: usize = 3;\n\n// quads with extreme widths (e.g. (0,1) (1,6) (0,3) width=5e7) recurse to point of failure\n// largest seen for normal cubics: 5, 26\n// largest seen for normal quads: 11\nconst RECURSIVE_LIMITS: [i32; 4] = [5 * 3, 26 * 3, 11 * 3, 11 * 3]; // 3x limits seen in practice\n\ntype CapProc = fn(\n pivot: Point,\n normal: Point,\n stop: Point,\n other_path: Option<&PathBuilder>,\n path: &mut PathBuilder,\n);\n\ntype JoinProc = fn(\n before_unit_normal: Point,\n pivot: Point,\n after_unit_normal: Point,\n radius: f32,\n inv_miter_limit: f32,\n prev_is_line: bool,\n curr_is_line: bool,\n builders: SwappableBuilders,\n);\n\n#[derive(Copy, Clone, PartialEq, PartialOrd, Debug)]\nenum ReductionType {\n Point, // all curve points are practically identical\n Line, // the control point is on the line between the ends\n Quad, // the control point is outside the line between the ends\n Degenerate, // the control point is on the line but outside the ends\n Degenerate2, // two control points are on the line but outside ends (cubic)\n Degenerate3, // three areas of max curvature found (for cubic)\n}\n\n#[derive(Copy, Clone, PartialEq, Debug)]\nenum StrokeType {\n Outer = 1, // use sign-opposite values later to flip perpendicular axis\n Inner = -1,\n}\n\n#[derive(Copy, Clone, PartialEq, Debug)]\nenum ResultType {\n Split, // the caller should split the quad stroke in two\n Degenerate, // the caller should add a line\n Quad, // the caller should (continue to try to) add a quad stroke\n}\n\n#[derive(Copy, Clone, PartialEq, Debug)]\nenum IntersectRayType {\n CtrlPt,\n ResultType,\n}\n\nimpl Path {\n /// Returns a stoked path.\n ///\n /// `resolution_scale` can be obtained via\n /// [`compute_resolution_scale`](PathStroker::compute_resolution_scale).\n ///\n /// If you plan stroking multiple paths, you can try using [`PathStroker`]\n /// which will preserve temporary allocations required during stroking.\n /// This might improve performance a bit.\n pub fn stroke(&self, stroke: &Stroke, resolution_scale: f32) -> Option {\n PathStroker::new().stroke(self, stroke, resolution_scale)\n }\n}\n\n/// A path stroker.\n#[allow(missing_debug_implementations)]\n#[derive(Clone)]\npub struct PathStroker {\n radius: f32,\n inv_miter_limit: f32,\n res_scale: f32,\n inv_res_scale: f32,\n inv_res_scale_squared: f32,\n\n first_normal: Point,\n prev_normal: Point,\n first_unit_normal: Point,\n prev_unit_normal: Point,\n\n // on original path\n first_pt: Point,\n prev_pt: Point,\n\n first_outer_pt: Point,\n first_outer_pt_index_in_contour: usize,\n segment_count: i32,\n prev_is_line: bool,\n\n capper: CapProc,\n joiner: JoinProc,\n\n // outer is our working answer, inner is temp\n inner: PathBuilder,\n outer: PathBuilder,\n cusper: PathBuilder,\n\n stroke_type: StrokeType,\n\n recursion_depth: i32, // track stack depth to abort if numerics run amok\n found_tangents: bool, // do less work until tangents meet (cubic)\n join_completed: bool, // previous join was not degenerate\n}\n\nimpl Default for PathStroker {\n fn default() -> Self {\n PathStroker::new()\n }\n}\n\nimpl PathStroker {\n /// Creates a new PathStroker.\n pub fn new() -> Self {\n PathStroker {\n radius: 0.0,\n inv_miter_limit: 0.0,\n res_scale: 1.0,\n inv_res_scale: 1.0,\n inv_res_scale_squared: 1.0,\n\n first_normal: Point::zero(),\n prev_normal: Point::zero(),\n first_unit_normal: Point::zero(),\n prev_unit_normal: Point::zero(),\n\n first_pt: Point::zero(),\n prev_pt: Point::zero(),\n\n first_outer_pt: Point::zero(),\n first_outer_pt_index_in_contour: 0,\n segment_count: -1,\n prev_is_line: false,\n\n capper: butt_capper,\n joiner: miter_joiner,\n\n inner: PathBuilder::new(),\n outer: PathBuilder::new(),\n cusper: PathBuilder::new(),\n\n stroke_type: StrokeType::Outer,\n\n recursion_depth: 0,\n found_tangents: false,\n join_completed: false,\n }\n }\n\n /// Computes a resolution scale.\n ///\n /// Resolution scale is the \"intended\" resolution for the output. Default is 1.0.\n ///\n /// Larger values (res > 1) indicate that the result should be more precise, since it will\n /// be zoomed up, and small errors will be magnified.\n ///\n /// Smaller values (0 < res < 1) indicate that the result can be less precise, since it will\n /// be zoomed down, and small errors may be invisible.\n pub fn compute_resolution_scale(ts: &Transform) -> f32 {\n let sx = Point::from_xy(ts.sx, ts.kx).length();\n let sy = Point::from_xy(ts.ky, ts.sy).length();\n if sx.is_finite() && sy.is_finite() {\n let scale = sx.max(sy);\n if scale > 0.0 {\n return scale;\n }\n }\n\n 1.0\n }\n\n /// Stokes the path.\n ///\n /// Can be called multiple times to reuse allocated buffers.\n ///\n /// `resolution_scale` can be obtained via\n /// [`compute_resolution_scale`](Self::compute_resolution_scale).\n pub fn stroke(&mut self, path: &Path, stroke: &Stroke, resolution_scale: f32) -> Option {\n let width = NonZeroPositiveF32::new(stroke.width)?;\n self.stroke_inner(\n path,\n width,\n stroke.miter_limit,\n stroke.line_cap,\n stroke.line_join,\n resolution_scale,\n )\n }\n\n fn stroke_inner(\n &mut self,\n path: &Path,\n width: NonZeroPositiveF32,\n miter_limit: f32,\n line_cap: LineCap,\n mut line_join: LineJoin,\n res_scale: f32,\n ) -> Option {\n // TODO: stroke_rect optimization\n\n let mut inv_miter_limit = 0.0;\n\n if line_join == LineJoin::Miter {\n if miter_limit <= 1.0 {\n line_join = LineJoin::Bevel;\n } else {\n inv_miter_limit = miter_limit.invert();\n }\n }\n\n self.res_scale = res_scale;\n // The '4' below matches the fill scan converter's error term.\n self.inv_res_scale = (res_scale * 4.0).invert();\n self.inv_res_scale_squared = self.inv_res_scale.sqr();\n\n self.radius = width.get().half();\n self.inv_miter_limit = inv_miter_limit;\n\n self.first_normal = Point::zero();\n self.prev_normal = Point::zero();\n self.first_unit_normal = Point::zero();\n self.prev_unit_normal = Point::zero();\n\n self.first_pt = Point::zero();\n self.prev_pt = Point::zero();\n\n self.first_outer_pt = Point::zero();\n self.first_outer_pt_index_in_contour = 0;\n self.segment_count = -1;\n self.prev_is_line = false;\n\n self.capper = cap_factory(line_cap);\n self.joiner = join_factory(line_join);\n\n // Need some estimate of how large our final result (fOuter)\n // and our per-contour temp (fInner) will be, so we don't spend\n // extra time repeatedly growing these arrays.\n //\n // 1x for inner == 'wag' (worst contour length would be better guess)\n self.inner.clear();\n self.inner.reserve(path.verbs.len(), path.points.len());\n\n // 3x for result == inner + outer + join (swag)\n self.outer.clear();\n self.outer\n .reserve(path.verbs.len() * 3, path.points.len() * 3);\n\n self.cusper.clear();\n\n self.stroke_type = StrokeType::Outer;\n\n self.recursion_depth = 0;\n self.found_tangents = false;\n self.join_completed = false;\n\n let mut last_segment_is_line = false;\n let mut iter = path.segments();\n iter.set_auto_close(true);\n while let Some(segment) = iter.next() {\n match segment {\n PathSegment::MoveTo(p) => {\n self.move_to(p);\n }\n PathSegment::LineTo(p) => {\n self.line_to(p, Some(&iter));\n last_segment_is_line = true;\n }\n PathSegment::QuadTo(p1, p2) => {\n self.quad_to(p1, p2);\n last_segment_is_line = false;\n }\n PathSegment::CubicTo(p1, p2, p3) => {\n self.cubic_to(p1, p2, p3);\n last_segment_is_line = false;\n }\n PathSegment::Close => {\n if line_cap != LineCap::Butt {\n // If the stroke consists of a moveTo followed by a close, treat it\n // as if it were followed by a zero-length line. Lines without length\n // can have square and round end caps.\n if self.has_only_move_to() {\n self.line_to(self.move_to_pt(), None);\n last_segment_is_line = true;\n continue;\n }\n\n // If the stroke consists of a moveTo followed by one or more zero-length\n // verbs, then followed by a close, treat is as if it were followed by a\n // zero-length line. Lines without length can have square & round end caps.\n if self.is_current_contour_empty() {\n last_segment_is_line = true;\n continue;\n }\n }\n\n self.close(last_segment_is_line);\n }\n }\n }\n\n self.finish(last_segment_is_line)\n }\n\n fn builders(&mut self) -> SwappableBuilders {\n SwappableBuilders {\n inner: &mut self.inner,\n outer: &mut self.outer,\n }\n }\n\n fn move_to_pt(&self) -> Point {\n self.first_pt\n }\n\n fn move_to(&mut self, p: Point) {\n if self.segment_count > 0 {\n self.finish_contour(false, false);\n }\n\n self.segment_count = 0;\n self.first_pt = p;\n self.prev_pt = p;\n self.join_completed = false;\n }\n\n fn line_to(&mut self, p: Point, iter: Option<&PathSegmentsIter>) {\n let teeny_line = self\n .prev_pt\n .equals_within_tolerance(p, SCALAR_NEARLY_ZERO * self.inv_res_scale);\n if fn_ptr_eq(self.capper, butt_capper) && teeny_line {\n return;\n }\n\n if teeny_line && (self.join_completed || iter.map(|i| i.has_valid_tangent()) == Some(true))\n {\n return;\n }\n\n let mut normal = Point::zero();\n let mut unit_normal = Point::zero();\n if !self.pre_join_to(p, true, &mut normal, &mut unit_normal) {\n return;\n }\n\n self.outer.line_to(p.x + normal.x, p.y + normal.y);\n self.inner.line_to(p.x - normal.x, p.y - normal.y);\n\n self.post_join_to(p, normal, unit_normal);\n }\n\n fn quad_to(&mut self, p1: Point, p2: Point) {\n let quad = [self.prev_pt, p1, p2];\n let (reduction, reduction_type) = check_quad_linear(&quad);\n if reduction_type == ReductionType::Point {\n // If the stroke consists of a moveTo followed by a degenerate curve, treat it\n // as if it were followed by a zero-length line. Lines without length\n // can have square and round end caps.\n self.line_to(p2, None);\n return;\n }\n\n if reduction_type == ReductionType::Line {\n self.line_to(p2, None);\n return;\n }\n\n if reduction_type == ReductionType::Degenerate {\n self.line_to(reduction, None);\n let save_joiner = self.joiner;\n self.joiner = round_joiner;\n self.line_to(p2, None);\n self.joiner = save_joiner;\n return;\n }\n\n debug_assert_eq!(reduction_type, ReductionType::Quad);\n\n let mut normal_ab = Point::zero();\n let mut unit_ab = Point::zero();\n let mut normal_bc = Point::zero();\n let mut unit_bc = Point::zero();\n if !self.pre_join_to(p1, false, &mut normal_ab, &mut unit_ab) {\n self.line_to(p2, None);\n return;\n }\n\n let mut quad_points = QuadConstruct::default();\n self.init_quad(\n StrokeType::Outer,\n NormalizedF32::ZERO,\n NormalizedF32::ONE,\n &mut quad_points,\n );\n self.quad_stroke(&quad, &mut quad_points);\n self.init_quad(\n StrokeType::Inner,\n NormalizedF32::ZERO,\n NormalizedF32::ONE,\n &mut quad_points,\n );\n self.quad_stroke(&quad, &mut quad_points);\n\n let ok = set_normal_unit_normal(\n quad[1],\n quad[2],\n self.res_scale,\n self.radius,\n &mut normal_bc,\n &mut unit_bc,\n );\n if !ok {\n normal_bc = normal_ab;\n unit_bc = unit_ab;\n }\n\n self.post_join_to(p2, normal_bc, unit_bc);\n }\n\n fn cubic_to(&mut self, pt1: Point, pt2: Point, pt3: Point) {\n let cubic = [self.prev_pt, pt1, pt2, pt3];\n let mut reduction = [Point::zero(); 3];\n let mut tangent_pt = Point::zero();\n let reduction_type = check_cubic_linear(&cubic, &mut reduction, Some(&mut tangent_pt));\n if reduction_type == ReductionType::Point {\n // If the stroke consists of a moveTo followed by a degenerate curve, treat it\n // as if it were followed by a zero-length line. Lines without length\n // can have square and round end caps.\n self.line_to(pt3, None);\n return;\n }\n\n if reduction_type == ReductionType::Line {\n self.line_to(pt3, None);\n return;\n }\n\n if ReductionType::Degenerate <= reduction_type\n && ReductionType::Degenerate3 >= reduction_type\n {\n self.line_to(reduction[0], None);\n let save_joiner = self.joiner;\n self.joiner = round_joiner;\n if ReductionType::Degenerate2 <= reduction_type {\n self.line_to(reduction[1], None);\n }\n\n if ReductionType::Degenerate3 == reduction_type {\n self.line_to(reduction[2], None);\n }\n\n self.line_to(pt3, None);\n self.joiner = save_joiner;\n return;\n }\n\n debug_assert_eq!(reduction_type, ReductionType::Quad);\n let mut normal_ab = Point::zero();\n let mut unit_ab = Point::zero();\n let mut normal_cd = Point::zero();\n let mut unit_cd = Point::zero();\n if !self.pre_join_to(tangent_pt, false, &mut normal_ab, &mut unit_ab) {\n self.line_to(pt3, None);\n return;\n }\n\n let mut t_values = path_geometry::new_t_values();\n let t_values = path_geometry::find_cubic_inflections(&cubic, &mut t_values);\n let mut last_t = NormalizedF32::ZERO;\n for index in 0..=t_values.len() {\n let next_t = t_values\n .get(index)\n .cloned()\n .map(|n| n.to_normalized())\n .unwrap_or(NormalizedF32::ONE);\n\n let mut quad_points = QuadConstruct::default();\n self.init_quad(StrokeType::Outer, last_t, next_t, &mut quad_points);\n self.cubic_stroke(&cubic, &mut quad_points);\n self.init_quad(StrokeType::Inner, last_t, next_t, &mut quad_points);\n self.cubic_stroke(&cubic, &mut quad_points);\n last_t = next_t;\n }\n\n if let Some(cusp) = path_geometry::find_cubic_cusp(&cubic) {\n let cusp_loc = path_geometry::eval_cubic_pos_at(&cubic, cusp.to_normalized());\n self.cusper.push_circle(cusp_loc.x, cusp_loc.y, self.radius);\n }\n\n // emit the join even if one stroke succeeded but the last one failed\n // this avoids reversing an inner stroke with a partial path followed by another moveto\n self.set_cubic_end_normal(&cubic, normal_ab, unit_ab, &mut normal_cd, &mut unit_cd);\n\n self.post_join_to(pt3, normal_cd, unit_cd);\n }\n\n fn cubic_stroke(&mut self, cubic: &[Point; 4], quad_points: &mut QuadConstruct) -> bool {\n if !self.found_tangents {\n let result_type = self.tangents_meet(cubic, quad_points);\n if result_type != ResultType::Quad {\n let ok = points_within_dist(\n quad_points.quad[0],\n quad_points.quad[2],\n self.inv_res_scale,\n );\n if (result_type == ResultType::Degenerate || ok)\n && self.cubic_mid_on_line(cubic, quad_points)\n {\n self.add_degenerate_line(quad_points);\n return true;\n }\n } else {\n self.found_tangents = true;\n }\n }\n\n if self.found_tangents {\n let result_type = self.compare_quad_cubic(cubic, quad_points);\n if result_type == ResultType::Quad {\n let stroke = &quad_points.quad;\n if self.stroke_type == StrokeType::Outer {\n self.outer\n .quad_to(stroke[1].x, stroke[1].y, stroke[2].x, stroke[2].y);\n } else {\n self.inner\n .quad_to(stroke[1].x, stroke[1].y, stroke[2].x, stroke[2].y);\n }\n\n return true;\n }\n\n if result_type == ResultType::Degenerate {\n if !quad_points.opposite_tangents {\n self.add_degenerate_line(quad_points);\n return true;\n }\n }\n }\n\n if !quad_points.quad[2].x.is_finite() || !quad_points.quad[2].x.is_finite() {\n return false; // just abort if projected quad isn't representable\n }\n\n self.recursion_depth += 1;\n if self.recursion_depth > RECURSIVE_LIMITS[self.found_tangents as usize] {\n return false; // just abort if projected quad isn't representable\n }\n\n let mut half = QuadConstruct::default();\n if !half.init_with_start(quad_points) {\n self.add_degenerate_line(quad_points);\n self.recursion_depth -= 1;\n return true;\n }\n\n if !self.cubic_stroke(cubic, &mut half) {\n return false;\n }\n\n if !half.init_with_end(quad_points) {\n self.add_degenerate_line(quad_points);\n self.recursion_depth -= 1;\n return true;\n }\n\n if !self.cubic_stroke(cubic, &mut half) {\n return false;\n }\n\n self.recursion_depth -= 1;\n true\n }\n\n fn cubic_mid_on_line(&self, cubic: &[Point; 4], quad_points: &mut QuadConstruct) -> bool {\n let mut stroke_mid = Point::zero();\n self.cubic_quad_mid(cubic, quad_points, &mut stroke_mid);\n let dist = pt_to_line(stroke_mid, quad_points.quad[0], quad_points.quad[2]);\n dist < self.inv_res_scale_squared\n }\n\n fn cubic_quad_mid(&self, cubic: &[Point; 4], quad_points: &mut QuadConstruct, mid: &mut Point) {\n let mut cubic_mid_pt = Point::zero();\n self.cubic_perp_ray(cubic, quad_points.mid_t, &mut cubic_mid_pt, mid, None);\n }\n\n // Given a cubic and t, return the point on curve,\n // its perpendicular, and the perpendicular tangent.\n fn cubic_perp_ray(\n &self,\n cubic: &[Point; 4],\n t: NormalizedF32,\n t_pt: &mut Point,\n on_pt: &mut Point,\n tangent: Option<&mut Point>,\n ) {\n *t_pt = path_geometry::eval_cubic_pos_at(cubic, t);\n let mut dxy = path_geometry::eval_cubic_tangent_at(cubic, t);\n\n let mut chopped = [Point::zero(); 7];\n if dxy.x == 0.0 && dxy.y == 0.0 {\n let mut c_points: &[Point] = cubic;\n if t.get().is_nearly_zero() {\n dxy = cubic[2] - cubic[0];\n } else if (1.0 - t.get()).is_nearly_zero() {\n dxy = cubic[3] - cubic[1];\n } else {\n // If the cubic inflection falls on the cusp, subdivide the cubic\n // to find the tangent at that point.\n //\n // Unwrap never fails, because we already checked that `t` is not 0/1,\n let t = NormalizedF32Exclusive::new(t.get()).unwrap();\n path_geometry::chop_cubic_at2(cubic, t, &mut chopped);\n dxy = chopped[3] - chopped[2];\n if dxy.x == 0.0 && dxy.y == 0.0 {\n dxy = chopped[3] - chopped[1];\n c_points = &chopped;\n }\n }\n\n if dxy.x == 0.0 && dxy.y == 0.0 {\n dxy = c_points[3] - c_points[0];\n }\n }\n\n self.set_ray_points(*t_pt, &mut dxy, on_pt, tangent);\n }\n\n fn set_cubic_end_normal(\n &mut self,\n cubic: &[Point; 4],\n normal_ab: Point,\n unit_normal_ab: Point,\n normal_cd: &mut Point,\n unit_normal_cd: &mut Point,\n ) {\n let mut ab = cubic[1] - cubic[0];\n let mut cd = cubic[3] - cubic[2];\n\n let mut degenerate_ab = degenerate_vector(ab);\n let mut degenerate_cb = degenerate_vector(cd);\n\n if degenerate_ab && degenerate_cb {\n *normal_cd = normal_ab;\n *unit_normal_cd = unit_normal_ab;\n return;\n }\n\n if degenerate_ab {\n ab = cubic[2] - cubic[0];\n degenerate_ab = degenerate_vector(ab);\n }\n\n if degenerate_cb {\n cd = cubic[3] - cubic[1];\n degenerate_cb = degenerate_vector(cd);\n }\n\n if degenerate_ab || degenerate_cb {\n *normal_cd = normal_ab;\n *unit_normal_cd = unit_normal_ab;\n return;\n }\n\n let res = set_normal_unit_normal2(cd, self.radius, normal_cd, unit_normal_cd);\n debug_assert!(res);\n }\n\n fn compare_quad_cubic(\n &self,\n cubic: &[Point; 4],\n quad_points: &mut QuadConstruct,\n ) -> ResultType {\n // get the quadratic approximation of the stroke\n self.cubic_quad_ends(cubic, quad_points);\n let result_type = self.intersect_ray(IntersectRayType::CtrlPt, quad_points);\n if result_type != ResultType::Quad {\n return result_type;\n }\n\n // project a ray from the curve to the stroke\n // points near midpoint on quad, midpoint on cubic\n let mut ray0 = Point::zero();\n let mut ray1 = Point::zero();\n self.cubic_perp_ray(cubic, quad_points.mid_t, &mut ray1, &mut ray0, None);\n self.stroke_close_enough(&quad_points.quad.clone(), &[ray0, ray1], quad_points)\n }\n\n // Given a cubic and a t range, find the start and end if they haven't been found already.\n fn cubic_quad_ends(&self, cubic: &[Point; 4], quad_points: &mut QuadConstruct) {\n if !quad_points.start_set {\n let mut cubic_start_pt = Point::zero();\n self.cubic_perp_ray(\n cubic,\n quad_points.start_t,\n &mut cubic_start_pt,\n &mut quad_points.quad[0],\n Some(&mut quad_points.tangent_start),\n );\n quad_points.start_set = true;\n }\n\n if !quad_points.end_set {\n let mut cubic_end_pt = Point::zero();\n self.cubic_perp_ray(\n cubic,\n quad_points.end_t,\n &mut cubic_end_pt,\n &mut quad_points.quad[2],\n Some(&mut quad_points.tangent_end),\n );\n quad_points.end_set = true;\n }\n }\n\n fn close(&mut self, is_line: bool) {\n self.finish_contour(true, is_line);\n }\n\n fn finish_contour(&mut self, close: bool, curr_is_line: bool) {\n if self.segment_count > 0 {\n if close {\n (self.joiner)(\n self.prev_unit_normal,\n self.prev_pt,\n self.first_unit_normal,\n self.radius,\n self.inv_miter_limit,\n self.prev_is_line,\n curr_is_line,\n self.builders(),\n );\n self.outer.close();\n\n // now add inner as its own contour\n let pt = self.inner.last_point().unwrap_or_default();\n self.outer.move_to(pt.x, pt.y);\n self.outer.reverse_path_to(&self.inner);\n self.outer.close();\n } else {\n // add caps to start and end\n\n // cap the end\n let pt = self.inner.last_point().unwrap_or_default();\n let other_path = if curr_is_line {\n Some(&self.inner)\n } else {\n None\n };\n (self.capper)(\n self.prev_pt,\n self.prev_normal,\n pt,\n other_path,\n &mut self.outer,\n );\n self.outer.reverse_path_to(&self.inner);\n\n // cap the start\n let other_path = if self.prev_is_line {\n Some(&self.inner)\n } else {\n None\n };\n (self.capper)(\n self.first_pt,\n -self.first_normal,\n self.first_outer_pt,\n other_path,\n &mut self.outer,\n );\n self.outer.close();\n }\n\n if !self.cusper.is_empty() {\n self.outer.push_path_builder(&self.cusper);\n self.cusper.clear();\n }\n }\n\n // since we may re-use `inner`, we rewind instead of reset, to save on\n // reallocating its internal storage.\n self.inner.clear();\n self.segment_count = -1;\n self.first_outer_pt_index_in_contour = self.outer.points.len();\n }\n\n fn pre_join_to(\n &mut self,\n p: Point,\n curr_is_line: bool,\n normal: &mut Point,\n unit_normal: &mut Point,\n ) -> bool {\n debug_assert!(self.segment_count >= 0);\n\n let prev_x = self.prev_pt.x;\n let prev_y = self.prev_pt.y;\n\n let normal_set = set_normal_unit_normal(\n self.prev_pt,\n p,\n self.res_scale,\n self.radius,\n normal,\n unit_normal,\n );\n if !normal_set {\n if fn_ptr_eq(self.capper, butt_capper) {\n return false;\n }\n\n // Square caps and round caps draw even if the segment length is zero.\n // Since the zero length segment has no direction, set the orientation\n // to upright as the default orientation.\n *normal = Point::from_xy(self.radius, 0.0);\n *unit_normal = Point::from_xy(1.0, 0.0);\n }\n\n if self.segment_count == 0 {\n self.first_normal = *normal;\n self.first_unit_normal = *unit_normal;\n self.first_outer_pt = Point::from_xy(prev_x + normal.x, prev_y + normal.y);\n\n self.outer\n .move_to(self.first_outer_pt.x, self.first_outer_pt.y);\n self.inner.move_to(prev_x - normal.x, prev_y - normal.y);\n } else {\n // we have a previous segment\n (self.joiner)(\n self.prev_unit_normal,\n self.prev_pt,\n *unit_normal,\n self.radius,\n self.inv_miter_limit,\n self.prev_is_line,\n curr_is_line,\n self.builders(),\n );\n }\n self.prev_is_line = curr_is_line;\n true\n }\n\n fn post_join_to(&mut self, p: Point, normal: Point, unit_normal: Point) {\n self.join_completed = true;\n self.prev_pt = p;\n self.prev_unit_normal = unit_normal;\n self.prev_normal = normal;\n self.segment_count += 1;\n }\n\n fn init_quad(\n &mut self,\n stroke_type: StrokeType,\n start: NormalizedF32,\n end: NormalizedF32,\n quad_points: &mut QuadConstruct,\n ) {\n self.stroke_type = stroke_type;\n self.found_tangents = false;\n quad_points.init(start, end);\n }\n\n fn quad_stroke(&mut self, quad: &[Point; 3], quad_points: &mut QuadConstruct) -> bool {\n let result_type = self.compare_quad_quad(quad, quad_points);\n if result_type == ResultType::Quad {\n let path = if self.stroke_type == StrokeType::Outer {\n &mut self.outer\n } else {\n &mut self.inner\n };\n\n path.quad_to(\n quad_points.quad[1].x,\n quad_points.quad[1].y,\n quad_points.quad[2].x,\n quad_points.quad[2].y,\n );\n\n return true;\n }\n\n if result_type == ResultType::Degenerate {\n self.add_degenerate_line(quad_points);\n return true;\n }\n\n self.recursion_depth += 1;\n if self.recursion_depth > RECURSIVE_LIMITS[QUAD_RECURSIVE_LIMIT] {\n return false; // just abort if projected quad isn't representable\n }\n\n let mut half = QuadConstruct::default();\n half.init_with_start(quad_points);\n if !self.quad_stroke(quad, &mut half) {\n return false;\n }\n\n half.init_with_end(quad_points);\n if !self.quad_stroke(quad, &mut half) {\n return false;\n }\n\n self.recursion_depth -= 1;\n true\n }\n\n fn compare_quad_quad(\n &mut self,\n quad: &[Point; 3],\n quad_points: &mut QuadConstruct,\n ) -> ResultType {\n // get the quadratic approximation of the stroke\n if !quad_points.start_set {\n let mut quad_start_pt = Point::zero();\n self.quad_perp_ray(\n quad,\n quad_points.start_t,\n &mut quad_start_pt,\n &mut quad_points.quad[0],\n Some(&mut quad_points.tangent_start),\n );\n quad_points.start_set = true;\n }\n\n if !quad_points.end_set {\n let mut quad_end_pt = Point::zero();\n self.quad_perp_ray(\n quad,\n quad_points.end_t,\n &mut quad_end_pt,\n &mut quad_points.quad[2],\n Some(&mut quad_points.tangent_end),\n );\n quad_points.end_set = true;\n }\n\n let result_type = self.intersect_ray(IntersectRayType::CtrlPt, quad_points);\n if result_type != ResultType::Quad {\n return result_type;\n }\n\n // project a ray from the curve to the stroke\n let mut ray0 = Point::zero();\n let mut ray1 = Point::zero();\n self.quad_perp_ray(quad, quad_points.mid_t, &mut ray1, &mut ray0, None);\n self.stroke_close_enough(&quad_points.quad.clone(), &[ray0, ray1], quad_points)\n }\n\n // Given a point on the curve and its derivative, scale the derivative by the radius, and\n // compute the perpendicular point and its tangent.\n fn set_ray_points(\n &self,\n tp: Point,\n dxy: &mut Point,\n on_p: &mut Point,\n mut tangent: Option<&mut Point>,\n ) {\n if !dxy.set_length(self.radius) {\n *dxy = Point::from_xy(self.radius, 0.0);\n }\n\n let axis_flip = self.stroke_type as i32 as f32; // go opposite ways for outer, inner\n on_p.x = tp.x + axis_flip * dxy.y;\n on_p.y = tp.y - axis_flip * dxy.x;\n\n if let Some(ref mut tangent) = tangent {\n tangent.x = on_p.x + dxy.x;\n tangent.y = on_p.y + dxy.y;\n }\n }\n\n // Given a quad and t, return the point on curve,\n // its perpendicular, and the perpendicular tangent.\n fn quad_perp_ray(\n &self,\n quad: &[Point; 3],\n t: NormalizedF32,\n tp: &mut Point,\n on_p: &mut Point,\n tangent: Option<&mut Point>,\n ) {\n *tp = path_geometry::eval_quad_at(quad, t);\n let mut dxy = path_geometry::eval_quad_tangent_at(quad, t);\n\n if dxy.is_zero() {\n dxy = quad[2] - quad[0];\n }\n\n self.set_ray_points(*tp, &mut dxy, on_p, tangent);\n }\n\n fn add_degenerate_line(&mut self, quad_points: &QuadConstruct) {\n if self.stroke_type == StrokeType::Outer {\n self.outer\n .line_to(quad_points.quad[2].x, quad_points.quad[2].y);\n } else {\n self.inner\n .line_to(quad_points.quad[2].x, quad_points.quad[2].y);\n }\n }\n\n fn stroke_close_enough(\n &self,\n stroke: &[Point; 3],\n ray: &[Point; 2],\n quad_points: &mut QuadConstruct,\n ) -> ResultType {\n let half = NormalizedF32::new_clamped(0.5);\n let stroke_mid = path_geometry::eval_quad_at(stroke, half);\n // measure the distance from the curve to the quad-stroke midpoint, compare to radius\n if points_within_dist(ray[0], stroke_mid, self.inv_res_scale) {\n // if the difference is small\n if sharp_angle(&quad_points.quad) {\n return ResultType::Split;\n }\n\n return ResultType::Quad;\n }\n\n // measure the distance to quad's bounds (quick reject)\n // an alternative : look for point in triangle\n if !pt_in_quad_bounds(stroke, ray[0], self.inv_res_scale) {\n // if far, subdivide\n return ResultType::Split;\n }\n\n // measure the curve ray distance to the quad-stroke\n let mut roots = path_geometry::new_t_values();\n let roots = intersect_quad_ray(ray, stroke, &mut roots);\n if roots.len() != 1 {\n return ResultType::Split;\n }\n\n let quad_pt = path_geometry::eval_quad_at(stroke, roots[0].to_normalized());\n let error = self.inv_res_scale * (1.0 - (roots[0].get() - 0.5).abs() * 2.0);\n if points_within_dist(ray[0], quad_pt, error) {\n // if the difference is small, we're done\n if sharp_angle(&quad_points.quad) {\n return ResultType::Split;\n }\n\n return ResultType::Quad;\n }\n\n // otherwise, subdivide\n ResultType::Split\n }\n\n // Find the intersection of the stroke tangents to construct a stroke quad.\n // Return whether the stroke is a degenerate (a line), a quad, or must be split.\n // Optionally compute the quad's control point.\n fn intersect_ray(\n &self,\n intersect_ray_type: IntersectRayType,\n quad_points: &mut QuadConstruct,\n ) -> ResultType {\n let start = quad_points.quad[0];\n let end = quad_points.quad[2];\n let a_len = quad_points.tangent_start - start;\n let b_len = quad_points.tangent_end - end;\n\n // Slopes match when denom goes to zero:\n // axLen / ayLen == bxLen / byLen\n // (ayLen * byLen) * axLen / ayLen == (ayLen * byLen) * bxLen / byLen\n // byLen * axLen == ayLen * bxLen\n // byLen * axLen - ayLen * bxLen ( == denom )\n let denom = a_len.cross(b_len);\n if denom == 0.0 || !denom.is_finite() {\n quad_points.opposite_tangents = a_len.dot(b_len) < 0.0;\n return ResultType::Degenerate;\n }\n\n quad_points.opposite_tangents = false;\n let ab0 = start - end;\n let mut numer_a = b_len.cross(ab0);\n let numer_b = a_len.cross(ab0);\n if (numer_a >= 0.0) == (numer_b >= 0.0) {\n // if the control point is outside the quad ends\n\n // if the perpendicular distances from the quad points to the opposite tangent line\n // are small, a straight line is good enough\n let dist1 = pt_to_line(start, end, quad_points.tangent_end);\n let dist2 = pt_to_line(end, start, quad_points.tangent_start);\n if dist1.max(dist2) <= self.inv_res_scale_squared {\n return ResultType::Degenerate;\n }\n\n return ResultType::Split;\n }\n\n // check to see if the denominator is teeny relative to the numerator\n // if the offset by one will be lost, the ratio is too large\n numer_a /= denom;\n let valid_divide = numer_a > numer_a - 1.0;\n if valid_divide {\n if intersect_ray_type == IntersectRayType::CtrlPt {\n // the intersection of the tangents need not be on the tangent segment\n // so 0 <= numerA <= 1 is not necessarily true\n quad_points.quad[1].x =\n start.x * (1.0 - numer_a) + quad_points.tangent_start.x * numer_a;\n quad_points.quad[1].y =\n start.y * (1.0 - numer_a) + quad_points.tangent_start.y * numer_a;\n }\n\n return ResultType::Quad;\n }\n\n quad_points.opposite_tangents = a_len.dot(b_len) < 0.0;\n\n // if the lines are parallel, straight line is good enough\n ResultType::Degenerate\n }\n\n // Given a cubic and a t-range, determine if the stroke can be described by a quadratic.\n fn tangents_meet(&self, cubic: &[Point; 4], quad_points: &mut QuadConstruct) -> ResultType {\n self.cubic_quad_ends(cubic, quad_points);\n self.intersect_ray(IntersectRayType::ResultType, quad_points)\n }\n\n fn finish(&mut self, is_line: bool) -> Option {\n self.finish_contour(false, is_line);\n\n // Swap out the outer builder.\n let mut buf = PathBuilder::new();\n core::mem::swap(&mut self.outer, &mut buf);\n\n buf.finish()\n }\n\n fn has_only_move_to(&self) -> bool {\n self.segment_count == 0\n }\n\n fn is_current_contour_empty(&self) -> bool {\n self.inner.is_zero_length_since_point(0)\n && self\n .outer\n .is_zero_length_since_point(self.first_outer_pt_index_in_contour)\n }\n}\n\nfn cap_factory(cap: LineCap) -> CapProc {\n match cap {\n LineCap::Butt => butt_capper,\n LineCap::Round => round_capper,\n LineCap::Square => square_capper,\n }\n}\n\nfn butt_capper(_: Point, _: Point, stop: Point, _: Option<&PathBuilder>, path: &mut PathBuilder) {\n path.line_to(stop.x, stop.y);\n}\n\nfn round_capper(\n pivot: Point,\n normal: Point,\n stop: Point,\n _: Option<&PathBuilder>,\n path: &mut PathBuilder,\n) {\n let mut parallel = normal;\n parallel.rotate_cw();\n\n let projected_center = pivot + parallel;\n\n path.conic_points_to(\n projected_center + normal,\n projected_center,\n SCALAR_ROOT_2_OVER_2,\n );\n path.conic_points_to(projected_center - normal, stop, SCALAR_ROOT_2_OVER_2);\n}\n\nfn square_capper(\n pivot: Point,\n normal: Point,\n stop: Point,\n other_path: Option<&PathBuilder>,\n path: &mut PathBuilder,\n) {\n let mut parallel = normal;\n parallel.rotate_cw();\n\n if other_path.is_some() {\n path.set_last_point(Point::from_xy(\n pivot.x + normal.x + parallel.x,\n pivot.y + normal.y + parallel.y,\n ));\n path.line_to(\n pivot.x - normal.x + parallel.x,\n pivot.y - normal.y + parallel.y,\n );\n } else {\n path.line_to(\n pivot.x + normal.x + parallel.x,\n pivot.y + normal.y + parallel.y,\n );\n path.line_to(\n pivot.x - normal.x + parallel.x,\n pivot.y - normal.y + parallel.y,\n );\n path.line_to(stop.x, stop.y);\n }\n}\n\nfn join_factory(join: LineJoin) -> JoinProc {\n match join {\n LineJoin::Miter => miter_joiner,\n LineJoin::Round => round_joiner,\n LineJoin::Bevel => bevel_joiner,\n }\n}\n\nfn is_clockwise(before: Point, after: Point) -> bool {\n before.x * after.y > before.y * after.x\n}\n\n#[derive(Copy, Clone, PartialEq, Debug)]\nenum AngleType {\n Nearly180,\n Sharp,\n Shallow,\n NearlyLine,\n}\n\nfn dot_to_angle_type(dot: f32) -> AngleType {\n if dot >= 0.0 {\n // shallow or line\n if (1.0 - dot).is_nearly_zero() {\n AngleType::NearlyLine\n } else {\n AngleType::Shallow\n }\n } else {\n // sharp or 180\n if (1.0 + dot).is_nearly_zero() {\n AngleType::Nearly180\n } else {\n AngleType::Sharp\n }\n }\n}\n\nfn handle_inner_join(pivot: Point, after: Point, inner: &mut PathBuilder) {\n // In the degenerate case that the stroke radius is larger than our segments\n // just connecting the two inner segments may \"show through\" as a funny\n // diagonal. To pseudo-fix this, we go through the pivot point. This adds\n // an extra point/edge, but I can't see a cheap way to know when this is\n // not needed :(\n inner.line_to(pivot.x, pivot.y);\n\n inner.line_to(pivot.x - after.x, pivot.y - after.y);\n}\n\nfn bevel_joiner(\n before_unit_normal: Point,\n pivot: Point,\n after_unit_normal: Point,\n radius: f32,\n _: f32,\n _: bool,\n _: bool,\n mut builders: SwappableBuilders,\n) {\n let mut after = after_unit_normal.scaled(radius);\n\n if !is_clockwise(before_unit_normal, after_unit_normal) {\n builders.swap();\n after = -after;\n }\n\n builders.outer.line_to(pivot.x + after.x, pivot.y + after.y);\n handle_inner_join(pivot, after, builders.inner);\n}\n\nfn round_joiner(\n before_unit_normal: Point,\n pivot: Point,\n after_unit_normal: Point,\n radius: f32,\n _: f32,\n _: bool,\n _: bool,\n mut builders: SwappableBuilders,\n) {\n let dot_prod = before_unit_normal.dot(after_unit_normal);\n let angle_type = dot_to_angle_type(dot_prod);\n\n if angle_type == AngleType::NearlyLine {\n return;\n }\n\n let mut before = before_unit_normal;\n let mut after = after_unit_normal;\n let mut dir = PathDirection::CW;\n\n if !is_clockwise(before, after) {\n builders.swap();\n before = -before;\n after = -after;\n dir = PathDirection::CCW;\n }\n\n let ts = Transform::from_row(radius, 0.0, 0.0, radius, pivot.x, pivot.y);\n\n let mut conics = [path_geometry::Conic::default(); 5];\n let conics = path_geometry::Conic::build_unit_arc(before, after, dir, ts, &mut conics);\n if let Some(conics) = conics {\n for conic in conics {\n builders\n .outer\n .conic_points_to(conic.points[1], conic.points[2], conic.weight);\n }\n\n after.scale(radius);\n handle_inner_join(pivot, after, builders.inner);\n }\n}\n\nfn miter_joiner(\n before_unit_normal: Point,\n pivot: Point,\n after_unit_normal: Point,\n radius: f32,\n inv_miter_limit: f32,\n prev_is_line: bool,\n mut curr_is_line: bool,\n mut builders: SwappableBuilders,\n) {\n fn do_blunt(\n builders: SwappableBuilders,\n pivot: Point,\n radius: f32,\n curr_is_line: bool,\n mut after: Point,\n ) {\n after.scale(radius);\n if !curr_is_line {\n builders.outer.line_to(pivot.x + after.x, pivot.y + after.y);\n }\n\n handle_inner_join(pivot, after, builders.inner);\n }\n\n fn do_miter(\n builders: SwappableBuilders,\n pivot: Point,\n radius: f32,\n prev_is_line: bool,\n curr_is_line: bool,\n mid: Point,\n after: Point,\n ) {\n if prev_is_line {\n builders\n .outer\n .set_last_point(Point::from_xy(pivot.x + mid.x, pivot.y + mid.y));\n } else {\n builders.outer.line_to(pivot.x + mid.x, pivot.y + mid.y);\n }\n\n do_blunt(builders, pivot, radius, curr_is_line, after);\n }\n\n // negate the dot since we're using normals instead of tangents\n let dot_prod = before_unit_normal.dot(after_unit_normal);\n let angle_type = dot_to_angle_type(dot_prod);\n let mut before = before_unit_normal;\n let mut after = after_unit_normal;\n let mut mid;\n\n if angle_type == AngleType::NearlyLine {\n return;\n }\n\n if angle_type == AngleType::Nearly180 {\n curr_is_line = false;\n do_blunt(builders, pivot, radius, curr_is_line, after);\n return;\n }\n\n let ccw = !is_clockwise(before, after);\n if ccw {\n builders.swap();\n before = -before;\n after = -after;\n }\n\n // Before we enter the world of square-roots and divides,\n // check if we're trying to join an upright right angle\n // (common case for stroking rectangles). If so, special case\n // that (for speed an accuracy).\n // Note: we only need to check one normal if dot==0\n if dot_prod == 0.0 && inv_miter_limit <= SCALAR_ROOT_2_OVER_2 {\n mid = (before + after).scaled(radius);\n do_miter(\n builders,\n pivot,\n radius,\n prev_is_line,\n curr_is_line,\n mid,\n after,\n );\n return;\n }\n\n // midLength = radius / sinHalfAngle\n // if (midLength > miterLimit * radius) abort\n // if (radius / sinHalf > miterLimit * radius) abort\n // if (1 / sinHalf > miterLimit) abort\n // if (1 / miterLimit > sinHalf) abort\n // My dotProd is opposite sign, since it is built from normals and not tangents\n // hence 1 + dot instead of 1 - dot in the formula\n let sin_half_angle = (1.0 + dot_prod).half().sqrt();\n if sin_half_angle < inv_miter_limit {\n curr_is_line = false;\n do_blunt(builders, pivot, radius, curr_is_line, after);\n return;\n }\n\n // choose the most accurate way to form the initial mid-vector\n if angle_type == AngleType::Sharp {\n mid = Point::from_xy(after.y - before.y, before.x - after.x);\n if ccw {\n mid = -mid;\n }\n } else {\n mid = Point::from_xy(before.x + after.x, before.y + after.y);\n }\n\n mid.set_length(radius / sin_half_angle);\n do_miter(\n builders,\n pivot,\n radius,\n prev_is_line,\n curr_is_line,\n mid,\n after,\n );\n}\n\nfn set_normal_unit_normal(\n before: Point,\n after: Point,\n scale: f32,\n radius: f32,\n normal: &mut Point,\n unit_normal: &mut Point,\n) -> bool {\n if !unit_normal.set_normalize((after.x - before.x) * scale, (after.y - before.y) * scale) {\n return false;\n }\n\n unit_normal.rotate_ccw();\n *normal = unit_normal.scaled(radius);\n true\n}\n\nfn set_normal_unit_normal2(\n vec: Point,\n radius: f32,\n normal: &mut Point,\n unit_normal: &mut Point,\n) -> bool {\n if !unit_normal.set_normalize(vec.x, vec.y) {\n return false;\n }\n\n unit_normal.rotate_ccw();\n *normal = unit_normal.scaled(radius);\n true\n}\n\nfn fn_ptr_eq(f1: CapProc, f2: CapProc) -> bool {\n core::ptr::eq(f1 as *const (), f2 as *const ())\n}\n\n#[derive(Debug)]\nstruct QuadConstruct {\n // The state of the quad stroke under construction.\n quad: [Point; 3], // the stroked quad parallel to the original curve\n tangent_start: Point, // a point tangent to quad[0]\n tangent_end: Point, // a point tangent to quad[2]\n start_t: NormalizedF32, // a segment of the original curve\n mid_t: NormalizedF32,\n end_t: NormalizedF32,\n start_set: bool, // state to share common points across structs\n end_set: bool,\n opposite_tangents: bool, // set if coincident tangents have opposite directions\n}\n\nimpl Default for QuadConstruct {\n fn default() -> Self {\n Self {\n quad: Default::default(),\n tangent_start: Point::default(),\n tangent_end: Point::default(),\n start_t: NormalizedF32::ZERO,\n mid_t: NormalizedF32::ZERO,\n end_t: NormalizedF32::ZERO,\n start_set: false,\n end_set: false,\n opposite_tangents: false,\n }\n }\n}\n\nimpl QuadConstruct {\n // return false if start and end are too close to have a unique middle\n fn init(&mut self, start: NormalizedF32, end: NormalizedF32) -> bool {\n self.start_t = start;\n self.mid_t = NormalizedF32::new_clamped((start.get() + end.get()).half());\n self.end_t = end;\n self.start_set = false;\n self.end_set = false;\n self.start_t < self.mid_t && self.mid_t < self.end_t\n }\n\n fn init_with_start(&mut self, parent: &Self) -> bool {\n if !self.init(parent.start_t, parent.mid_t) {\n return false;\n }\n\n self.quad[0] = parent.quad[0];\n self.tangent_start = parent.tangent_start;\n self.start_set = true;\n true\n }\n\n fn init_with_end(&mut self, parent: &Self) -> bool {\n if !self.init(parent.mid_t, parent.end_t) {\n return false;\n }\n\n self.quad[2] = parent.quad[2];\n self.tangent_end = parent.tangent_end;\n self.end_set = true;\n true\n }\n}\n\nfn check_quad_linear(quad: &[Point; 3]) -> (Point, ReductionType) {\n let degenerate_ab = degenerate_vector(quad[1] - quad[0]);\n let degenerate_bc = degenerate_vector(quad[2] - quad[1]);\n if degenerate_ab & degenerate_bc {\n return (Point::zero(), ReductionType::Point);\n }\n\n if degenerate_ab | degenerate_bc {\n return (Point::zero(), ReductionType::Line);\n }\n\n if !quad_in_line(quad) {\n return (Point::zero(), ReductionType::Quad);\n }\n\n let t = path_geometry::find_quad_max_curvature(quad);\n if t == NormalizedF32::ZERO || t == NormalizedF32::ONE {\n return (Point::zero(), ReductionType::Line);\n }\n\n (\n path_geometry::eval_quad_at(quad, t),\n ReductionType::Degenerate,\n )\n}\n\nfn degenerate_vector(v: Point) -> bool {\n !v.can_normalize()\n}\n\n/// Given quad, see if all there points are in a line.\n/// Return true if the inside point is close to a line connecting the outermost points.\n///\n/// Find the outermost point by looking for the largest difference in X or Y.\n/// Since the XOR of the indices is 3 (0 ^ 1 ^ 2)\n/// the missing index equals: outer_1 ^ outer_2 ^ 3.\nfn quad_in_line(quad: &[Point; 3]) -> bool {\n let mut pt_max = -1.0;\n let mut outer1 = 0;\n let mut outer2 = 0;\n for index in 0..2 {\n for inner in index + 1..3 {\n let test_diff = quad[inner] - quad[index];\n let test_max = test_diff.x.abs().max(test_diff.y.abs());\n if pt_max < test_max {\n outer1 = index;\n outer2 = inner;\n pt_max = test_max;\n }\n }\n }\n\n debug_assert!(outer1 <= 1);\n debug_assert!(outer2 >= 1 && outer2 <= 2);\n debug_assert!(outer1 < outer2);\n\n let mid = outer1 ^ outer2 ^ 3;\n const CURVATURE_SLOP: f32 = 0.000005; // this multiplier is pulled out of the air\n let line_slop = pt_max * pt_max * CURVATURE_SLOP;\n pt_to_line(quad[mid], quad[outer1], quad[outer2]) <= line_slop\n}\n\n// returns the distance squared from the point to the line\nfn pt_to_line(pt: Point, line_start: Point, line_end: Point) -> f32 {\n let dxy = line_end - line_start;\n let ab0 = pt - line_start;\n let numer = dxy.dot(ab0);\n let denom = dxy.dot(dxy);\n let t = numer / denom;\n if t >= 0.0 && t <= 1.0 {\n let hit = Point::from_xy(\n line_start.x * (1.0 - t) + line_end.x * t,\n line_start.y * (1.0 - t) + line_end.y * t,\n );\n hit.distance_to_sqd(pt)\n } else {\n pt.distance_to_sqd(line_start)\n }\n}\n\n// Intersect the line with the quad and return the t values on the quad where the line crosses.\nfn intersect_quad_ray<'a>(\n line: &[Point; 2],\n quad: &[Point; 3],\n roots: &'a mut [NormalizedF32Exclusive; 3],\n) -> &'a [NormalizedF32Exclusive] {\n let vec = line[1] - line[0];\n let mut r = [0.0; 3];\n for n in 0..3 {\n r[n] = (quad[n].y - line[0].y) * vec.x - (quad[n].x - line[0].x) * vec.y;\n }\n let mut a = r[2];\n let mut b = r[1];\n let c = r[0];\n a += c - 2.0 * b; // A = a - 2*b + c\n b -= c; // B = -(b - c)\n\n let len = path_geometry::find_unit_quad_roots(a, 2.0 * b, c, roots);\n &roots[0..len]\n}\n\nfn points_within_dist(near_pt: Point, far_pt: Point, limit: f32) -> bool {\n near_pt.distance_to_sqd(far_pt) <= limit * limit\n}\n\nfn sharp_angle(quad: &[Point; 3]) -> bool {\n let mut smaller = quad[1] - quad[0];\n let mut larger = quad[1] - quad[2];\n let smaller_len = smaller.length_sqd();\n let mut larger_len = larger.length_sqd();\n if smaller_len > larger_len {\n core::mem::swap(&mut smaller, &mut larger);\n larger_len = smaller_len;\n }\n\n if !smaller.set_length(larger_len) {\n return false;\n }\n\n let dot = smaller.dot(larger);\n dot > 0.0\n}\n\n// Return true if the point is close to the bounds of the quad. This is used as a quick reject.\nfn pt_in_quad_bounds(quad: &[Point; 3], pt: Point, inv_res_scale: f32) -> bool {\n let x_min = quad[0].x.min(quad[1].x).min(quad[2].x);\n if pt.x + inv_res_scale < x_min {\n return false;\n }\n\n let x_max = quad[0].x.max(quad[1].x).max(quad[2].x);\n if pt.x - inv_res_scale > x_max {\n return false;\n }\n\n let y_min = quad[0].y.min(quad[1].y).min(quad[2].y);\n if pt.y + inv_res_scale < y_min {\n return false;\n }\n\n let y_max = quad[0].y.max(quad[1].y).max(quad[2].y);\n if pt.y - inv_res_scale > y_max {\n return false;\n }\n\n true\n}\n\nfn check_cubic_linear(\n cubic: &[Point; 4],\n reduction: &mut [Point; 3],\n tangent_pt: Option<&mut Point>,\n) -> ReductionType {\n let degenerate_ab = degenerate_vector(cubic[1] - cubic[0]);\n let degenerate_bc = degenerate_vector(cubic[2] - cubic[1]);\n let degenerate_cd = degenerate_vector(cubic[3] - cubic[2]);\n if degenerate_ab & degenerate_bc & degenerate_cd {\n return ReductionType::Point;\n }\n\n if degenerate_ab as i32 + degenerate_bc as i32 + degenerate_cd as i32 == 2 {\n return ReductionType::Line;\n }\n\n if !cubic_in_line(cubic) {\n if let Some(tangent_pt) = tangent_pt {\n *tangent_pt = if degenerate_ab { cubic[2] } else { cubic[1] };\n }\n\n return ReductionType::Quad;\n }\n\n let mut t_values = [NormalizedF32::ZERO; 3];\n let t_values = path_geometry::find_cubic_max_curvature(cubic, &mut t_values);\n let mut r_count = 0;\n // Now loop over the t-values, and reject any that evaluate to either end-point\n for t in t_values {\n if 0.0 >= t.get() || t.get() >= 1.0 {\n continue;\n }\n\n reduction[r_count] = path_geometry::eval_cubic_pos_at(cubic, *t);\n if reduction[r_count] != cubic[0] && reduction[r_count] != cubic[3] {\n r_count += 1;\n }\n }\n\n match r_count {\n 0 => ReductionType::Line,\n 1 => ReductionType::Degenerate,\n 2 => ReductionType::Degenerate2,\n 3 => ReductionType::Degenerate3,\n _ => unreachable!(),\n }\n}\n\n/// Given a cubic, determine if all four points are in a line.\n///\n/// Return true if the inner points is close to a line connecting the outermost points.\n///\n/// Find the outermost point by looking for the largest difference in X or Y.\n/// Given the indices of the outermost points, and that outer_1 is greater than outer_2,\n/// this table shows the index of the smaller of the remaining points:\n///\n/// ```text\n/// outer_2\n/// 0 1 2 3\n/// outer_1 ----------------\n/// 0 | - 2 1 1\n/// 1 | - - 0 0\n/// 2 | - - - 0\n/// 3 | - - - -\n/// ```\n///\n/// If outer_1 == 0 and outer_2 == 1, the smaller of the remaining indices (2 and 3) is 2.\n///\n/// This table can be collapsed to: (1 + (2 >> outer_2)) >> outer_1\n///\n/// Given three indices (outer_1 outer_2 mid_1) from 0..3, the remaining index is:\n///\n/// ```text\n/// mid_2 == (outer_1 ^ outer_2 ^ mid_1)\n/// ```\nfn cubic_in_line(cubic: &[Point; 4]) -> bool {\n let mut pt_max = -1.0;\n let mut outer1 = 0;\n let mut outer2 = 0;\n for index in 0..3 {\n for inner in index + 1..4 {\n let test_diff = cubic[inner] - cubic[index];\n let test_max = test_diff.x.abs().max(test_diff.y.abs());\n if pt_max < test_max {\n outer1 = index;\n outer2 = inner;\n pt_max = test_max;\n }\n }\n }\n debug_assert!(outer1 <= 2);\n debug_assert!(outer2 >= 1 && outer2 <= 3);\n debug_assert!(outer1 < outer2);\n let mid1 = (1 + (2 >> outer2)) >> outer1;\n debug_assert!(mid1 <= 2);\n debug_assert!(outer1 != mid1 && outer2 != mid1);\n let mid2 = outer1 ^ outer2 ^ mid1;\n debug_assert!(mid2 >= 1 && mid2 <= 3);\n debug_assert!(mid2 != outer1 && mid2 != outer2 && mid2 != mid1);\n debug_assert!(((1 << outer1) | (1 << outer2) | (1 << mid1) | (1 << mid2)) == 0x0f);\n let line_slop = pt_max * pt_max * 0.00001; // this multiplier is pulled out of the air\n\n pt_to_line(cubic[mid1], cubic[outer1], cubic[outer2]) <= line_slop\n && pt_to_line(cubic[mid2], cubic[outer1], cubic[outer2]) <= line_slop\n}\n\n#[rustfmt::skip]\n#[cfg(test)]\nmod tests {\n use super::*;\n\n impl PathSegment {\n fn new_move_to(x: f32, y: f32) -> Self {\n PathSegment::MoveTo(Point::from_xy(x, y))\n }\n\n fn new_line_to(x: f32, y: f32) -> Self {\n PathSegment::LineTo(Point::from_xy(x, y))\n }\n\n // fn new_quad_to(x1: f32, y1: f32, x: f32, y: f32) -> Self {\n // PathSegment::QuadTo(Point::from_xy(x1, y1), Point::from_xy(x, y))\n // }\n\n // fn new_cubic_to(x1: f32, y1: f32, x2: f32, y2: f32, x: f32, y: f32) -> Self {\n // PathSegment::CubicTo(Point::from_xy(x1, y1), Point::from_xy(x2, y2), Point::from_xy(x, y))\n // }\n\n fn new_close() -> Self {\n PathSegment::Close\n }\n }\n\n // Make sure that subpath auto-closing is enabled.\n #[test]\n fn auto_close() {\n // A triangle.\n let mut pb = PathBuilder::new();\n pb.move_to(10.0, 10.0);\n pb.line_to(20.0, 50.0);\n pb.line_to(30.0, 10.0);\n pb.close();\n let path = pb.finish().unwrap();\n\n let stroke = Stroke::default();\n let stroke_path = PathStroker::new().stroke(&path, &stroke, 1.0).unwrap();\n\n let mut iter = stroke_path.segments();\n iter.set_auto_close(true);\n\n assert_eq!(iter.next().unwrap(), PathSegment::new_move_to(10.485071, 9.878732));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(20.485071, 49.878731));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(20.0, 50.0));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(19.514929, 49.878731));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(29.514929, 9.878732));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(30.0, 10.0));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(30.0, 10.5));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(10.0, 10.5));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(10.0, 10.0));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(10.485071, 9.878732));\n assert_eq!(iter.next().unwrap(), PathSegment::new_close());\n assert_eq!(iter.next().unwrap(), PathSegment::new_move_to(9.3596115, 9.5));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(30.640388, 9.5));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(20.485071, 50.121269));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(19.514929, 50.121269));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(9.514929, 10.121268));\n assert_eq!(iter.next().unwrap(), PathSegment::new_line_to(9.3596115, 9.5));\n assert_eq!(iter.next().unwrap(), PathSegment::new_close());\n }\n\n // From skia/tests/StrokeTest.cpp\n #[test]\n fn cubic_1() {\n let mut pb = PathBuilder::new();\n pb.move_to(51.0161362, 1511.52478);\n pb.cubic_to(\n 51.0161362, 1511.52478,\n 51.0161362, 1511.52478,\n 51.0161362, 1511.52478,\n );\n let path = pb.finish().unwrap();\n\n let mut stroke = Stroke::default();\n stroke.width = 0.394537568;\n\n assert!(PathStroker::new().stroke(&path, &stroke, 1.0).is_none());\n }\n\n // From skia/tests/StrokeTest.cpp\n #[test]\n fn cubic_2() {\n let mut pb = PathBuilder::new();\n pb.move_to(f32::from_bits(0x424c1086), f32::from_bits(0x44bcf0cb)); // 51.0161362, 1511.52478\n pb.cubic_to(\n f32::from_bits(0x424c107c), f32::from_bits(0x44bcf0cb), // 51.0160980, 1511.52478\n f32::from_bits(0x424c10c2), f32::from_bits(0x44bcf0cb), // 51.0163651, 1511.52478\n f32::from_bits(0x424c1119), f32::from_bits(0x44bcf0ca), // 51.0166969, 1511.52466\n );\n let path = pb.finish().unwrap();\n\n let mut stroke = Stroke::default();\n stroke.width = 0.394537568;\n\n assert!(PathStroker::new().stroke(&path, &stroke, 1.0).is_some());\n }\n\n // From skia/tests/StrokeTest.cpp\n // From skbug.com/6491. The large stroke width can cause numerical instabilities.\n #[test]\n fn big() {\n // Skia uses `kStrokeAndFill_Style` here, but we do not support it.\n\n let mut pb = PathBuilder::new();\n pb.move_to(f32::from_bits(0x46380000), f32::from_bits(0xc6380000)); // 11776, -11776\n pb.line_to(f32::from_bits(0x46a00000), f32::from_bits(0xc6a00000)); // 20480, -20480\n pb.line_to(f32::from_bits(0x468c0000), f32::from_bits(0xc68c0000)); // 17920, -17920\n pb.line_to(f32::from_bits(0x46100000), f32::from_bits(0xc6100000)); // 9216, -9216\n pb.line_to(f32::from_bits(0x46380000), f32::from_bits(0xc6380000)); // 11776, -11776\n pb.close();\n let path = pb.finish().unwrap();\n\n let mut stroke = Stroke::default();\n stroke.width = 1.49679073e+10;\n\n assert!(PathStroker::new().stroke(&path, &stroke, 1.0).is_some());\n }\n\n // From skia/tests/StrokerTest.cpp\n #[test]\n fn quad_stroker_one_off() {\n let mut pb = PathBuilder::new();\n pb.move_to(f32::from_bits(0x43c99223), f32::from_bits(0x42b7417e));\n pb.quad_to(\n f32::from_bits(0x4285d839), f32::from_bits(0x43ed6645),\n f32::from_bits(0x43c941c8), f32::from_bits(0x42b3ace3),\n );\n let path = pb.finish().unwrap();\n\n let mut stroke = Stroke::default();\n stroke.width = 164.683548;\n\n assert!(PathStroker::new().stroke(&path, &stroke, 1.0).is_some());\n }\n\n // From skia/tests/StrokerTest.cpp\n #[test]\n fn cubic_stroker_one_off() {\n let mut pb = PathBuilder::new();\n pb.move_to(f32::from_bits(0x433f5370), f32::from_bits(0x43d1f4b3));\n pb.cubic_to(\n f32::from_bits(0x4331cb76), f32::from_bits(0x43ea3340),\n f32::from_bits(0x4388f498), f32::from_bits(0x42f7f08d),\n f32::from_bits(0x43f1cd32), f32::from_bits(0x42802ec1),\n );\n let path = pb.finish().unwrap();\n\n let mut stroke = Stroke::default();\n stroke.width = 42.835968;\n\n assert!(PathStroker::new().stroke(&path, &stroke, 1.0).is_some());\n }\n}\n" }, { "package": "tiny-skia-path@0.10.0", "sourcePath": "src/transform.rs", "sha256": "a84e29e3a0819fd563b55a5570deadb332ef2e2008cfbc102176ec37df3ad61b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2006 The Android Open Source Project\n// Copyright 2020 Yevhenii Reizner\n//\n// Use of this source code is governed by a BSD-style license that can be\n// found in the LICENSE file.\n\nuse crate::{NonZeroRect, Point};\n\nuse crate::scalar::{Scalar, SCALAR_NEARLY_ZERO};\n\n#[cfg(all(not(feature = \"std\"), feature = \"no-std-float\"))]\nuse crate::NoStdFloat;\n\n/// An affine transformation matrix.\n///\n/// Unlike other types, doesn't guarantee to be valid. This is Skia quirk.\n/// Meaning Transform(0, 0, 0, 0, 0, 0) is ok, while it's technically not.\n/// Non-finite values are also not an error.\n#[allow(missing_docs)]\n#[derive(Copy, Clone, PartialEq, Debug)]\npub struct Transform {\n pub sx: f32,\n pub kx: f32,\n pub ky: f32,\n pub sy: f32,\n pub tx: f32,\n pub ty: f32,\n}\n\nimpl Default for Transform {\n fn default() -> Self {\n Transform {\n sx: 1.0,\n kx: 0.0,\n ky: 0.0,\n sy: 1.0,\n tx: 0.0,\n ty: 0.0,\n }\n }\n}\n\nimpl Transform {\n /// Creates an identity transform.\n pub fn identity() -> Self {\n Transform::default()\n }\n\n /// Creates a new `Transform`.\n ///\n /// We are using column-major-column-vector matrix notation, therefore it's ky-kx, not kx-ky.\n pub fn from_row(sx: f32, ky: f32, kx: f32, sy: f32, tx: f32, ty: f32) -> Self {\n Transform {\n sx,\n ky,\n kx,\n sy,\n tx,\n ty,\n }\n }\n\n /// Creates a new translating `Transform`.\n pub fn from_translate(tx: f32, ty: f32) -> Self {\n Transform::from_row(1.0, 0.0, 0.0, 1.0, tx, ty)\n }\n\n /// Creates a new scaling `Transform`.\n pub fn from_scale(sx: f32, sy: f32) -> Self {\n Transform::from_row(sx, 0.0, 0.0, sy, 0.0, 0.0)\n }\n\n /// Creates a new skewing `Transform`.\n pub fn from_skew(kx: f32, ky: f32) -> Self {\n Transform::from_row(1.0, ky, kx, 1.0, 0.0, 0.0)\n }\n\n /// Creates a new rotating `Transform`.\n ///\n /// `angle` in degrees.\n pub fn from_rotate(angle: f32) -> Self {\n let v = angle.to_radians();\n let a = v.cos();\n let b = v.sin();\n let c = -b;\n let d = a;\n Transform::from_row(a, b, c, d, 0.0, 0.0)\n }\n\n /// Creates a new rotating `Transform` at the specified position.\n ///\n /// `angle` in degrees.\n pub fn from_rotate_at(angle: f32, tx: f32, ty: f32) -> Self {\n let mut ts = Self::default();\n ts = ts.pre_translate(tx, ty);\n ts = ts.pre_concat(Transform::from_rotate(angle));\n ts = ts.pre_translate(-tx, -ty);\n ts\n }\n\n /// Converts `Rect` into a bounding box `Transform`.\n #[inline]\n pub fn from_bbox(bbox: NonZeroRect) -> Self {\n Transform::from_row(bbox.width(), 0.0, 0.0, bbox.height(), bbox.x(), bbox.y())\n }\n\n /// Checks that transform is finite.\n pub fn is_finite(&self) -> bool {\n self.sx.is_finite()\n && self.ky.is_finite()\n && self.kx.is_finite()\n && self.sy.is_finite()\n && self.tx.is_finite()\n && self.ty.is_finite()\n }\n\n /// Checks that transform is finite and has non-zero scale.\n pub fn is_valid(&self) -> bool {\n if self.is_finite() {\n let (sx, sy) = self.get_scale();\n if sx.is_nearly_zero() || sy.is_nearly_zero() {\n false\n } else {\n true\n }\n } else {\n false\n }\n }\n\n /// Checks that transform is identity.\n pub fn is_identity(&self) -> bool {\n *self == Transform::default()\n }\n\n /// Checks that transform is scale-only.\n pub fn is_scale(&self) -> bool {\n self.has_scale() && !self.has_skew() && !self.has_translate()\n }\n\n /// Checks that transform is skew-only.\n pub fn is_skew(&self) -> bool {\n !self.has_scale() && self.has_skew() && !self.has_translate()\n }\n\n /// Checks that transform is translate-only.\n pub fn is_translate(&self) -> bool {\n !self.has_scale() && !self.has_skew() && self.has_translate()\n }\n\n /// Checks that transform contains only scale and translate.\n pub fn is_scale_translate(&self) -> bool {\n (self.has_scale() || self.has_translate()) && !self.has_skew()\n }\n\n /// Checks that transform contains a scale part.\n pub fn has_scale(&self) -> bool {\n self.sx != 1.0 || self.sy != 1.0\n }\n\n /// Checks that transform contains a skew part.\n pub fn has_skew(&self) -> bool {\n self.kx != 0.0 || self.ky != 0.0\n }\n\n /// Checks that transform contains a translate part.\n pub fn has_translate(&self) -> bool {\n self.tx != 0.0 || self.ty != 0.0\n }\n\n /// Returns transform's scale part.\n pub fn get_scale(&self) -> (f32, f32) {\n let x_scale = (self.sx * self.sx + self.kx * self.kx).sqrt();\n let y_scale = (self.ky * self.ky + self.sy * self.sy).sqrt();\n (x_scale, y_scale)\n }\n\n /// Pre-scales the current transform.\n #[must_use]\n pub fn pre_scale(&self, sx: f32, sy: f32) -> Self {\n self.pre_concat(Transform::from_scale(sx, sy))\n }\n\n /// Post-scales the current transform.\n #[must_use]\n pub fn post_scale(&self, sx: f32, sy: f32) -> Self {\n self.post_concat(Transform::from_scale(sx, sy))\n }\n\n /// Pre-translates the current transform.\n #[must_use]\n pub fn pre_translate(&self, tx: f32, ty: f32) -> Self {\n self.pre_concat(Transform::from_translate(tx, ty))\n }\n\n /// Post-translates the current transform.\n #[must_use]\n pub fn post_translate(&self, tx: f32, ty: f32) -> Self {\n self.post_concat(Transform::from_translate(tx, ty))\n }\n\n /// Pre-rotates the current transform.\n ///\n /// `angle` in degrees.\n #[must_use]\n pub fn pre_rotate(&self, angle: f32) -> Self {\n self.pre_concat(Transform::from_rotate(angle))\n }\n\n /// Post-rotates the current transform.\n ///\n /// `angle` in degrees.\n #[must_use]\n pub fn post_rotate(&self, angle: f32) -> Self {\n self.post_concat(Transform::from_rotate(angle))\n }\n\n /// Pre-rotates the current transform by the specified position.\n ///\n /// `angle` in degrees.\n #[must_use]\n pub fn pre_rotate_at(&self, angle: f32, tx: f32, ty: f32) -> Self {\n self.pre_concat(Transform::from_rotate_at(angle, tx, ty))\n }\n\n /// Post-rotates the current transform by the specified position.\n ///\n /// `angle` in degrees.\n #[must_use]\n pub fn post_rotate_at(&self, angle: f32, tx: f32, ty: f32) -> Self {\n self.post_concat(Transform::from_rotate_at(angle, tx, ty))\n }\n\n /// Pre-concats the current transform.\n #[must_use]\n pub fn pre_concat(&self, other: Self) -> Self {\n concat(*self, other)\n }\n\n /// Post-concats the current transform.\n #[must_use]\n pub fn post_concat(&self, other: Self) -> Self {\n concat(other, *self)\n }\n\n pub(crate) fn from_sin_cos(sin: f32, cos: f32) -> Self {\n Transform::from_row(cos, sin, -sin, cos, 0.0, 0.0)\n }\n\n /// Transforms a points using the current transform.\n pub fn map_point(&self, point: &mut Point) {\n if self.is_identity() {\n // Do nothing.\n } else if self.is_translate() {\n point.x += self.tx;\n point.y += self.ty;\n } else if self.is_scale_translate() {\n point.x = point.x * self.sx + self.tx;\n point.y = point.y * self.sy + self.ty;\n } else {\n let x = point.x * self.sx + point.y * self.kx + self.tx;\n let y = point.x * self.ky + point.y * self.sy + self.ty;\n point.x = x;\n point.y = y;\n }\n }\n\n /// Transforms a slice of points using the current transform.\n pub fn map_points(&self, points: &mut [Point]) {\n if points.is_empty() {\n return;\n }\n\n // TODO: simd\n\n if self.is_identity() {\n // Do nothing.\n } else if self.is_translate() {\n for p in points {\n p.x += self.tx;\n p.y += self.ty;\n }\n } else if self.is_scale_translate() {\n for p in points {\n p.x = p.x * self.sx + self.tx;\n p.y = p.y * self.sy + self.ty;\n }\n } else {\n for p in points {\n let x = p.x * self.sx + p.y * self.kx + self.tx;\n let y = p.x * self.ky + p.y * self.sy + self.ty;\n p.x = x;\n p.y = y;\n }\n }\n }\n\n /// Returns an inverted transform.\n pub fn invert(&self) -> Option {\n // Allow the trivial case to be inlined.\n if self.is_identity() {\n return Some(*self);\n }\n\n invert(self)\n }\n}\n\nfn invert(ts: &Transform) -> Option {\n debug_assert!(!ts.is_identity());\n\n if ts.is_scale_translate() {\n if ts.has_scale() {\n let inv_x = ts.sx.invert();\n let inv_y = ts.sy.invert();\n Some(Transform::from_row(\n inv_x,\n 0.0,\n 0.0,\n inv_y,\n -ts.tx * inv_x,\n -ts.ty * inv_y,\n ))\n } else {\n // translate only\n Some(Transform::from_translate(-ts.tx, -ts.ty))\n }\n } else {\n let inv_det = inv_determinant(ts)?;\n let inv_ts = compute_inv(ts, inv_det);\n\n if inv_ts.is_finite() {\n Some(inv_ts)\n } else {\n None\n }\n }\n}\n\nfn inv_determinant(ts: &Transform) -> Option {\n let det = dcross(ts.sx as f64, ts.sy as f64, ts.kx as f64, ts.ky as f64);\n\n // Since the determinant is on the order of the cube of the matrix members,\n // compare to the cube of the default nearly-zero constant (although an\n // estimate of the condition number would be better if it wasn't so expensive).\n let tolerance = SCALAR_NEARLY_ZERO * SCALAR_NEARLY_ZERO * SCALAR_NEARLY_ZERO;\n if (det as f32).is_nearly_zero_within_tolerance(tolerance) {\n None\n } else {\n Some(1.0 / det)\n }\n}\n\nfn compute_inv(ts: &Transform, inv_det: f64) -> Transform {\n Transform::from_row(\n (ts.sy as f64 * inv_det) as f32,\n (-ts.ky as f64 * inv_det) as f32,\n (-ts.kx as f64 * inv_det) as f32,\n (ts.sx as f64 * inv_det) as f32,\n dcross_dscale(ts.kx, ts.ty, ts.sy, ts.tx, inv_det),\n dcross_dscale(ts.ky, ts.tx, ts.sx, ts.ty, inv_det),\n )\n}\n\nfn dcross(a: f64, b: f64, c: f64, d: f64) -> f64 {\n a * b - c * d\n}\n\nfn dcross_dscale(a: f32, b: f32, c: f32, d: f32, scale: f64) -> f32 {\n (dcross(a as f64, b as f64, c as f64, d as f64) * scale) as f32\n}\n\nfn concat(a: Transform, b: Transform) -> Transform {\n if a.is_identity() {\n b\n } else if b.is_identity() {\n a\n } else if !a.has_skew() && !b.has_skew() {\n // just scale and translate\n Transform::from_row(\n a.sx * b.sx,\n 0.0,\n 0.0,\n a.sy * b.sy,\n a.sx * b.tx + a.tx,\n a.sy * b.ty + a.ty,\n )\n } else {\n Transform::from_row(\n mul_add_mul(a.sx, b.sx, a.kx, b.ky),\n mul_add_mul(a.ky, b.sx, a.sy, b.ky),\n mul_add_mul(a.sx, b.kx, a.kx, b.sy),\n mul_add_mul(a.ky, b.kx, a.sy, b.sy),\n mul_add_mul(a.sx, b.tx, a.kx, b.ty) + a.tx,\n mul_add_mul(a.ky, b.tx, a.sy, b.ty) + a.ty,\n )\n }\n}\n\nfn mul_add_mul(a: f32, b: f32, c: f32, d: f32) -> f32 {\n (f64::from(a) * f64::from(b) + f64::from(c) * f64::from(d)) as f32\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn transform() {\n assert_eq!(\n Transform::identity(),\n Transform::from_row(1.0, 0.0, 0.0, 1.0, 0.0, 0.0)\n );\n\n assert_eq!(\n Transform::from_scale(1.0, 2.0),\n Transform::from_row(1.0, 0.0, 0.0, 2.0, 0.0, 0.0)\n );\n\n assert_eq!(\n Transform::from_skew(2.0, 3.0),\n Transform::from_row(1.0, 3.0, 2.0, 1.0, 0.0, 0.0)\n );\n\n assert_eq!(\n Transform::from_translate(5.0, 6.0),\n Transform::from_row(1.0, 0.0, 0.0, 1.0, 5.0, 6.0)\n );\n\n let ts = Transform::identity();\n assert_eq!(ts.is_identity(), true);\n assert_eq!(ts.is_scale(), false);\n assert_eq!(ts.is_skew(), false);\n assert_eq!(ts.is_translate(), false);\n assert_eq!(ts.is_scale_translate(), false);\n assert_eq!(ts.has_scale(), false);\n assert_eq!(ts.has_skew(), false);\n assert_eq!(ts.has_translate(), false);\n\n let ts = Transform::from_scale(2.0, 3.0);\n assert_eq!(ts.is_identity(), false);\n assert_eq!(ts.is_scale(), true);\n assert_eq!(ts.is_skew(), false);\n assert_eq!(ts.is_translate(), false);\n assert_eq!(ts.is_scale_translate(), true);\n assert_eq!(ts.has_scale(), true);\n assert_eq!(ts.has_skew(), false);\n assert_eq!(ts.has_translate(), false);\n\n let ts = Transform::from_skew(2.0, 3.0);\n assert_eq!(ts.is_identity(), false);\n assert_eq!(ts.is_scale(), false);\n assert_eq!(ts.is_skew(), true);\n assert_eq!(ts.is_translate(), false);\n assert_eq!(ts.is_scale_translate(), false);\n assert_eq!(ts.has_scale(), false);\n assert_eq!(ts.has_skew(), true);\n assert_eq!(ts.has_translate(), false);\n\n let ts = Transform::from_translate(2.0, 3.0);\n assert_eq!(ts.is_identity(), false);\n assert_eq!(ts.is_scale(), false);\n assert_eq!(ts.is_skew(), false);\n assert_eq!(ts.is_translate(), true);\n assert_eq!(ts.is_scale_translate(), true);\n assert_eq!(ts.has_scale(), false);\n assert_eq!(ts.has_skew(), false);\n assert_eq!(ts.has_translate(), true);\n\n let ts = Transform::from_row(1.0, 2.0, 3.0, 4.0, 5.0, 6.0);\n assert_eq!(ts.is_identity(), false);\n assert_eq!(ts.is_scale(), false);\n assert_eq!(ts.is_skew(), false);\n assert_eq!(ts.is_translate(), false);\n assert_eq!(ts.is_scale_translate(), false);\n assert_eq!(ts.has_scale(), true);\n assert_eq!(ts.has_skew(), true);\n assert_eq!(ts.has_translate(), true);\n\n let ts = Transform::from_scale(1.0, 1.0);\n assert_eq!(ts.has_scale(), false);\n\n let ts = Transform::from_skew(0.0, 0.0);\n assert_eq!(ts.has_skew(), false);\n\n let ts = Transform::from_translate(0.0, 0.0);\n assert_eq!(ts.has_translate(), false);\n }\n\n #[test]\n fn concat() {\n let mut ts = Transform::from_row(1.2, 3.4, -5.6, -7.8, 1.2, 3.4);\n ts = ts.pre_scale(2.0, -4.0);\n assert_eq!(ts, Transform::from_row(2.4, 6.8, 22.4, 31.2, 1.2, 3.4));\n\n let mut ts = Transform::from_row(1.2, 3.4, -5.6, -7.8, 1.2, 3.4);\n ts = ts.post_scale(2.0, -4.0);\n assert_eq!(ts, Transform::from_row(2.4, -13.6, -11.2, 31.2, 2.4, -13.6));\n }\n}\n" }, { "package": "tinyvec@1.13.3", "sourcePath": "LICENSE-APACHE.md", "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "package": "tinyvec@1.13.3", "sourcePath": "LICENSE-MIT.md", "sha256": "fd80a26fbb3f644af1fa994134446702932968519797227e07a1368dea80f0bc", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "package": "tinyvec@1.13.3", "sourcePath": "LICENSE-ZLIB.md", "sha256": "84b34dd7608f7fb9b17bd588a6bf392bf7de504e2716f024a77d89f1b145a151", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Copyright (c) 2019 Daniel \"Lokathor\" Gee.\n\nThis software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software.\n\nPermission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions:\n\n1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.\n\n2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.\n\n3. This notice may not be removed or altered from any source distribution.\n" }, { "package": "tokio@1.53.1", "sourcePath": "README.md", "sha256": "d66d59d170ce56b4f80e16e7564fe91ca611daffffe82a8625a9a67bb85c8800", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "*[TokioConf 2026 program and tickets are now available!](https://tokioconf.com)*\n\n---\n\n# Tokio\n\nA runtime for writing reliable, asynchronous, and slim applications with\nthe Rust programming language. It is:\n\n* **Fast**: Tokio's zero-cost abstractions give you bare-metal\n performance.\n\n* **Reliable**: Tokio leverages Rust's ownership, type system, and\n concurrency model to reduce bugs and ensure thread safety.\n\n* **Scalable**: Tokio has a minimal footprint, and handles backpressure\n and cancellation naturally.\n\n[![Crates.io][crates-badge]][crates-url]\n[![MIT licensed][mit-badge]][mit-url]\n[![Build Status][actions-badge]][actions-url]\n[![Discord chat][discord-badge]][discord-url]\n\n[crates-badge]: https://img.shields.io/crates/v/tokio.svg\n[crates-url]: https://crates.io/crates/tokio\n[mit-badge]: https://img.shields.io/badge/license-MIT-blue.svg\n[mit-url]: https://github.com/tokio-rs/tokio/blob/master/LICENSE\n[actions-badge]: https://github.com/tokio-rs/tokio/workflows/CI/badge.svg\n[actions-url]: https://github.com/tokio-rs/tokio/actions?query=workflow%3ACI+branch%3Amaster\n[discord-badge]: https://img.shields.io/discord/500028886025895936.svg?logo=discord&style=flat-square\n[discord-url]: https://discord.gg/tokio\n\n[Website](https://tokio.rs) |\n[Guides](https://tokio.rs/tokio/tutorial) |\n[API Docs](https://docs.rs/tokio/latest/tokio) |\n[Chat](https://discord.gg/tokio)\n\n## Overview\n\nTokio is an event-driven, non-blocking I/O platform for writing\nasynchronous applications with the Rust programming language. At a high\nlevel, it provides a few major components:\n\n* A multithreaded, work-stealing based task [scheduler].\n* A reactor backed by the operating system's event queue (epoll, kqueue,\n IOCP, etc.).\n* Asynchronous [TCP and UDP][net] sockets.\n\nThese components provide the runtime components necessary for building\nan asynchronous application.\n\n[net]: https://docs.rs/tokio/latest/tokio/net/index.html\n[scheduler]: https://docs.rs/tokio/latest/tokio/runtime/index.html\n\n## Example\n\nA basic TCP echo server with Tokio.\n\nMake sure you enable the full features of the tokio crate on Cargo.toml:\n\n```toml\n[dependencies]\ntokio = { version = \"1.53.1\", features = [\"full\"] }\n```\nThen, on your main.rs:\n\n```rust,no_run\nuse tokio::net::TcpListener;\nuse tokio::io::{AsyncReadExt, AsyncWriteExt};\n\n#[tokio::main]\nasync fn main() -> Result<(), Box> {\n let listener = TcpListener::bind(\"127.0.0.1:8080\").await?;\n\n loop {\n let (mut socket, _) = listener.accept().await?;\n\n tokio::spawn(async move {\n let mut buf = [0; 1024];\n\n // In a loop, read data from the socket and write the data back.\n loop {\n let n = match socket.read(&mut buf).await {\n // socket closed\n Ok(0) => return,\n Ok(n) => n,\n Err(e) => {\n eprintln!(\"failed to read from socket; err = {:?}\", e);\n return;\n }\n };\n\n // Write the data back\n if let Err(e) = socket.write_all(&buf[0..n]).await {\n eprintln!(\"failed to write to socket; err = {:?}\", e);\n return;\n }\n }\n });\n }\n}\n```\n\nMore examples can be found [here][examples]. For a larger \"real world\" example, see the\n[mini-redis] repository.\n\n[examples]: https://github.com/tokio-rs/tokio/tree/master/examples\n[mini-redis]: https://github.com/tokio-rs/mini-redis/\n\nTo see a list of the available feature flags that can be enabled, check our\n[docs][feature-flag-docs].\n\n## Getting Help\n\nFirst, see if the answer to your question can be found in the [Guides] or the\n[API documentation]. If the answer is not there, there is an active community in\nthe [Tokio Discord server][chat]. We would be happy to try to answer your\nquestion. You can also ask your question on [the discussions page][discussions].\n\n[Guides]: https://tokio.rs/tokio/tutorial\n[API documentation]: https://docs.rs/tokio/latest/tokio\n[chat]: https://discord.gg/tokio\n[discussions]: https://github.com/tokio-rs/tokio/discussions\n[feature-flag-docs]: https://docs.rs/tokio/#feature-flags\n\n## Contributing\n\n:balloon: Thanks for your help improving the project! We are so happy to have\nyou! We have a [contributing guide][guide] to help you get involved in the Tokio\nproject.\n\n[guide]: https://github.com/tokio-rs/tokio/blob/master/docs/contributing/README.md\n\n## Related Projects\n\nIn addition to the crates in this repository, the Tokio project also maintains\nseveral other libraries, including:\n\n* [`axum`]: A web application framework that focuses on ergonomics and modularity.\n\n* [`hyper`]: A fast and correct HTTP/1.1 and HTTP/2 implementation for Rust.\n\n* [`tonic`]: A gRPC over HTTP/2 implementation focused on high performance, interoperability, and flexibility.\n\n* [`warp`]: A super-easy, composable, web server framework for warp speeds.\n\n* [`tower`]: A library of modular and reusable components for building robust networking clients and servers.\n\n* [`tracing`] (formerly `tokio-trace`): A framework for application-level tracing and async-aware diagnostics.\n\n* [`mio`]: A low-level, cross-platform abstraction over OS I/O APIs that powers `tokio`.\n\n* [`bytes`]: Utilities for working with bytes, including efficient byte buffers.\n\n* [`loom`]: A testing tool for concurrent Rust code.\n\n[`axum`]: https://github.com/tokio-rs/axum\n[`warp`]: https://github.com/seanmonstar/warp\n[`hyper`]: https://github.com/hyperium/hyper\n[`tonic`]: https://github.com/hyperium/tonic\n[`tower`]: https://github.com/tower-rs/tower\n[`loom`]: https://github.com/tokio-rs/loom\n[`tracing`]: https://github.com/tokio-rs/tracing\n[`mio`]: https://github.com/tokio-rs/mio\n[`bytes`]: https://github.com/tokio-rs/bytes\n\n## Changelog\n\nThe Tokio repository contains multiple crates. Each crate has its own changelog.\n\n * `tokio` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio/CHANGELOG.md)\n * `tokio-util` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-util/CHANGELOG.md)\n * `tokio-stream` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-stream/CHANGELOG.md)\n * `tokio-macros` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-macros/CHANGELOG.md)\n * `tokio-test` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-test/CHANGELOG.md)\n\n## Supported Rust Versions\n\n\n\nTokio will keep a rolling MSRV (minimum supported rust version) policy of **at\nleast** 6 months. When increasing the MSRV, the new Rust version must have been\nreleased at least six months ago. The current MSRV is 1.71.\n\nNote that the MSRV is not increased automatically, and only as part of a minor\nrelease. The MSRV history for past minor releases can be found below:\n\n * 1.48 to now - Rust 1.71\n * 1.39 to 1.47 - Rust 1.70\n * 1.30 to 1.38 - Rust 1.63\n * 1.27 to 1.29 - Rust 1.56\n * 1.17 to 1.26 - Rust 1.49\n * 1.15 to 1.16 - Rust 1.46\n * 1.0 to 1.14 - Rust 1.45\n\nNote that although we try to avoid the situation where a dependency transitively\nincreases the MSRV of Tokio, we do not guarantee that this does not happen.\nHowever, every minor release will have some set of versions of dependencies that\nworks with the MSRV of that minor release.\n\n## Release schedule\n\nTokio doesn't follow a fixed release schedule, but we typically make one minor\nrelease each month. We make patch releases for bugfixes as necessary.\n\n## Bug patching policy\n\nFor the purposes of making patch releases with bugfixes, we have designated\ncertain minor releases as LTS (long term support) releases. Whenever a bug\nwarrants a patch release with a fix for the bug, it will be backported and\nreleased as a new patch release for each LTS minor version. Our current LTS\nreleases are:\n\n * `1.47.x` - LTS release until September 2026. (MSRV 1.70)\n * `1.51.x` - LTS release until March 2027. (MSRV 1.71)\n\nEach LTS release will continue to receive backported fixes for at least a year.\nIf you wish to use a fixed minor release in your project, we recommend that you\nuse an LTS release.\n\nTo use a fixed minor version, you can specify the version with a tilde. For\nexample, to specify that you wish to use the newest `1.47.x` patch release, you\ncan use the following dependency specification:\n```text\ntokio = { version = \"~1.47\", features = [...] }\n```\n\n### Previous LTS releases\n\n * `1.8.x` - LTS release until February 2022.\n * `1.14.x` - LTS release until June 2022.\n * `1.18.x` - LTS release until June 2023.\n * `1.20.x` - LTS release until September 2023.\n * `1.25.x` - LTS release until March 2024.\n * `1.32.x` - LTS release until September 2024.\n * `1.36.x` - LTS release until March 2025.\n * `1.38.x` - LTS release until July 2025.\n * `1.43.x` - LTS release until March 2026.\n\n## License\n\nThis project is licensed under the [MIT license].\n\n[MIT license]: https://github.com/tokio-rs/tokio/blob/master/LICENSE\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in Tokio by you shall be licensed as MIT, without any additional\nterms or conditions.\n" }, { "package": "ttf-parser@0.18.1", "sourcePath": "README.md", "sha256": "f2d796cd4a2acc5b881ee245b8187793452367cce94ce7af247e0d2b748a5c1d", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "## ttf-parser\n![Build Status](https://github.com/RazrFalcon/ttf-parser/workflows/Rust/badge.svg)\n[![Crates.io](https://img.shields.io/crates/v/ttf-parser.svg)](https://crates.io/crates/ttf-parser)\n[![Documentation](https://docs.rs/ttf-parser/badge.svg)](https://docs.rs/ttf-parser)\n[![Rust 1.42+](https://img.shields.io/badge/rust-1.42+-orange.svg)](https://www.rust-lang.org)\n![](https://img.shields.io/badge/unsafe-forbidden-brightgreen.svg)\n\nA high-level, safe, zero-allocation TrueType font parser.\n\nSupports [TrueType](https://docs.microsoft.com/en-us/typography/truetype/),\n[OpenType](https://docs.microsoft.com/en-us/typography/opentype/spec/)\nand [AAT](https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6AATIntro.html)\nfonts.\n\nCan be used as Rust and as C library.\n\n### Features\n\n- A high-level API for most common properties, hiding all parsing and data resolving logic.\n- A low-level, but safe API to access TrueType tables data.\n- Highly configurable. You can disable most of the features, reducing binary size.\n You can also parse TrueType tables separately, without loading the whole font/face.\n- Zero heap allocations.\n- Zero unsafe.\n- Zero dependencies.\n- `no_std`/WASM compatible.\n- A basic [C API](./c-api).\n- Fast.\n- Stateless. All parsing methods are immutable.\n- Simple and maintainable code (no magic numbers).\n\n### Safety\n\n- The library must not panic. Any panic considered as a critical bug and should be reported.\n- The library forbids unsafe code.\n- No heap allocations, so crash due to OOM is not possible.\n- All recursive methods have a depth limit.\n- Technically, should use less than 64KiB of stack in the worst case scenario.\n- Most of arithmetic operations are checked.\n- Most of numeric casts are checked.\n\n### Alternatives\n\nIt's very hard to compare different libraries, so we are using table-based comparison.\nThere are roughly three types of TrueType tables:\n\n- A table with a list of properties (like `head`, `OS/2`, etc.).
\n If a library tries to parse it at all then we mark it as supported.\n- A table that contains a single type of data (`glyf`, `CFF` (kinda), `hmtx`, etc.).
\n Can only be supported or not.\n- A table that contains multiple subtables (`cmap`, `kern`, `GPOS`, etc.).
\n Can be partially supported and we note which subtables are actually supported.\n\n| Feature/Library | ttf-parser | FreeType | stb_truetype |\n| ----------------- | :--------------------: | :-----------------: | :----------------------------: |\n| Memory safe | ✓ | | |\n| Thread safe | ✓ | | ~ (mostly reentrant) |\n| Zero allocation | ✓ | | |\n| Variable fonts | ✓ | ✓ | |\n| Rendering | -1 | ✓ | ~ (very primitive) |\n| `ankr` table | ✓ | | |\n| `avar` table | ✓ | ✓ | |\n| `bdat` table | | ✓ | |\n| `bloc` table | | ✓ | |\n| `CBDT` table | ✓ | ✓ | |\n| `CBLC` table | ✓ | ✓ | |\n| `COLR` table | | ✓ | |\n| `CPAL` table | | ✓ | |\n| `CFF ` table | ✓ | ✓ | ~ (no `seac` support) |\n| `CFF2` table | ✓ | ✓ | |\n| `cmap` table | ~ (no 8) | ✓ | ~ (no 2,8,10,14; Unicode-only) |\n| `EBDT` table | | ✓ | |\n| `EBLC` table | | ✓ | |\n| `feat` table | ✓ | | |\n| `fvar` table | ✓ | ✓ | |\n| `gasp` table | | ✓ | |\n| `GDEF` table | ~ | | |\n| `glyf` table | ~2 | ✓ | ~2 |\n| `GPOS` table | ✓ | | ~ (only 2) |\n| `GSUB` table | ✓ | | |\n| `gvar` table | ✓ | ✓ | |\n| `head` table | ✓ | ✓ | ✓ |\n| `hhea` table | ✓ | ✓ | ✓ |\n| `hmtx` table | ✓ | ✓ | ✓ |\n| `HVAR` table | ✓ | ✓ | |\n| `kern` table | ✓ | ~ (only 0) | ~ (only 0) |\n| `kerx` table | ✓ | | |\n| `MATH` table | ✓ | | |\n| `maxp` table | ✓ | ✓ | ✓ |\n| `morx` table | ✓ | | |\n| `MVAR` table | ✓ | ✓ | |\n| `name` table | ✓ | ✓ | |\n| `OS/2` table | ✓ | ✓ | |\n| `post` table | ✓ | ✓ | |\n| `sbix` table | ~ (PNG only) | ~ (PNG only) | |\n| `SVG ` table | ✓ | ✓ | ✓ |\n| `trak` table | ✓ | | |\n| `vhea` table | ✓ | ✓ | |\n| `vmtx` table | ✓ | ✓ | |\n| `VORG` table | ✓ | ✓ | |\n| `VVAR` table | ✓ | ✓ | |\n| Language | Rust + C API | C | C |\n| Tested version | 0.17.0 | 2.12.0 | 1.24 |\n| License | MIT / Apache-2.0 | FTL / GPLv2 | public domain |\n\nLegend:\n\n- ✓ - supported\n- ~ - partial\n- *nothing* - not supported\n\nNotes:\n\n1. While `ttf-parser` doesn't support rendering by itself,\n there are multiple rendering libraries on top of it:\n [rusttype](https://gitlab.redox-os.org/redox-os/rusttype),\n [ab-glyph](https://github.com/alexheretic/ab-glyph)\n and [fontdue](https://github.com/mooman219/fontdue).\n2. Matching points are not supported.\n\n### Performance\n\nTrueType fonts designed for fast querying, so most of the methods are very fast.\nThe main exception is glyph outlining. Glyphs can be stored using two different methods:\nusing [Glyph Data Format](https://docs.microsoft.com/en-us/typography/opentype/spec/glyf)\nand [Compact Font Format](http://wwwimages.adobe.com/content/dam/Adobe/en/devnet/font/pdfs/5176.CFF.pdf) (pdf).\nThe first one is fairly simple which makes it faster to process.\nThe second one is basically a tiny language with a stack-based VM, which makes it way harder to process.\n\nThe [benchmark](./benches/outline/) tests how long it takes to outline all glyphs in a font.\n\nx86 (AMD 3700X)\n\n| Table/Library | ttf-parser | FreeType | stb_truetype |\n| ------------- | -------------: | ---------: | -------------: |\n| `glyf` | `0.901 ms` | `1.171 ms` | **`0.675 ms`** |\n| `gvar` | **`2.972 ms`** | `4.132 ms` | - |\n| `CFF` | **`1.197 ms`** | `5.647 ms` | `2.813 ms` |\n| `CFF2` | **`1.968 ms`** | `6.392 ms` | - |\n\nARM (Apple M1)\n\n| Table/Library | ttf-parser | FreeType | stb_truetype |\n| ------------- | -------------: | ---------: | -------------: |\n| `glyf` | **`0.550 ms`** | `0.854 ms` | `0.703 ms` |\n| `gvar` | **`2.270 ms`** | `4.594 ms` | - |\n| `CFF` | **`1.054 ms`** | `5.223 ms` | `3.262 ms` |\n| `CFF2` | **`1.765 ms`** | `5.995 ms` | - |\n\n**Note:** FreeType is surprisingly slow, so I'm worried that I've messed something up.\n\nAnd here are some methods benchmarks:\n\n```text\ntest outline_glyph_276_from_cff2 ... bench: 867 ns/iter (+/- 15)\ntest from_data_otf_cff ... bench: 968 ns/iter (+/- 13)\ntest from_data_otf_cff2 ... bench: 887 ns/iter (+/- 25)\ntest outline_glyph_276_from_cff ... bench: 678 ns/iter (+/- 41)\ntest outline_glyph_276_from_glyf ... bench: 649 ns/iter (+/- 11)\ntest outline_glyph_8_from_cff2 ... bench: 534 ns/iter (+/- 14)\ntest from_data_ttf ... bench: 467 ns/iter (+/- 11)\ntest glyph_name_post_276 ... bench: 223 ns/iter (+/- 5)\ntest outline_glyph_8_from_cff ... bench: 315 ns/iter (+/- 13)\ntest outline_glyph_8_from_glyf ... bench: 291 ns/iter (+/- 5)\ntest family_name ... bench: 183 ns/iter (+/- 102)\ntest glyph_name_cff_276 ... bench: 62 ns/iter (+/- 1)\ntest glyph_index_u41 ... bench: 16 ns/iter (+/- 0)\ntest glyph_name_cff_8 ... bench: 5 ns/iter (+/- 0)\ntest glyph_name_post_8 ... bench: 2 ns/iter (+/- 0)\ntest subscript_metrics ... bench: 2 ns/iter (+/- 0)\ntest glyph_hor_advance ... bench: 2 ns/iter (+/- 0)\ntest glyph_hor_side_bearing ... bench: 2 ns/iter (+/- 0)\ntest glyph_name_8 ... bench: 1 ns/iter (+/- 0)\ntest ascender ... bench: 1 ns/iter (+/- 0)\ntest underline_metrics ... bench: 1 ns/iter (+/- 0)\ntest strikeout_metrics ... bench: 1 ns/iter (+/- 0)\ntest x_height ... bench: 1 ns/iter (+/- 0)\ntest units_per_em ... bench: 0.5 ns/iter (+/- 0)\ntest width ... bench: 0.2 ns/iter (+/- 0)\n```\n\n### License\n\nLicensed under either of\n\n- Apache License, Version 2.0\n ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n- MIT license\n ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "package": "ttf-parser@0.18.1", "sourcePath": "src/tables/cff/std_names.rs", "sha256": "28f192e64551773c97d0457ddabc774cfabe535585a2ef65f37ca48022221923", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "pub const STANDARD_NAMES: &[&str] = &[\n \".notdef\",\n \"space\",\n \"exclam\",\n \"quotedbl\",\n \"numbersign\",\n \"dollar\",\n \"percent\",\n \"ampersand\",\n \"quoteright\",\n \"parenleft\",\n \"parenright\",\n \"asterisk\",\n \"plus\",\n \"comma\",\n \"hyphen\",\n \"period\",\n \"slash\",\n \"zero\",\n \"one\",\n \"two\",\n \"three\",\n \"four\",\n \"five\",\n \"six\",\n \"seven\",\n \"eight\",\n \"nine\",\n \"colon\",\n \"semicolon\",\n \"less\",\n \"equal\",\n \"greater\",\n \"question\",\n \"at\",\n \"A\",\n \"B\",\n \"C\",\n \"D\",\n \"E\",\n \"F\",\n \"G\",\n \"H\",\n \"I\",\n \"J\",\n \"K\",\n \"L\",\n \"M\",\n \"N\",\n \"O\",\n \"P\",\n \"Q\",\n \"R\",\n \"S\",\n \"T\",\n \"U\",\n \"V\",\n \"W\",\n \"X\",\n \"Y\",\n \"Z\",\n \"bracketleft\",\n \"backslash\",\n \"bracketright\",\n \"asciicircum\",\n \"underscore\",\n \"quoteleft\",\n \"a\",\n \"b\",\n \"c\",\n \"d\",\n \"e\",\n \"f\",\n \"g\",\n \"h\",\n \"i\",\n \"j\",\n \"k\",\n \"l\",\n \"m\",\n \"n\",\n \"o\",\n \"p\",\n \"q\",\n \"r\",\n \"s\",\n \"t\",\n \"u\",\n \"v\",\n \"w\",\n \"x\",\n \"y\",\n \"z\",\n \"braceleft\",\n \"bar\",\n \"braceright\",\n \"asciitilde\",\n \"exclamdown\",\n \"cent\",\n \"sterling\",\n \"fraction\",\n \"yen\",\n \"florin\",\n \"section\",\n \"currency\",\n \"quotesingle\",\n \"quotedblleft\",\n \"guillemotleft\",\n \"guilsinglleft\",\n \"guilsinglright\",\n \"fi\",\n \"fl\",\n \"endash\",\n \"dagger\",\n \"daggerdbl\",\n \"periodcentered\",\n \"paragraph\",\n \"bullet\",\n \"quotesinglbase\",\n \"quotedblbase\",\n \"quotedblright\",\n \"guillemotright\",\n \"ellipsis\",\n \"perthousand\",\n \"questiondown\",\n \"grave\",\n \"acute\",\n \"circumflex\",\n \"tilde\",\n \"macron\",\n \"breve\",\n \"dotaccent\",\n \"dieresis\",\n \"ring\",\n \"cedilla\",\n \"hungarumlaut\",\n \"ogonek\",\n \"caron\",\n \"emdash\",\n \"AE\",\n \"ordfeminine\",\n \"Lslash\",\n \"Oslash\",\n \"OE\",\n \"ordmasculine\",\n \"ae\",\n \"dotlessi\",\n \"lslash\",\n \"oslash\",\n \"oe\",\n \"germandbls\",\n \"onesuperior\",\n \"logicalnot\",\n \"mu\",\n \"trademark\",\n \"Eth\",\n \"onehalf\",\n \"plusminus\",\n \"Thorn\",\n \"onequarter\",\n \"divide\",\n \"brokenbar\",\n \"degree\",\n \"thorn\",\n \"threequarters\",\n \"twosuperior\",\n \"registered\",\n \"minus\",\n \"eth\",\n \"multiply\",\n \"threesuperior\",\n \"copyright\",\n \"Aacute\",\n \"Acircumflex\",\n \"Adieresis\",\n \"Agrave\",\n \"Aring\",\n \"Atilde\",\n \"Ccedilla\",\n \"Eacute\",\n \"Ecircumflex\",\n \"Edieresis\",\n \"Egrave\",\n \"Iacute\",\n \"Icircumflex\",\n \"Idieresis\",\n \"Igrave\",\n \"Ntilde\",\n \"Oacute\",\n \"Ocircumflex\",\n \"Odieresis\",\n \"Ograve\",\n \"Otilde\",\n \"Scaron\",\n \"Uacute\",\n \"Ucircumflex\",\n \"Udieresis\",\n \"Ugrave\",\n \"Yacute\",\n \"Ydieresis\",\n \"Zcaron\",\n \"aacute\",\n \"acircumflex\",\n \"adieresis\",\n \"agrave\",\n \"aring\",\n \"atilde\",\n \"ccedilla\",\n \"eacute\",\n \"ecircumflex\",\n \"edieresis\",\n \"egrave\",\n \"iacute\",\n \"icircumflex\",\n \"idieresis\",\n \"igrave\",\n \"ntilde\",\n \"oacute\",\n \"ocircumflex\",\n \"odieresis\",\n \"ograve\",\n \"otilde\",\n \"scaron\",\n \"uacute\",\n \"ucircumflex\",\n \"udieresis\",\n \"ugrave\",\n \"yacute\",\n \"ydieresis\",\n \"zcaron\",\n \"exclamsmall\",\n \"Hungarumlautsmall\",\n \"dollaroldstyle\",\n \"dollarsuperior\",\n \"ampersandsmall\",\n \"Acutesmall\",\n \"parenleftsuperior\",\n \"parenrightsuperior\",\n \"twodotenleader\",\n \"onedotenleader\",\n \"zerooldstyle\",\n \"oneoldstyle\",\n \"twooldstyle\",\n \"threeoldstyle\",\n \"fouroldstyle\",\n \"fiveoldstyle\",\n \"sixoldstyle\",\n \"sevenoldstyle\",\n \"eightoldstyle\",\n \"nineoldstyle\",\n \"commasuperior\",\n \"threequartersemdash\",\n \"periodsuperior\",\n \"questionsmall\",\n \"asuperior\",\n \"bsuperior\",\n \"centsuperior\",\n \"dsuperior\",\n \"esuperior\",\n \"isuperior\",\n \"lsuperior\",\n \"msuperior\",\n \"nsuperior\",\n \"osuperior\",\n \"rsuperior\",\n \"ssuperior\",\n \"tsuperior\",\n \"ff\",\n \"ffi\",\n \"ffl\",\n \"parenleftinferior\",\n \"parenrightinferior\",\n \"Circumflexsmall\",\n \"hyphensuperior\",\n \"Gravesmall\",\n \"Asmall\",\n \"Bsmall\",\n \"Csmall\",\n \"Dsmall\",\n \"Esmall\",\n \"Fsmall\",\n \"Gsmall\",\n \"Hsmall\",\n \"Ismall\",\n \"Jsmall\",\n \"Ksmall\",\n \"Lsmall\",\n \"Msmall\",\n \"Nsmall\",\n \"Osmall\",\n \"Psmall\",\n \"Qsmall\",\n \"Rsmall\",\n \"Ssmall\",\n \"Tsmall\",\n \"Usmall\",\n \"Vsmall\",\n \"Wsmall\",\n \"Xsmall\",\n \"Ysmall\",\n \"Zsmall\",\n \"colonmonetary\",\n \"onefitted\",\n \"rupiah\",\n \"Tildesmall\",\n \"exclamdownsmall\",\n \"centoldstyle\",\n \"Lslashsmall\",\n \"Scaronsmall\",\n \"Zcaronsmall\",\n \"Dieresissmall\",\n \"Brevesmall\",\n \"Caronsmall\",\n \"Dotaccentsmall\",\n \"Macronsmall\",\n \"figuredash\",\n \"hypheninferior\",\n \"Ogoneksmall\",\n \"Ringsmall\",\n \"Cedillasmall\",\n \"questiondownsmall\",\n \"oneeighth\",\n \"threeeighths\",\n \"fiveeighths\",\n \"seveneighths\",\n \"onethird\",\n \"twothirds\",\n \"zerosuperior\",\n \"foursuperior\",\n \"fivesuperior\",\n \"sixsuperior\",\n \"sevensuperior\",\n \"eightsuperior\",\n \"ninesuperior\",\n \"zeroinferior\",\n \"oneinferior\",\n \"twoinferior\",\n \"threeinferior\",\n \"fourinferior\",\n \"fiveinferior\",\n \"sixinferior\",\n \"seveninferior\",\n \"eightinferior\",\n \"nineinferior\",\n \"centinferior\",\n \"dollarinferior\",\n \"periodinferior\",\n \"commainferior\",\n \"Agravesmall\",\n \"Aacutesmall\",\n \"Acircumflexsmall\",\n \"Atildesmall\",\n \"Adieresissmall\",\n \"Aringsmall\",\n \"AEsmall\",\n \"Ccedillasmall\",\n \"Egravesmall\",\n \"Eacutesmall\",\n \"Ecircumflexsmall\",\n \"Edieresissmall\",\n \"Igravesmall\",\n \"Iacutesmall\",\n \"Icircumflexsmall\",\n \"Idieresissmall\",\n \"Ethsmall\",\n \"Ntildesmall\",\n \"Ogravesmall\",\n \"Oacutesmall\",\n \"Ocircumflexsmall\",\n \"Otildesmall\",\n \"Odieresissmall\",\n \"OEsmall\",\n \"Oslashsmall\",\n \"Ugravesmall\",\n \"Uacutesmall\",\n \"Ucircumflexsmall\",\n \"Udieresissmall\",\n \"Yacutesmall\",\n \"Thornsmall\",\n \"Ydieresissmall\",\n \"001.000\",\n \"001.001\",\n \"001.002\",\n \"001.003\",\n \"Black\",\n \"Bold\",\n \"Book\",\n \"Light\",\n \"Medium\",\n \"Regular\",\n \"Roman\",\n \"Semibold\",\n];\n" }, { "package": "ttf-parser@0.18.1", "sourcePath": "src/tables/name.rs", "sha256": "12ff8509c8fc14c12a244bc5b76ef43556d2d6800336e30d8ed9c96b9274bab8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "//! A [Naming Table](\n//! https://docs.microsoft.com/en-us/typography/opentype/spec/name) implementation.\n\n#[cfg(feature = \"std\")]\nuse std::string::String;\n#[cfg(feature = \"std\")]\nuse std::vec::Vec;\n\nuse crate::parser::{FromData, LazyArray16, Offset, Offset16, Stream};\nuse crate::Language;\n\n/// A list of [name ID](https://docs.microsoft.com/en-us/typography/opentype/spec/name#name-ids)'s.\npub mod name_id {\n #![allow(missing_docs)]\n\n pub const COPYRIGHT_NOTICE: u16 = 0;\n pub const FAMILY: u16 = 1;\n pub const SUBFAMILY: u16 = 2;\n pub const UNIQUE_ID: u16 = 3;\n pub const FULL_NAME: u16 = 4;\n pub const VERSION: u16 = 5;\n pub const POST_SCRIPT_NAME: u16 = 6;\n pub const TRADEMARK: u16 = 7;\n pub const MANUFACTURER: u16 = 8;\n pub const DESIGNER: u16 = 9;\n pub const DESCRIPTION: u16 = 10;\n pub const VENDOR_URL: u16 = 11;\n pub const DESIGNER_URL: u16 = 12;\n pub const LICENSE: u16 = 13;\n pub const LICENSE_URL: u16 = 14;\n // RESERVED = 15\n pub const TYPOGRAPHIC_FAMILY: u16 = 16;\n pub const TYPOGRAPHIC_SUBFAMILY: u16 = 17;\n pub const COMPATIBLE_FULL: u16 = 18;\n pub const SAMPLE_TEXT: u16 = 19;\n pub const POST_SCRIPT_CID: u16 = 20;\n pub const WWS_FAMILY: u16 = 21;\n pub const WWS_SUBFAMILY: u16 = 22;\n pub const LIGHT_BACKGROUND_PALETTE: u16 = 23;\n pub const DARK_BACKGROUND_PALETTE: u16 = 24;\n pub const VARIATIONS_POST_SCRIPT_NAME_PREFIX: u16 = 25;\n}\n\n/// A [platform ID](https://docs.microsoft.com/en-us/typography/opentype/spec/name#platform-ids).\n#[allow(missing_docs)]\n#[derive(Clone, Copy, PartialEq, Eq, Debug)]\npub enum PlatformId {\n Unicode,\n Macintosh,\n Iso,\n Windows,\n Custom,\n}\n\nimpl FromData for PlatformId {\n const SIZE: usize = 2;\n\n #[inline]\n fn parse(data: &[u8]) -> Option {\n match u16::parse(data)? {\n 0 => Some(PlatformId::Unicode),\n 1 => Some(PlatformId::Macintosh),\n 2 => Some(PlatformId::Iso),\n 3 => Some(PlatformId::Windows),\n 4 => Some(PlatformId::Custom),\n _ => None,\n }\n }\n}\n\n#[inline]\nfn is_unicode_encoding(platform_id: PlatformId, encoding_id: u16) -> bool {\n // https://docs.microsoft.com/en-us/typography/opentype/spec/name#windows-encoding-ids\n const WINDOWS_SYMBOL_ENCODING_ID: u16 = 0;\n const WINDOWS_UNICODE_BMP_ENCODING_ID: u16 = 1;\n\n match platform_id {\n PlatformId::Unicode => true,\n PlatformId::Windows => matches!(\n encoding_id,\n WINDOWS_SYMBOL_ENCODING_ID | WINDOWS_UNICODE_BMP_ENCODING_ID\n ),\n _ => false,\n }\n}\n\n#[derive(Clone, Copy)]\nstruct NameRecord {\n platform_id: PlatformId,\n encoding_id: u16,\n language_id: u16,\n name_id: u16,\n length: u16,\n offset: Offset16,\n}\n\nimpl FromData for NameRecord {\n const SIZE: usize = 12;\n\n #[inline]\n fn parse(data: &[u8]) -> Option {\n let mut s = Stream::new(data);\n Some(NameRecord {\n platform_id: s.read::()?,\n encoding_id: s.read::()?,\n language_id: s.read::()?,\n name_id: s.read::()?,\n length: s.read::()?,\n offset: s.read::()?,\n })\n }\n}\n\n/// A [Name Record](https://docs.microsoft.com/en-us/typography/opentype/spec/name#name-records).\n#[derive(Clone, Copy)]\npub struct Name<'a> {\n /// A platform ID.\n pub platform_id: PlatformId,\n /// A platform-specific encoding ID.\n pub encoding_id: u16,\n /// A language ID.\n pub language_id: u16,\n /// A [Name ID](https://docs.microsoft.com/en-us/typography/opentype/spec/name#name-ids).\n ///\n /// A predefined list of ID's can be found in the [`name_id`](name_id/index.html) module.\n pub name_id: u16,\n /// A raw name data.\n ///\n /// Can be in any encoding. Can be empty.\n pub name: &'a [u8],\n}\n\nimpl<'a> Name<'a> {\n /// Returns the Name's data as a UTF-8 string.\n ///\n /// Only Unicode names are supported. And since they are stored as UTF-16BE,\n /// we can't return `&str` and have to allocate a `String`.\n ///\n /// Supports:\n /// - Unicode Platform ID\n /// - Windows Platform ID + Symbol\n /// - Windows Platform ID + Unicode BMP\n #[cfg(feature = \"std\")]\n #[inline(never)]\n pub fn to_string(&self) -> Option {\n if self.is_unicode() {\n self.name_from_utf16_be()\n } else {\n None\n }\n }\n\n /// Checks that the current Name data has a Unicode encoding.\n #[inline]\n pub fn is_unicode(&self) -> bool {\n is_unicode_encoding(self.platform_id, self.encoding_id)\n }\n\n #[cfg(feature = \"std\")]\n #[inline(never)]\n fn name_from_utf16_be(&self) -> Option {\n let mut name: Vec = Vec::new();\n for c in LazyArray16::::new(self.name) {\n name.push(c);\n }\n\n String::from_utf16(&name).ok()\n }\n\n /// Returns a Name language.\n pub fn language(&self) -> Language {\n if self.platform_id == PlatformId::Windows {\n Language::windows_language(self.language_id)\n } else if self.platform_id == PlatformId::Macintosh\n && self.encoding_id == 0\n && self.language_id == 0\n {\n Language::English_UnitedStates\n } else {\n Language::Unknown\n }\n }\n}\n\n#[cfg(feature = \"std\")]\nimpl<'a> core::fmt::Debug for Name<'a> {\n fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {\n let name = self.to_string();\n f.debug_struct(\"Name\")\n .field(\"name\", &name.as_deref().unwrap_or(\"unsupported encoding\"))\n .field(\"platform_id\", &self.platform_id)\n .field(\"encoding_id\", &self.encoding_id)\n .field(\"language_id\", &self.language_id)\n .field(\"language\", &self.language())\n .field(\"name_id\", &self.name_id)\n .finish()\n }\n}\n\n#[cfg(not(feature = \"std\"))]\nimpl<'a> core::fmt::Debug for Name<'a> {\n fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {\n f.debug_struct(\"Name\")\n .field(\"name\", &self.name)\n .field(\"platform_id\", &self.platform_id)\n .field(\"encoding_id\", &self.encoding_id)\n .field(\"language_id\", &self.language_id)\n .field(\"language\", &self.language())\n .field(\"name_id\", &self.name_id)\n .finish()\n }\n}\n\n/// A list of face names.\n#[derive(Clone, Copy, Default)]\npub struct Names<'a> {\n records: LazyArray16<'a, NameRecord>,\n storage: &'a [u8],\n}\n\nimpl<'a> Names<'a> {\n /// Returns a name at index.\n pub fn get(&self, index: u16) -> Option> {\n let record = self.records.get(index)?;\n let name_start = record.offset.to_usize();\n let name_end = name_start + usize::from(record.length);\n let name = self.storage.get(name_start..name_end)?;\n Some(Name {\n platform_id: record.platform_id,\n encoding_id: record.encoding_id,\n language_id: record.language_id,\n name_id: record.name_id,\n name,\n })\n }\n\n /// Returns a number of name records.\n pub fn len(&self) -> u16 {\n self.records.len()\n }\n\n /// Checks if there are any name records.\n pub fn is_empty(&self) -> bool {\n self.records.is_empty()\n }\n}\n\nimpl core::fmt::Debug for Names<'_> {\n fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {\n write!(f, \"Names {{ ... }}\")\n }\n}\n\nimpl<'a> IntoIterator for Names<'a> {\n type Item = Name<'a>;\n type IntoIter = NamesIter<'a>;\n\n #[inline]\n fn into_iter(self) -> Self::IntoIter {\n NamesIter {\n names: self,\n index: 0,\n }\n }\n}\n\n/// An iterator over face names.\n#[derive(Clone, Copy)]\n#[allow(missing_debug_implementations)]\npub struct NamesIter<'a> {\n names: Names<'a>,\n index: u16,\n}\n\nimpl<'a> Iterator for NamesIter<'a> {\n type Item = Name<'a>;\n\n fn next(&mut self) -> Option {\n if self.index < self.names.len() {\n self.index += 1;\n self.names.get(self.index - 1)\n } else {\n None\n }\n }\n\n #[inline]\n fn count(self) -> usize {\n usize::from(self.names.len().saturating_sub(self.index))\n }\n}\n\n/// A [Naming Table](\n/// https://docs.microsoft.com/en-us/typography/opentype/spec/name).\n#[derive(Clone, Copy, Default, Debug)]\npub struct Table<'a> {\n /// A list of names.\n pub names: Names<'a>,\n}\n\nimpl<'a> Table<'a> {\n /// Parses a table from raw data.\n pub fn parse(data: &'a [u8]) -> Option {\n // https://docs.microsoft.com/en-us/typography/opentype/spec/name#naming-table-format-1\n const LANG_TAG_RECORD_SIZE: u16 = 4;\n\n let mut s = Stream::new(data);\n let version = s.read::()?;\n let count = s.read::()?;\n let storage_offset = s.read::()?.to_usize();\n\n if version == 0 {\n // Do nothing.\n } else if version == 1 {\n let lang_tag_count = s.read::()?;\n let lang_tag_len = lang_tag_count.checked_mul(LANG_TAG_RECORD_SIZE)?;\n s.advance(usize::from(lang_tag_len)); // langTagRecords\n } else {\n // Unsupported version.\n return None;\n }\n\n let records = s.read_array16::(count)?;\n\n if s.offset() < storage_offset {\n s.advance(storage_offset - s.offset());\n }\n\n let storage = s.tail()?;\n\n Some(Table {\n names: Names { records, storage },\n })\n }\n}\n" }, { "package": "ttf-parser@0.18.1", "sourcePath": "src/tables/post.rs", "sha256": "8736dfc782c57e17690b1aed11020e49709b4580f34e989b7263b4e9311648b8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "//! A [PostScript Table](\n//! https://docs.microsoft.com/en-us/typography/opentype/spec/post) implementation.\n\nuse crate::parser::{Fixed, LazyArray16, Stream};\n#[cfg(feature = \"glyph-names\")]\nuse crate::GlyphId;\nuse crate::LineMetrics;\n\nconst ITALIC_ANGLE_OFFSET: usize = 4;\nconst UNDERLINE_POSITION_OFFSET: usize = 8;\nconst UNDERLINE_THICKNESS_OFFSET: usize = 10;\nconst IS_FIXED_PITCH_OFFSET: usize = 12;\n\n// https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6post.html\n/// A list of Macintosh glyph names.\n#[cfg(feature = \"glyph-names\")]\nconst MACINTOSH_NAMES: &[&str] = &[\n \".notdef\",\n \".null\",\n \"nonmarkingreturn\",\n \"space\",\n \"exclam\",\n \"quotedbl\",\n \"numbersign\",\n \"dollar\",\n \"percent\",\n \"ampersand\",\n \"quotesingle\",\n \"parenleft\",\n \"parenright\",\n \"asterisk\",\n \"plus\",\n \"comma\",\n \"hyphen\",\n \"period\",\n \"slash\",\n \"zero\",\n \"one\",\n \"two\",\n \"three\",\n \"four\",\n \"five\",\n \"six\",\n \"seven\",\n \"eight\",\n \"nine\",\n \"colon\",\n \"semicolon\",\n \"less\",\n \"equal\",\n \"greater\",\n \"question\",\n \"at\",\n \"A\",\n \"B\",\n \"C\",\n \"D\",\n \"E\",\n \"F\",\n \"G\",\n \"H\",\n \"I\",\n \"J\",\n \"K\",\n \"L\",\n \"M\",\n \"N\",\n \"O\",\n \"P\",\n \"Q\",\n \"R\",\n \"S\",\n \"T\",\n \"U\",\n \"V\",\n \"W\",\n \"X\",\n \"Y\",\n \"Z\",\n \"bracketleft\",\n \"backslash\",\n \"bracketright\",\n \"asciicircum\",\n \"underscore\",\n \"grave\",\n \"a\",\n \"b\",\n \"c\",\n \"d\",\n \"e\",\n \"f\",\n \"g\",\n \"h\",\n \"i\",\n \"j\",\n \"k\",\n \"l\",\n \"m\",\n \"n\",\n \"o\",\n \"p\",\n \"q\",\n \"r\",\n \"s\",\n \"t\",\n \"u\",\n \"v\",\n \"w\",\n \"x\",\n \"y\",\n \"z\",\n \"braceleft\",\n \"bar\",\n \"braceright\",\n \"asciitilde\",\n \"Adieresis\",\n \"Aring\",\n \"Ccedilla\",\n \"Eacute\",\n \"Ntilde\",\n \"Odieresis\",\n \"Udieresis\",\n \"aacute\",\n \"agrave\",\n \"acircumflex\",\n \"adieresis\",\n \"atilde\",\n \"aring\",\n \"ccedilla\",\n \"eacute\",\n \"egrave\",\n \"ecircumflex\",\n \"edieresis\",\n \"iacute\",\n \"igrave\",\n \"icircumflex\",\n \"idieresis\",\n \"ntilde\",\n \"oacute\",\n \"ograve\",\n \"ocircumflex\",\n \"odieresis\",\n \"otilde\",\n \"uacute\",\n \"ugrave\",\n \"ucircumflex\",\n \"udieresis\",\n \"dagger\",\n \"degree\",\n \"cent\",\n \"sterling\",\n \"section\",\n \"bullet\",\n \"paragraph\",\n \"germandbls\",\n \"registered\",\n \"copyright\",\n \"trademark\",\n \"acute\",\n \"dieresis\",\n \"notequal\",\n \"AE\",\n \"Oslash\",\n \"infinity\",\n \"plusminus\",\n \"lessequal\",\n \"greaterequal\",\n \"yen\",\n \"mu\",\n \"partialdiff\",\n \"summation\",\n \"product\",\n \"pi\",\n \"integral\",\n \"ordfeminine\",\n \"ordmasculine\",\n \"Omega\",\n \"ae\",\n \"oslash\",\n \"questiondown\",\n \"exclamdown\",\n \"logicalnot\",\n \"radical\",\n \"florin\",\n \"approxequal\",\n \"Delta\",\n \"guillemotleft\",\n \"guillemotright\",\n \"ellipsis\",\n \"nonbreakingspace\",\n \"Agrave\",\n \"Atilde\",\n \"Otilde\",\n \"OE\",\n \"oe\",\n \"endash\",\n \"emdash\",\n \"quotedblleft\",\n \"quotedblright\",\n \"quoteleft\",\n \"quoteright\",\n \"divide\",\n \"lozenge\",\n \"ydieresis\",\n \"Ydieresis\",\n \"fraction\",\n \"currency\",\n \"guilsinglleft\",\n \"guilsinglright\",\n \"fi\",\n \"fl\",\n \"daggerdbl\",\n \"periodcentered\",\n \"quotesinglbase\",\n \"quotedblbase\",\n \"perthousand\",\n \"Acircumflex\",\n \"Ecircumflex\",\n \"Aacute\",\n \"Edieresis\",\n \"Egrave\",\n \"Iacute\",\n \"Icircumflex\",\n \"Idieresis\",\n \"Igrave\",\n \"Oacute\",\n \"Ocircumflex\",\n \"apple\",\n \"Ograve\",\n \"Uacute\",\n \"Ucircumflex\",\n \"Ugrave\",\n \"dotlessi\",\n \"circumflex\",\n \"tilde\",\n \"macron\",\n \"breve\",\n \"dotaccent\",\n \"ring\",\n \"cedilla\",\n \"hungarumlaut\",\n \"ogonek\",\n \"caron\",\n \"Lslash\",\n \"lslash\",\n \"Scaron\",\n \"scaron\",\n \"Zcaron\",\n \"zcaron\",\n \"brokenbar\",\n \"Eth\",\n \"eth\",\n \"Yacute\",\n \"yacute\",\n \"Thorn\",\n \"thorn\",\n \"minus\",\n \"multiply\",\n \"onesuperior\",\n \"twosuperior\",\n \"threesuperior\",\n \"onehalf\",\n \"onequarter\",\n \"threequarters\",\n \"franc\",\n \"Gbreve\",\n \"gbreve\",\n \"Idotaccent\",\n \"Scedilla\",\n \"scedilla\",\n \"Cacute\",\n \"cacute\",\n \"Ccaron\",\n \"ccaron\",\n \"dcroat\",\n];\n\n/// An iterator over glyph names.\n///\n/// The `post` table doesn't provide the glyph names count,\n/// so we have to simply iterate over all of them to find it out.\n#[derive(Clone, Copy, Default)]\npub struct Names<'a> {\n data: &'a [u8],\n offset: usize,\n}\n\nimpl core::fmt::Debug for Names<'_> {\n fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {\n write!(f, \"Names {{ ... }}\")\n }\n}\n\nimpl<'a> Iterator for Names<'a> {\n type Item = &'a str;\n\n fn next(&mut self) -> Option {\n // Glyph names are stored as Pascal Strings.\n // Meaning u8 (len) + [u8] (data).\n\n if self.offset >= self.data.len() {\n return None;\n }\n\n let len = self.data[self.offset];\n self.offset += 1;\n\n // An empty name is an error.\n if len == 0 {\n return None;\n }\n\n let name = self.data.get(self.offset..self.offset + usize::from(len))?;\n self.offset += usize::from(len);\n core::str::from_utf8(name).ok()\n }\n}\n\n/// A [PostScript Table](https://docs.microsoft.com/en-us/typography/opentype/spec/post).\n#[derive(Clone, Copy, Debug)]\npub struct Table<'a> {\n /// Italic angle in counter-clockwise degrees from the vertical.\n pub italic_angle: f32,\n /// Underline metrics.\n pub underline_metrics: LineMetrics,\n /// Flag that indicates that the font is monospaced.\n pub is_monospaced: bool,\n\n glyph_indexes: LazyArray16<'a, u16>,\n names_data: &'a [u8],\n}\n\nimpl<'a> Table<'a> {\n /// Parses a table from raw data.\n pub fn parse(data: &'a [u8]) -> Option {\n // Do not check the exact length, because some fonts include\n // padding in table's length in table records, which is incorrect.\n if data.len() < 32 {\n return None;\n }\n\n let version = Stream::new(data).read::()?;\n if !(version == 0x00010000\n || version == 0x00020000\n || version == 0x00025000\n || version == 0x00030000\n || version == 0x00040000)\n {\n return None;\n }\n\n let italic_angle = Stream::read_at::(data, ITALIC_ANGLE_OFFSET)?.0;\n\n let underline_metrics = LineMetrics {\n position: Stream::read_at::(data, UNDERLINE_POSITION_OFFSET)?,\n thickness: Stream::read_at::(data, UNDERLINE_THICKNESS_OFFSET)?,\n };\n\n let is_monospaced = Stream::read_at::(data, IS_FIXED_PITCH_OFFSET)? != 0;\n\n let mut names_data: &[u8] = &[];\n let mut glyph_indexes = LazyArray16::default();\n // Only version 2.0 of the table has data at the end.\n if version == 0x00020000 {\n let mut s = Stream::new_at(data, 32)?;\n let indexes_count = s.read::()?;\n glyph_indexes = s.read_array16::(indexes_count)?;\n names_data = s.tail()?;\n }\n\n Some(Table {\n italic_angle,\n underline_metrics,\n is_monospaced,\n names_data,\n glyph_indexes,\n })\n }\n\n /// Returns a glyph name by ID.\n #[cfg(feature = \"glyph-names\")]\n pub fn glyph_name(&self, glyph_id: GlyphId) -> Option<&'a str> {\n let mut index = self.glyph_indexes.get(glyph_id.0)?;\n\n // 'If the name index is between 0 and 257, treat the name index\n // as a glyph index in the Macintosh standard order.'\n if usize::from(index) < MACINTOSH_NAMES.len() {\n Some(MACINTOSH_NAMES[usize::from(index)])\n } else {\n // 'If the name index is between 258 and 65535, then subtract 258 and use that\n // to index into the list of Pascal strings at the end of the table.'\n index -= MACINTOSH_NAMES.len() as u16;\n self.names().nth(usize::from(index))\n }\n }\n\n /// Returns a glyph ID by a name.\n #[cfg(feature = \"glyph-names\")]\n pub fn glyph_index_by_name(&self, name: &str) -> Option {\n let id = if let Some(index) = MACINTOSH_NAMES.iter().position(|n| *n == name) {\n self.glyph_indexes\n .into_iter()\n .position(|i| usize::from(i) == index)?\n } else {\n let mut index = self.names().position(|n| n == name)?;\n index += MACINTOSH_NAMES.len();\n self.glyph_indexes\n .into_iter()\n .position(|i| usize::from(i) == index)?\n };\n\n Some(GlyphId(id as u16))\n }\n\n /// Returns an iterator over glyph names.\n ///\n /// Default/predefined names are not included. Just the one in the font file.\n pub fn names(&self) -> Names<'a> {\n Names {\n data: self.names_data,\n offset: 0,\n }\n }\n}\n" }, { "package": "ttf-parser@0.19.2", "sourcePath": "README.md", "sha256": "827bef54227d335ccbc62b1ca799ada622dd0584c57a41a17175dd195d2fe11a", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "## ttf-parser\n![Build Status](https://github.com/RazrFalcon/ttf-parser/workflows/Rust/badge.svg)\n[![Crates.io](https://img.shields.io/crates/v/ttf-parser.svg)](https://crates.io/crates/ttf-parser)\n[![Documentation](https://docs.rs/ttf-parser/badge.svg)](https://docs.rs/ttf-parser)\n[![Rust 1.42+](https://img.shields.io/badge/rust-1.42+-orange.svg)](https://www.rust-lang.org)\n![](https://img.shields.io/badge/unsafe-forbidden-brightgreen.svg)\n\nA high-level, safe, zero-allocation TrueType font parser.\n\nSupports [TrueType](https://docs.microsoft.com/en-us/typography/truetype/),\n[OpenType](https://docs.microsoft.com/en-us/typography/opentype/spec/)\nand [AAT](https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6AATIntro.html)\nfonts.\n\nCan be used as Rust and as C library.\n\n### Features\n\n- A high-level API for most common properties, hiding all parsing and data resolving logic.\n- A low-level, but safe API to access TrueType tables data.\n- Highly configurable. You can disable most of the features, reducing binary size.\n You can also parse TrueType tables separately, without loading the whole font/face.\n- Zero heap allocations.\n- Zero unsafe.\n- Zero dependencies.\n- `no_std`/WASM compatible.\n- A basic [C API](./c-api).\n- Fast.\n- Stateless. All parsing methods are immutable.\n- Simple and maintainable code (no magic numbers).\n\n### Safety\n\n- The library must not panic. Any panic considered as a critical bug and should be reported.\n- The library forbids unsafe code.\n- No heap allocations, so crash due to OOM is not possible.\n- All recursive methods have a depth limit.\n- Technically, should use less than 64KiB of stack in the worst case scenario.\n- Most of arithmetic operations are checked.\n- Most of numeric casts are checked.\n\n### Alternatives\n\nIt's very hard to compare different libraries, so we are using table-based comparison.\nThere are roughly three types of TrueType tables:\n\n- A table with a list of properties (like `head`, `OS/2`, etc.).
\n If a library tries to parse it at all then we mark it as supported.\n- A table that contains a single type of data (`glyf`, `CFF` (kinda), `hmtx`, etc.).
\n Can only be supported or not.\n- A table that contains multiple subtables (`cmap`, `kern`, `GPOS`, etc.).
\n Can be partially supported and we note which subtables are actually supported.\n\n| Feature/Library | ttf-parser | FreeType | stb_truetype |\n| ----------------- | :--------------------: | :-----------------: | :----------------------------: |\n| Memory safe | ✓ | | |\n| Thread safe | ✓ | | ~ (mostly reentrant) |\n| Zero allocation | ✓ | | |\n| Variable fonts | ✓ | ✓ | |\n| Rendering | -1 | ✓ | ~ (very primitive) |\n| `ankr` table | ✓ | | |\n| `avar` table | ✓ | ✓ | |\n| `bdat` table | ~ (no 4) | ✓ | |\n| `bloc` table | ✓ | ✓ | |\n| `CBDT` table | ~ (no 8, 9) | ✓ | |\n| `CBLC` table | ✓ | ✓ | |\n| `COLR` table | | ✓ | |\n| `CPAL` table | | ✓ | |\n| `CFF ` table | ✓ | ✓ | ~ (no `seac` support) |\n| `CFF2` table | ✓ | ✓ | |\n| `cmap` table | ~ (no 8) | ✓ | ~ (no 2,8,10,14; Unicode-only) |\n| `EBDT` table | ~ (no 8, 9) | ✓ | |\n| `EBLC` table | ✓ | ✓ | |\n| `feat` table | ✓ | | |\n| `fvar` table | ✓ | ✓ | |\n| `gasp` table | | ✓ | |\n| `GDEF` table | ~ | | |\n| `glyf` table | ~2 | ✓ | ~2 |\n| `GPOS` table | ✓ | | ~ (only 2) |\n| `GSUB` table | ✓ | | |\n| `gvar` table | ✓ | ✓ | |\n| `head` table | ✓ | ✓ | ✓ |\n| `hhea` table | ✓ | ✓ | ✓ |\n| `hmtx` table | ✓ | ✓ | ✓ |\n| `HVAR` table | ✓ | ✓ | |\n| `kern` table | ✓ | ~ (only 0) | ~ (only 0) |\n| `kerx` table | ✓ | | |\n| `MATH` table | ✓ | | |\n| `maxp` table | ✓ | ✓ | ✓ |\n| `morx` table | ✓ | | |\n| `MVAR` table | ✓ | ✓ | |\n| `name` table | ✓ | ✓ | |\n| `OS/2` table | ✓ | ✓ | |\n| `post` table | ✓ | ✓ | |\n| `sbix` table | ~ (PNG only) | ~ (PNG only) | |\n| `SVG ` table | ✓ | ✓ | ✓ |\n| `trak` table | ✓ | | |\n| `vhea` table | ✓ | ✓ | |\n| `vmtx` table | ✓ | ✓ | |\n| `VORG` table | ✓ | ✓ | |\n| `VVAR` table | ✓ | ✓ | |\n| Language | Rust + C API | C | C |\n| Tested version | 0.17.0 | 2.12.0 | 1.24 |\n| License | MIT / Apache-2.0 | FTL / GPLv2 | public domain |\n\nLegend:\n\n- ✓ - supported\n- ~ - partial\n- *nothing* - not supported\n\nNotes:\n\n1. While `ttf-parser` doesn't support rendering by itself,\n there are multiple rendering libraries on top of it:\n [rusttype](https://gitlab.redox-os.org/redox-os/rusttype),\n [ab-glyph](https://github.com/alexheretic/ab-glyph)\n and [fontdue](https://github.com/mooman219/fontdue).\n2. Matching points are not supported.\n\n### Performance\n\nTrueType fonts designed for fast querying, so most of the methods are very fast.\nThe main exception is glyph outlining. Glyphs can be stored using two different methods:\nusing [Glyph Data Format](https://docs.microsoft.com/en-us/typography/opentype/spec/glyf)\nand [Compact Font Format](http://wwwimages.adobe.com/content/dam/Adobe/en/devnet/font/pdfs/5176.CFF.pdf) (pdf).\nThe first one is fairly simple which makes it faster to process.\nThe second one is basically a tiny language with a stack-based VM, which makes it way harder to process.\n\nThe [benchmark](./benches/outline/) tests how long it takes to outline all glyphs in a font.\n\nx86 (AMD 3700X)\n\n| Table/Library | ttf-parser | FreeType | stb_truetype |\n| ------------- | -------------: | ---------: | -------------: |\n| `glyf` | `0.901 ms` | `1.171 ms` | **`0.675 ms`** |\n| `gvar` | **`2.972 ms`** | `4.132 ms` | - |\n| `CFF` | **`1.197 ms`** | `5.647 ms` | `2.813 ms` |\n| `CFF2` | **`1.968 ms`** | `6.392 ms` | - |\n\nARM (Apple M1)\n\n| Table/Library | ttf-parser | FreeType | stb_truetype |\n| ------------- | -------------: | ---------: | -------------: |\n| `glyf` | **`0.550 ms`** | `0.854 ms` | `0.703 ms` |\n| `gvar` | **`2.270 ms`** | `4.594 ms` | - |\n| `CFF` | **`1.054 ms`** | `5.223 ms` | `3.262 ms` |\n| `CFF2` | **`1.765 ms`** | `5.995 ms` | - |\n\n**Note:** FreeType is surprisingly slow, so I'm worried that I've messed something up.\n\nAnd here are some methods benchmarks:\n\n```text\ntest outline_glyph_276_from_cff2 ... bench: 867 ns/iter (+/- 15)\ntest from_data_otf_cff ... bench: 968 ns/iter (+/- 13)\ntest from_data_otf_cff2 ... bench: 887 ns/iter (+/- 25)\ntest outline_glyph_276_from_cff ... bench: 678 ns/iter (+/- 41)\ntest outline_glyph_276_from_glyf ... bench: 649 ns/iter (+/- 11)\ntest outline_glyph_8_from_cff2 ... bench: 534 ns/iter (+/- 14)\ntest from_data_ttf ... bench: 467 ns/iter (+/- 11)\ntest glyph_name_post_276 ... bench: 223 ns/iter (+/- 5)\ntest outline_glyph_8_from_cff ... bench: 315 ns/iter (+/- 13)\ntest outline_glyph_8_from_glyf ... bench: 291 ns/iter (+/- 5)\ntest family_name ... bench: 183 ns/iter (+/- 102)\ntest glyph_name_cff_276 ... bench: 62 ns/iter (+/- 1)\ntest glyph_index_u41 ... bench: 16 ns/iter (+/- 0)\ntest glyph_name_cff_8 ... bench: 5 ns/iter (+/- 0)\ntest glyph_name_post_8 ... bench: 2 ns/iter (+/- 0)\ntest subscript_metrics ... bench: 2 ns/iter (+/- 0)\ntest glyph_hor_advance ... bench: 2 ns/iter (+/- 0)\ntest glyph_hor_side_bearing ... bench: 2 ns/iter (+/- 0)\ntest glyph_name_8 ... bench: 1 ns/iter (+/- 0)\ntest ascender ... bench: 1 ns/iter (+/- 0)\ntest underline_metrics ... bench: 1 ns/iter (+/- 0)\ntest strikeout_metrics ... bench: 1 ns/iter (+/- 0)\ntest x_height ... bench: 1 ns/iter (+/- 0)\ntest units_per_em ... bench: 0.5 ns/iter (+/- 0)\ntest width ... bench: 0.2 ns/iter (+/- 0)\n```\n\n### License\n\nLicensed under either of\n\n- Apache License, Version 2.0\n ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n- MIT license\n ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "package": "ttf-parser@0.19.2", "sourcePath": "src/tables/cff/std_names.rs", "sha256": "28f192e64551773c97d0457ddabc774cfabe535585a2ef65f37ca48022221923", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "pub const STANDARD_NAMES: &[&str] = &[\n \".notdef\",\n \"space\",\n \"exclam\",\n \"quotedbl\",\n \"numbersign\",\n \"dollar\",\n \"percent\",\n \"ampersand\",\n \"quoteright\",\n \"parenleft\",\n \"parenright\",\n \"asterisk\",\n \"plus\",\n \"comma\",\n \"hyphen\",\n \"period\",\n \"slash\",\n \"zero\",\n \"one\",\n \"two\",\n \"three\",\n \"four\",\n \"five\",\n \"six\",\n \"seven\",\n \"eight\",\n \"nine\",\n \"colon\",\n \"semicolon\",\n \"less\",\n \"equal\",\n \"greater\",\n \"question\",\n \"at\",\n \"A\",\n \"B\",\n \"C\",\n \"D\",\n \"E\",\n \"F\",\n \"G\",\n \"H\",\n \"I\",\n \"J\",\n \"K\",\n \"L\",\n \"M\",\n \"N\",\n \"O\",\n \"P\",\n \"Q\",\n \"R\",\n \"S\",\n \"T\",\n \"U\",\n \"V\",\n \"W\",\n \"X\",\n \"Y\",\n \"Z\",\n \"bracketleft\",\n \"backslash\",\n \"bracketright\",\n \"asciicircum\",\n \"underscore\",\n \"quoteleft\",\n \"a\",\n \"b\",\n \"c\",\n \"d\",\n \"e\",\n \"f\",\n \"g\",\n \"h\",\n \"i\",\n \"j\",\n \"k\",\n \"l\",\n \"m\",\n \"n\",\n \"o\",\n \"p\",\n \"q\",\n \"r\",\n \"s\",\n \"t\",\n \"u\",\n \"v\",\n \"w\",\n \"x\",\n \"y\",\n \"z\",\n \"braceleft\",\n \"bar\",\n \"braceright\",\n \"asciitilde\",\n \"exclamdown\",\n \"cent\",\n \"sterling\",\n \"fraction\",\n \"yen\",\n \"florin\",\n \"section\",\n \"currency\",\n \"quotesingle\",\n \"quotedblleft\",\n \"guillemotleft\",\n \"guilsinglleft\",\n \"guilsinglright\",\n \"fi\",\n \"fl\",\n \"endash\",\n \"dagger\",\n \"daggerdbl\",\n \"periodcentered\",\n \"paragraph\",\n \"bullet\",\n \"quotesinglbase\",\n \"quotedblbase\",\n \"quotedblright\",\n \"guillemotright\",\n \"ellipsis\",\n \"perthousand\",\n \"questiondown\",\n \"grave\",\n \"acute\",\n \"circumflex\",\n \"tilde\",\n \"macron\",\n \"breve\",\n \"dotaccent\",\n \"dieresis\",\n \"ring\",\n \"cedilla\",\n \"hungarumlaut\",\n \"ogonek\",\n \"caron\",\n \"emdash\",\n \"AE\",\n \"ordfeminine\",\n \"Lslash\",\n \"Oslash\",\n \"OE\",\n \"ordmasculine\",\n \"ae\",\n \"dotlessi\",\n \"lslash\",\n \"oslash\",\n \"oe\",\n \"germandbls\",\n \"onesuperior\",\n \"logicalnot\",\n \"mu\",\n \"trademark\",\n \"Eth\",\n \"onehalf\",\n \"plusminus\",\n \"Thorn\",\n \"onequarter\",\n \"divide\",\n \"brokenbar\",\n \"degree\",\n \"thorn\",\n \"threequarters\",\n \"twosuperior\",\n \"registered\",\n \"minus\",\n \"eth\",\n \"multiply\",\n \"threesuperior\",\n \"copyright\",\n \"Aacute\",\n \"Acircumflex\",\n \"Adieresis\",\n \"Agrave\",\n \"Aring\",\n \"Atilde\",\n \"Ccedilla\",\n \"Eacute\",\n \"Ecircumflex\",\n \"Edieresis\",\n \"Egrave\",\n \"Iacute\",\n \"Icircumflex\",\n \"Idieresis\",\n \"Igrave\",\n \"Ntilde\",\n \"Oacute\",\n \"Ocircumflex\",\n \"Odieresis\",\n \"Ograve\",\n \"Otilde\",\n \"Scaron\",\n \"Uacute\",\n \"Ucircumflex\",\n \"Udieresis\",\n \"Ugrave\",\n \"Yacute\",\n \"Ydieresis\",\n \"Zcaron\",\n \"aacute\",\n \"acircumflex\",\n \"adieresis\",\n \"agrave\",\n \"aring\",\n \"atilde\",\n \"ccedilla\",\n \"eacute\",\n \"ecircumflex\",\n \"edieresis\",\n \"egrave\",\n \"iacute\",\n \"icircumflex\",\n \"idieresis\",\n \"igrave\",\n \"ntilde\",\n \"oacute\",\n \"ocircumflex\",\n \"odieresis\",\n \"ograve\",\n \"otilde\",\n \"scaron\",\n \"uacute\",\n \"ucircumflex\",\n \"udieresis\",\n \"ugrave\",\n \"yacute\",\n \"ydieresis\",\n \"zcaron\",\n \"exclamsmall\",\n \"Hungarumlautsmall\",\n \"dollaroldstyle\",\n \"dollarsuperior\",\n \"ampersandsmall\",\n \"Acutesmall\",\n \"parenleftsuperior\",\n \"parenrightsuperior\",\n \"twodotenleader\",\n \"onedotenleader\",\n \"zerooldstyle\",\n \"oneoldstyle\",\n \"twooldstyle\",\n \"threeoldstyle\",\n \"fouroldstyle\",\n \"fiveoldstyle\",\n \"sixoldstyle\",\n \"sevenoldstyle\",\n \"eightoldstyle\",\n \"nineoldstyle\",\n \"commasuperior\",\n \"threequartersemdash\",\n \"periodsuperior\",\n \"questionsmall\",\n \"asuperior\",\n \"bsuperior\",\n \"centsuperior\",\n \"dsuperior\",\n \"esuperior\",\n \"isuperior\",\n \"lsuperior\",\n \"msuperior\",\n \"nsuperior\",\n \"osuperior\",\n \"rsuperior\",\n \"ssuperior\",\n \"tsuperior\",\n \"ff\",\n \"ffi\",\n \"ffl\",\n \"parenleftinferior\",\n \"parenrightinferior\",\n \"Circumflexsmall\",\n \"hyphensuperior\",\n \"Gravesmall\",\n \"Asmall\",\n \"Bsmall\",\n \"Csmall\",\n \"Dsmall\",\n \"Esmall\",\n \"Fsmall\",\n \"Gsmall\",\n \"Hsmall\",\n \"Ismall\",\n \"Jsmall\",\n \"Ksmall\",\n \"Lsmall\",\n \"Msmall\",\n \"Nsmall\",\n \"Osmall\",\n \"Psmall\",\n \"Qsmall\",\n \"Rsmall\",\n \"Ssmall\",\n \"Tsmall\",\n \"Usmall\",\n \"Vsmall\",\n \"Wsmall\",\n \"Xsmall\",\n \"Ysmall\",\n \"Zsmall\",\n \"colonmonetary\",\n \"onefitted\",\n \"rupiah\",\n \"Tildesmall\",\n \"exclamdownsmall\",\n \"centoldstyle\",\n \"Lslashsmall\",\n \"Scaronsmall\",\n \"Zcaronsmall\",\n \"Dieresissmall\",\n \"Brevesmall\",\n \"Caronsmall\",\n \"Dotaccentsmall\",\n \"Macronsmall\",\n \"figuredash\",\n \"hypheninferior\",\n \"Ogoneksmall\",\n \"Ringsmall\",\n \"Cedillasmall\",\n \"questiondownsmall\",\n \"oneeighth\",\n \"threeeighths\",\n \"fiveeighths\",\n \"seveneighths\",\n \"onethird\",\n \"twothirds\",\n \"zerosuperior\",\n \"foursuperior\",\n \"fivesuperior\",\n \"sixsuperior\",\n \"sevensuperior\",\n \"eightsuperior\",\n \"ninesuperior\",\n \"zeroinferior\",\n \"oneinferior\",\n \"twoinferior\",\n \"threeinferior\",\n \"fourinferior\",\n \"fiveinferior\",\n \"sixinferior\",\n \"seveninferior\",\n \"eightinferior\",\n \"nineinferior\",\n \"centinferior\",\n \"dollarinferior\",\n \"periodinferior\",\n \"commainferior\",\n \"Agravesmall\",\n \"Aacutesmall\",\n \"Acircumflexsmall\",\n \"Atildesmall\",\n \"Adieresissmall\",\n \"Aringsmall\",\n \"AEsmall\",\n \"Ccedillasmall\",\n \"Egravesmall\",\n \"Eacutesmall\",\n \"Ecircumflexsmall\",\n \"Edieresissmall\",\n \"Igravesmall\",\n \"Iacutesmall\",\n \"Icircumflexsmall\",\n \"Idieresissmall\",\n \"Ethsmall\",\n \"Ntildesmall\",\n \"Ogravesmall\",\n \"Oacutesmall\",\n \"Ocircumflexsmall\",\n \"Otildesmall\",\n \"Odieresissmall\",\n \"OEsmall\",\n \"Oslashsmall\",\n \"Ugravesmall\",\n \"Uacutesmall\",\n \"Ucircumflexsmall\",\n \"Udieresissmall\",\n \"Yacutesmall\",\n \"Thornsmall\",\n \"Ydieresissmall\",\n \"001.000\",\n \"001.001\",\n \"001.002\",\n \"001.003\",\n \"Black\",\n \"Bold\",\n \"Book\",\n \"Light\",\n \"Medium\",\n \"Regular\",\n \"Roman\",\n \"Semibold\",\n];\n" }, { "package": "ttf-parser@0.19.2", "sourcePath": "src/tables/name.rs", "sha256": "12ff8509c8fc14c12a244bc5b76ef43556d2d6800336e30d8ed9c96b9274bab8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "//! A [Naming Table](\n//! https://docs.microsoft.com/en-us/typography/opentype/spec/name) implementation.\n\n#[cfg(feature = \"std\")]\nuse std::string::String;\n#[cfg(feature = \"std\")]\nuse std::vec::Vec;\n\nuse crate::parser::{FromData, LazyArray16, Offset, Offset16, Stream};\nuse crate::Language;\n\n/// A list of [name ID](https://docs.microsoft.com/en-us/typography/opentype/spec/name#name-ids)'s.\npub mod name_id {\n #![allow(missing_docs)]\n\n pub const COPYRIGHT_NOTICE: u16 = 0;\n pub const FAMILY: u16 = 1;\n pub const SUBFAMILY: u16 = 2;\n pub const UNIQUE_ID: u16 = 3;\n pub const FULL_NAME: u16 = 4;\n pub const VERSION: u16 = 5;\n pub const POST_SCRIPT_NAME: u16 = 6;\n pub const TRADEMARK: u16 = 7;\n pub const MANUFACTURER: u16 = 8;\n pub const DESIGNER: u16 = 9;\n pub const DESCRIPTION: u16 = 10;\n pub const VENDOR_URL: u16 = 11;\n pub const DESIGNER_URL: u16 = 12;\n pub const LICENSE: u16 = 13;\n pub const LICENSE_URL: u16 = 14;\n // RESERVED = 15\n pub const TYPOGRAPHIC_FAMILY: u16 = 16;\n pub const TYPOGRAPHIC_SUBFAMILY: u16 = 17;\n pub const COMPATIBLE_FULL: u16 = 18;\n pub const SAMPLE_TEXT: u16 = 19;\n pub const POST_SCRIPT_CID: u16 = 20;\n pub const WWS_FAMILY: u16 = 21;\n pub const WWS_SUBFAMILY: u16 = 22;\n pub const LIGHT_BACKGROUND_PALETTE: u16 = 23;\n pub const DARK_BACKGROUND_PALETTE: u16 = 24;\n pub const VARIATIONS_POST_SCRIPT_NAME_PREFIX: u16 = 25;\n}\n\n/// A [platform ID](https://docs.microsoft.com/en-us/typography/opentype/spec/name#platform-ids).\n#[allow(missing_docs)]\n#[derive(Clone, Copy, PartialEq, Eq, Debug)]\npub enum PlatformId {\n Unicode,\n Macintosh,\n Iso,\n Windows,\n Custom,\n}\n\nimpl FromData for PlatformId {\n const SIZE: usize = 2;\n\n #[inline]\n fn parse(data: &[u8]) -> Option {\n match u16::parse(data)? {\n 0 => Some(PlatformId::Unicode),\n 1 => Some(PlatformId::Macintosh),\n 2 => Some(PlatformId::Iso),\n 3 => Some(PlatformId::Windows),\n 4 => Some(PlatformId::Custom),\n _ => None,\n }\n }\n}\n\n#[inline]\nfn is_unicode_encoding(platform_id: PlatformId, encoding_id: u16) -> bool {\n // https://docs.microsoft.com/en-us/typography/opentype/spec/name#windows-encoding-ids\n const WINDOWS_SYMBOL_ENCODING_ID: u16 = 0;\n const WINDOWS_UNICODE_BMP_ENCODING_ID: u16 = 1;\n\n match platform_id {\n PlatformId::Unicode => true,\n PlatformId::Windows => matches!(\n encoding_id,\n WINDOWS_SYMBOL_ENCODING_ID | WINDOWS_UNICODE_BMP_ENCODING_ID\n ),\n _ => false,\n }\n}\n\n#[derive(Clone, Copy)]\nstruct NameRecord {\n platform_id: PlatformId,\n encoding_id: u16,\n language_id: u16,\n name_id: u16,\n length: u16,\n offset: Offset16,\n}\n\nimpl FromData for NameRecord {\n const SIZE: usize = 12;\n\n #[inline]\n fn parse(data: &[u8]) -> Option {\n let mut s = Stream::new(data);\n Some(NameRecord {\n platform_id: s.read::()?,\n encoding_id: s.read::()?,\n language_id: s.read::()?,\n name_id: s.read::()?,\n length: s.read::()?,\n offset: s.read::()?,\n })\n }\n}\n\n/// A [Name Record](https://docs.microsoft.com/en-us/typography/opentype/spec/name#name-records).\n#[derive(Clone, Copy)]\npub struct Name<'a> {\n /// A platform ID.\n pub platform_id: PlatformId,\n /// A platform-specific encoding ID.\n pub encoding_id: u16,\n /// A language ID.\n pub language_id: u16,\n /// A [Name ID](https://docs.microsoft.com/en-us/typography/opentype/spec/name#name-ids).\n ///\n /// A predefined list of ID's can be found in the [`name_id`](name_id/index.html) module.\n pub name_id: u16,\n /// A raw name data.\n ///\n /// Can be in any encoding. Can be empty.\n pub name: &'a [u8],\n}\n\nimpl<'a> Name<'a> {\n /// Returns the Name's data as a UTF-8 string.\n ///\n /// Only Unicode names are supported. And since they are stored as UTF-16BE,\n /// we can't return `&str` and have to allocate a `String`.\n ///\n /// Supports:\n /// - Unicode Platform ID\n /// - Windows Platform ID + Symbol\n /// - Windows Platform ID + Unicode BMP\n #[cfg(feature = \"std\")]\n #[inline(never)]\n pub fn to_string(&self) -> Option {\n if self.is_unicode() {\n self.name_from_utf16_be()\n } else {\n None\n }\n }\n\n /// Checks that the current Name data has a Unicode encoding.\n #[inline]\n pub fn is_unicode(&self) -> bool {\n is_unicode_encoding(self.platform_id, self.encoding_id)\n }\n\n #[cfg(feature = \"std\")]\n #[inline(never)]\n fn name_from_utf16_be(&self) -> Option {\n let mut name: Vec = Vec::new();\n for c in LazyArray16::::new(self.name) {\n name.push(c);\n }\n\n String::from_utf16(&name).ok()\n }\n\n /// Returns a Name language.\n pub fn language(&self) -> Language {\n if self.platform_id == PlatformId::Windows {\n Language::windows_language(self.language_id)\n } else if self.platform_id == PlatformId::Macintosh\n && self.encoding_id == 0\n && self.language_id == 0\n {\n Language::English_UnitedStates\n } else {\n Language::Unknown\n }\n }\n}\n\n#[cfg(feature = \"std\")]\nimpl<'a> core::fmt::Debug for Name<'a> {\n fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {\n let name = self.to_string();\n f.debug_struct(\"Name\")\n .field(\"name\", &name.as_deref().unwrap_or(\"unsupported encoding\"))\n .field(\"platform_id\", &self.platform_id)\n .field(\"encoding_id\", &self.encoding_id)\n .field(\"language_id\", &self.language_id)\n .field(\"language\", &self.language())\n .field(\"name_id\", &self.name_id)\n .finish()\n }\n}\n\n#[cfg(not(feature = \"std\"))]\nimpl<'a> core::fmt::Debug for Name<'a> {\n fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {\n f.debug_struct(\"Name\")\n .field(\"name\", &self.name)\n .field(\"platform_id\", &self.platform_id)\n .field(\"encoding_id\", &self.encoding_id)\n .field(\"language_id\", &self.language_id)\n .field(\"language\", &self.language())\n .field(\"name_id\", &self.name_id)\n .finish()\n }\n}\n\n/// A list of face names.\n#[derive(Clone, Copy, Default)]\npub struct Names<'a> {\n records: LazyArray16<'a, NameRecord>,\n storage: &'a [u8],\n}\n\nimpl<'a> Names<'a> {\n /// Returns a name at index.\n pub fn get(&self, index: u16) -> Option> {\n let record = self.records.get(index)?;\n let name_start = record.offset.to_usize();\n let name_end = name_start + usize::from(record.length);\n let name = self.storage.get(name_start..name_end)?;\n Some(Name {\n platform_id: record.platform_id,\n encoding_id: record.encoding_id,\n language_id: record.language_id,\n name_id: record.name_id,\n name,\n })\n }\n\n /// Returns a number of name records.\n pub fn len(&self) -> u16 {\n self.records.len()\n }\n\n /// Checks if there are any name records.\n pub fn is_empty(&self) -> bool {\n self.records.is_empty()\n }\n}\n\nimpl core::fmt::Debug for Names<'_> {\n fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {\n write!(f, \"Names {{ ... }}\")\n }\n}\n\nimpl<'a> IntoIterator for Names<'a> {\n type Item = Name<'a>;\n type IntoIter = NamesIter<'a>;\n\n #[inline]\n fn into_iter(self) -> Self::IntoIter {\n NamesIter {\n names: self,\n index: 0,\n }\n }\n}\n\n/// An iterator over face names.\n#[derive(Clone, Copy)]\n#[allow(missing_debug_implementations)]\npub struct NamesIter<'a> {\n names: Names<'a>,\n index: u16,\n}\n\nimpl<'a> Iterator for NamesIter<'a> {\n type Item = Name<'a>;\n\n fn next(&mut self) -> Option {\n if self.index < self.names.len() {\n self.index += 1;\n self.names.get(self.index - 1)\n } else {\n None\n }\n }\n\n #[inline]\n fn count(self) -> usize {\n usize::from(self.names.len().saturating_sub(self.index))\n }\n}\n\n/// A [Naming Table](\n/// https://docs.microsoft.com/en-us/typography/opentype/spec/name).\n#[derive(Clone, Copy, Default, Debug)]\npub struct Table<'a> {\n /// A list of names.\n pub names: Names<'a>,\n}\n\nimpl<'a> Table<'a> {\n /// Parses a table from raw data.\n pub fn parse(data: &'a [u8]) -> Option {\n // https://docs.microsoft.com/en-us/typography/opentype/spec/name#naming-table-format-1\n const LANG_TAG_RECORD_SIZE: u16 = 4;\n\n let mut s = Stream::new(data);\n let version = s.read::()?;\n let count = s.read::()?;\n let storage_offset = s.read::()?.to_usize();\n\n if version == 0 {\n // Do nothing.\n } else if version == 1 {\n let lang_tag_count = s.read::()?;\n let lang_tag_len = lang_tag_count.checked_mul(LANG_TAG_RECORD_SIZE)?;\n s.advance(usize::from(lang_tag_len)); // langTagRecords\n } else {\n // Unsupported version.\n return None;\n }\n\n let records = s.read_array16::(count)?;\n\n if s.offset() < storage_offset {\n s.advance(storage_offset - s.offset());\n }\n\n let storage = s.tail()?;\n\n Some(Table {\n names: Names { records, storage },\n })\n }\n}\n" }, { "package": "ttf-parser@0.19.2", "sourcePath": "src/tables/post.rs", "sha256": "8736dfc782c57e17690b1aed11020e49709b4580f34e989b7263b4e9311648b8", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "//! A [PostScript Table](\n//! https://docs.microsoft.com/en-us/typography/opentype/spec/post) implementation.\n\nuse crate::parser::{Fixed, LazyArray16, Stream};\n#[cfg(feature = \"glyph-names\")]\nuse crate::GlyphId;\nuse crate::LineMetrics;\n\nconst ITALIC_ANGLE_OFFSET: usize = 4;\nconst UNDERLINE_POSITION_OFFSET: usize = 8;\nconst UNDERLINE_THICKNESS_OFFSET: usize = 10;\nconst IS_FIXED_PITCH_OFFSET: usize = 12;\n\n// https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6post.html\n/// A list of Macintosh glyph names.\n#[cfg(feature = \"glyph-names\")]\nconst MACINTOSH_NAMES: &[&str] = &[\n \".notdef\",\n \".null\",\n \"nonmarkingreturn\",\n \"space\",\n \"exclam\",\n \"quotedbl\",\n \"numbersign\",\n \"dollar\",\n \"percent\",\n \"ampersand\",\n \"quotesingle\",\n \"parenleft\",\n \"parenright\",\n \"asterisk\",\n \"plus\",\n \"comma\",\n \"hyphen\",\n \"period\",\n \"slash\",\n \"zero\",\n \"one\",\n \"two\",\n \"three\",\n \"four\",\n \"five\",\n \"six\",\n \"seven\",\n \"eight\",\n \"nine\",\n \"colon\",\n \"semicolon\",\n \"less\",\n \"equal\",\n \"greater\",\n \"question\",\n \"at\",\n \"A\",\n \"B\",\n \"C\",\n \"D\",\n \"E\",\n \"F\",\n \"G\",\n \"H\",\n \"I\",\n \"J\",\n \"K\",\n \"L\",\n \"M\",\n \"N\",\n \"O\",\n \"P\",\n \"Q\",\n \"R\",\n \"S\",\n \"T\",\n \"U\",\n \"V\",\n \"W\",\n \"X\",\n \"Y\",\n \"Z\",\n \"bracketleft\",\n \"backslash\",\n \"bracketright\",\n \"asciicircum\",\n \"underscore\",\n \"grave\",\n \"a\",\n \"b\",\n \"c\",\n \"d\",\n \"e\",\n \"f\",\n \"g\",\n \"h\",\n \"i\",\n \"j\",\n \"k\",\n \"l\",\n \"m\",\n \"n\",\n \"o\",\n \"p\",\n \"q\",\n \"r\",\n \"s\",\n \"t\",\n \"u\",\n \"v\",\n \"w\",\n \"x\",\n \"y\",\n \"z\",\n \"braceleft\",\n \"bar\",\n \"braceright\",\n \"asciitilde\",\n \"Adieresis\",\n \"Aring\",\n \"Ccedilla\",\n \"Eacute\",\n \"Ntilde\",\n \"Odieresis\",\n \"Udieresis\",\n \"aacute\",\n \"agrave\",\n \"acircumflex\",\n \"adieresis\",\n \"atilde\",\n \"aring\",\n \"ccedilla\",\n \"eacute\",\n \"egrave\",\n \"ecircumflex\",\n \"edieresis\",\n \"iacute\",\n \"igrave\",\n \"icircumflex\",\n \"idieresis\",\n \"ntilde\",\n \"oacute\",\n \"ograve\",\n \"ocircumflex\",\n \"odieresis\",\n \"otilde\",\n \"uacute\",\n \"ugrave\",\n \"ucircumflex\",\n \"udieresis\",\n \"dagger\",\n \"degree\",\n \"cent\",\n \"sterling\",\n \"section\",\n \"bullet\",\n \"paragraph\",\n \"germandbls\",\n \"registered\",\n \"copyright\",\n \"trademark\",\n \"acute\",\n \"dieresis\",\n \"notequal\",\n \"AE\",\n \"Oslash\",\n \"infinity\",\n \"plusminus\",\n \"lessequal\",\n \"greaterequal\",\n \"yen\",\n \"mu\",\n \"partialdiff\",\n \"summation\",\n \"product\",\n \"pi\",\n \"integral\",\n \"ordfeminine\",\n \"ordmasculine\",\n \"Omega\",\n \"ae\",\n \"oslash\",\n \"questiondown\",\n \"exclamdown\",\n \"logicalnot\",\n \"radical\",\n \"florin\",\n \"approxequal\",\n \"Delta\",\n \"guillemotleft\",\n \"guillemotright\",\n \"ellipsis\",\n \"nonbreakingspace\",\n \"Agrave\",\n \"Atilde\",\n \"Otilde\",\n \"OE\",\n \"oe\",\n \"endash\",\n \"emdash\",\n \"quotedblleft\",\n \"quotedblright\",\n \"quoteleft\",\n \"quoteright\",\n \"divide\",\n \"lozenge\",\n \"ydieresis\",\n \"Ydieresis\",\n \"fraction\",\n \"currency\",\n \"guilsinglleft\",\n \"guilsinglright\",\n \"fi\",\n \"fl\",\n \"daggerdbl\",\n \"periodcentered\",\n \"quotesinglbase\",\n \"quotedblbase\",\n \"perthousand\",\n \"Acircumflex\",\n \"Ecircumflex\",\n \"Aacute\",\n \"Edieresis\",\n \"Egrave\",\n \"Iacute\",\n \"Icircumflex\",\n \"Idieresis\",\n \"Igrave\",\n \"Oacute\",\n \"Ocircumflex\",\n \"apple\",\n \"Ograve\",\n \"Uacute\",\n \"Ucircumflex\",\n \"Ugrave\",\n \"dotlessi\",\n \"circumflex\",\n \"tilde\",\n \"macron\",\n \"breve\",\n \"dotaccent\",\n \"ring\",\n \"cedilla\",\n \"hungarumlaut\",\n \"ogonek\",\n \"caron\",\n \"Lslash\",\n \"lslash\",\n \"Scaron\",\n \"scaron\",\n \"Zcaron\",\n \"zcaron\",\n \"brokenbar\",\n \"Eth\",\n \"eth\",\n \"Yacute\",\n \"yacute\",\n \"Thorn\",\n \"thorn\",\n \"minus\",\n \"multiply\",\n \"onesuperior\",\n \"twosuperior\",\n \"threesuperior\",\n \"onehalf\",\n \"onequarter\",\n \"threequarters\",\n \"franc\",\n \"Gbreve\",\n \"gbreve\",\n \"Idotaccent\",\n \"Scedilla\",\n \"scedilla\",\n \"Cacute\",\n \"cacute\",\n \"Ccaron\",\n \"ccaron\",\n \"dcroat\",\n];\n\n/// An iterator over glyph names.\n///\n/// The `post` table doesn't provide the glyph names count,\n/// so we have to simply iterate over all of them to find it out.\n#[derive(Clone, Copy, Default)]\npub struct Names<'a> {\n data: &'a [u8],\n offset: usize,\n}\n\nimpl core::fmt::Debug for Names<'_> {\n fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {\n write!(f, \"Names {{ ... }}\")\n }\n}\n\nimpl<'a> Iterator for Names<'a> {\n type Item = &'a str;\n\n fn next(&mut self) -> Option {\n // Glyph names are stored as Pascal Strings.\n // Meaning u8 (len) + [u8] (data).\n\n if self.offset >= self.data.len() {\n return None;\n }\n\n let len = self.data[self.offset];\n self.offset += 1;\n\n // An empty name is an error.\n if len == 0 {\n return None;\n }\n\n let name = self.data.get(self.offset..self.offset + usize::from(len))?;\n self.offset += usize::from(len);\n core::str::from_utf8(name).ok()\n }\n}\n\n/// A [PostScript Table](https://docs.microsoft.com/en-us/typography/opentype/spec/post).\n#[derive(Clone, Copy, Debug)]\npub struct Table<'a> {\n /// Italic angle in counter-clockwise degrees from the vertical.\n pub italic_angle: f32,\n /// Underline metrics.\n pub underline_metrics: LineMetrics,\n /// Flag that indicates that the font is monospaced.\n pub is_monospaced: bool,\n\n glyph_indexes: LazyArray16<'a, u16>,\n names_data: &'a [u8],\n}\n\nimpl<'a> Table<'a> {\n /// Parses a table from raw data.\n pub fn parse(data: &'a [u8]) -> Option {\n // Do not check the exact length, because some fonts include\n // padding in table's length in table records, which is incorrect.\n if data.len() < 32 {\n return None;\n }\n\n let version = Stream::new(data).read::()?;\n if !(version == 0x00010000\n || version == 0x00020000\n || version == 0x00025000\n || version == 0x00030000\n || version == 0x00040000)\n {\n return None;\n }\n\n let italic_angle = Stream::read_at::(data, ITALIC_ANGLE_OFFSET)?.0;\n\n let underline_metrics = LineMetrics {\n position: Stream::read_at::(data, UNDERLINE_POSITION_OFFSET)?,\n thickness: Stream::read_at::(data, UNDERLINE_THICKNESS_OFFSET)?,\n };\n\n let is_monospaced = Stream::read_at::(data, IS_FIXED_PITCH_OFFSET)? != 0;\n\n let mut names_data: &[u8] = &[];\n let mut glyph_indexes = LazyArray16::default();\n // Only version 2.0 of the table has data at the end.\n if version == 0x00020000 {\n let mut s = Stream::new_at(data, 32)?;\n let indexes_count = s.read::()?;\n glyph_indexes = s.read_array16::(indexes_count)?;\n names_data = s.tail()?;\n }\n\n Some(Table {\n italic_angle,\n underline_metrics,\n is_monospaced,\n names_data,\n glyph_indexes,\n })\n }\n\n /// Returns a glyph name by ID.\n #[cfg(feature = \"glyph-names\")]\n pub fn glyph_name(&self, glyph_id: GlyphId) -> Option<&'a str> {\n let mut index = self.glyph_indexes.get(glyph_id.0)?;\n\n // 'If the name index is between 0 and 257, treat the name index\n // as a glyph index in the Macintosh standard order.'\n if usize::from(index) < MACINTOSH_NAMES.len() {\n Some(MACINTOSH_NAMES[usize::from(index)])\n } else {\n // 'If the name index is between 258 and 65535, then subtract 258 and use that\n // to index into the list of Pascal strings at the end of the table.'\n index -= MACINTOSH_NAMES.len() as u16;\n self.names().nth(usize::from(index))\n }\n }\n\n /// Returns a glyph ID by a name.\n #[cfg(feature = \"glyph-names\")]\n pub fn glyph_index_by_name(&self, name: &str) -> Option {\n let id = if let Some(index) = MACINTOSH_NAMES.iter().position(|n| *n == name) {\n self.glyph_indexes\n .into_iter()\n .position(|i| usize::from(i) == index)?\n } else {\n let mut index = self.names().position(|n| n == name)?;\n index += MACINTOSH_NAMES.len();\n self.glyph_indexes\n .into_iter()\n .position(|i| usize::from(i) == index)?\n };\n\n Some(GlyphId(id as u16))\n }\n\n /// Returns an iterator over glyph names.\n ///\n /// Default/predefined names are not included. Just the one in the font file.\n pub fn names(&self) -> Names<'a> {\n Names {\n data: self.names_data,\n offset: 0,\n }\n }\n}\n" }, { "package": "unicode-bidi@0.3.18", "sourcePath": "src/char_data/mod.rs", "sha256": "8cbdcaacddb3dd9b70d615693fa73d0e7dca6332102a95f0d3ce447df7645284", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Servo Project Developers. See the\n// COPYRIGHT file at the top-level directory of this distribution.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Accessor for `Bidi_Class` property from Unicode Character Database (UCD)\n\nmod tables;\n\npub use self::tables::{BidiClass, UNICODE_VERSION};\n#[cfg(feature = \"hardcoded-data\")]\nuse core::char;\n#[cfg(feature = \"hardcoded-data\")]\nuse core::cmp::Ordering::{Equal, Greater, Less};\n\n#[cfg(feature = \"hardcoded-data\")]\nuse self::tables::bidi_class_table;\nuse crate::data_source::BidiMatchedOpeningBracket;\nuse crate::BidiClass::*;\n#[cfg(feature = \"hardcoded-data\")]\nuse crate::BidiDataSource;\n/// Hardcoded Bidi data that ships with the unicode-bidi crate.\n///\n/// This can be enabled with the default `hardcoded-data` Cargo feature.\n#[cfg(feature = \"hardcoded-data\")]\npub struct HardcodedBidiData;\n\n#[cfg(feature = \"hardcoded-data\")]\nimpl BidiDataSource for HardcodedBidiData {\n fn bidi_class(&self, c: char) -> BidiClass {\n bsearch_range_value_table(c, bidi_class_table)\n }\n}\n\n/// Find the `BidiClass` of a single char.\n#[cfg(feature = \"hardcoded-data\")]\npub fn bidi_class(c: char) -> BidiClass {\n bsearch_range_value_table(c, bidi_class_table)\n}\n\n/// If this character is a bracket according to BidiBrackets.txt,\n/// return the corresponding *normalized* *opening bracket* of the pair,\n/// and whether or not it itself is an opening bracket.\npub(crate) fn bidi_matched_opening_bracket(c: char) -> Option {\n for pair in self::tables::bidi_pairs_table {\n if pair.0 == c || pair.1 == c {\n let skeleton = pair.2.unwrap_or(pair.0);\n return Some(BidiMatchedOpeningBracket {\n opening: skeleton,\n is_open: pair.0 == c,\n });\n }\n }\n None\n}\n\npub fn is_rtl(bidi_class: BidiClass) -> bool {\n matches!(bidi_class, RLE | RLO | RLI)\n}\n\n#[cfg(feature = \"hardcoded-data\")]\nfn bsearch_range_value_table(c: char, r: &'static [(char, char, BidiClass)]) -> BidiClass {\n match r.binary_search_by(|&(lo, hi, _)| {\n if lo <= c && c <= hi {\n Equal\n } else if hi < c {\n Less\n } else {\n Greater\n }\n }) {\n Ok(idx) => {\n let (_, _, cat) = r[idx];\n cat\n }\n // UCD/extracted/DerivedBidiClass.txt: \"All code points not explicitly listed\n // for Bidi_Class have the value Left_To_Right (L).\"\n Err(_) => L,\n }\n}\n\n#[cfg(all(test, feature = \"hardcoded-data\"))]\nmod tests {\n use super::*;\n\n #[test]\n fn test_ascii() {\n assert_eq!(bidi_class('\\u{0000}'), BN);\n assert_eq!(bidi_class('\\u{0040}'), ON);\n assert_eq!(bidi_class('\\u{0041}'), L);\n assert_eq!(bidi_class('\\u{0062}'), L);\n assert_eq!(bidi_class('\\u{007F}'), BN);\n }\n\n #[test]\n fn test_bmp() {\n // Hebrew\n assert_eq!(bidi_class('\\u{0590}'), R);\n assert_eq!(bidi_class('\\u{05D0}'), R);\n assert_eq!(bidi_class('\\u{05D1}'), R);\n assert_eq!(bidi_class('\\u{05FF}'), R);\n\n // Arabic\n assert_eq!(bidi_class('\\u{0600}'), AN);\n assert_eq!(bidi_class('\\u{0627}'), AL);\n assert_eq!(bidi_class('\\u{07BF}'), AL);\n\n // Default R + Arabic Extras\n assert_eq!(bidi_class('\\u{07C0}'), R);\n assert_eq!(bidi_class('\\u{085F}'), R);\n assert_eq!(bidi_class('\\u{0860}'), AL);\n assert_eq!(bidi_class('\\u{0870}'), AL);\n assert_eq!(bidi_class('\\u{089F}'), NSM);\n assert_eq!(bidi_class('\\u{08A0}'), AL);\n assert_eq!(bidi_class('\\u{089F}'), NSM);\n assert_eq!(bidi_class('\\u{08FF}'), NSM);\n\n // Default ET\n assert_eq!(bidi_class('\\u{20A0}'), ET);\n assert_eq!(bidi_class('\\u{20CF}'), ET);\n\n // Arabic Presentation Forms\n assert_eq!(bidi_class('\\u{FB1D}'), R);\n assert_eq!(bidi_class('\\u{FB4F}'), R);\n assert_eq!(bidi_class('\\u{FB50}'), AL);\n assert_eq!(bidi_class('\\u{FDCF}'), ON);\n assert_eq!(bidi_class('\\u{FDF0}'), AL);\n assert_eq!(bidi_class('\\u{FDFF}'), ON);\n assert_eq!(bidi_class('\\u{FE70}'), AL);\n assert_eq!(bidi_class('\\u{FEFE}'), AL);\n assert_eq!(bidi_class('\\u{FEFF}'), BN);\n\n // noncharacters\n assert_eq!(bidi_class('\\u{FDD0}'), L);\n assert_eq!(bidi_class('\\u{FDD1}'), L);\n assert_eq!(bidi_class('\\u{FDEE}'), L);\n assert_eq!(bidi_class('\\u{FDEF}'), L);\n assert_eq!(bidi_class('\\u{FFFE}'), L);\n assert_eq!(bidi_class('\\u{FFFF}'), L);\n }\n\n #[test]\n fn test_smp() {\n // Default AL + R\n assert_eq!(bidi_class('\\u{10800}'), R);\n assert_eq!(bidi_class('\\u{10FFF}'), R);\n assert_eq!(bidi_class('\\u{1E800}'), R);\n assert_eq!(bidi_class('\\u{1EDFF}'), R);\n assert_eq!(bidi_class('\\u{1EE00}'), AL);\n assert_eq!(bidi_class('\\u{1EEFF}'), AL);\n assert_eq!(bidi_class('\\u{1EF00}'), R);\n assert_eq!(bidi_class('\\u{1EFFF}'), R);\n }\n\n #[test]\n fn test_unassigned_planes() {\n assert_eq!(bidi_class('\\u{30000}'), L);\n assert_eq!(bidi_class('\\u{40000}'), L);\n assert_eq!(bidi_class('\\u{50000}'), L);\n assert_eq!(bidi_class('\\u{60000}'), L);\n assert_eq!(bidi_class('\\u{70000}'), L);\n assert_eq!(bidi_class('\\u{80000}'), L);\n assert_eq!(bidi_class('\\u{90000}'), L);\n assert_eq!(bidi_class('\\u{a0000}'), L);\n }\n}\n" }, { "package": "unicode-bidi@0.3.18", "sourcePath": "src/data_source.rs", "sha256": "36fa0785e51c549c1f72f09040cfe515b848d1b23fb30d469770a6b4b17b49df", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Servo Project Developers. See the\n// COPYRIGHT file at the top-level directory of this distribution.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse crate::BidiClass;\n\n/// This is the return value of [`BidiDataSource::bidi_matched_opening_bracket()`].\n///\n/// It represents the matching *normalized* opening bracket for a given bracket in a bracket pair,\n/// and whether or not that bracket is opening.\n#[derive(Debug, Copy, Clone)]\npub struct BidiMatchedOpeningBracket {\n /// The corresponding opening bracket in this bracket pair, normalized\n ///\n /// In case of opening brackets, this will be the bracket itself, except for when the bracket\n /// is not normalized, in which case it will be the normalized form.\n pub opening: char,\n /// Whether or not the requested bracket was an opening bracket. True for opening\n pub is_open: bool,\n}\n\n/// This trait abstracts over a data source that is able to produce the Unicode Bidi class for a given\n/// character\npub trait BidiDataSource {\n fn bidi_class(&self, c: char) -> BidiClass;\n /// If this character is a bracket according to BidiBrackets.txt,\n /// return the corresponding *normalized* *opening bracket* of the pair,\n /// and whether or not it itself is an opening bracket.\n ///\n /// This effectively buckets brackets into equivalence classes keyed on the\n /// normalized opening bracket.\n ///\n /// The default implementation will pull in a small amount of hardcoded data,\n /// regardless of the `hardcoded-data` feature. This is in part for convenience\n /// (since this data is small and changes less often), and in part so that this method can be\n /// added without needing a breaking version bump.\n /// Override this method in your custom data source to prevent the use of hardcoded data.\n fn bidi_matched_opening_bracket(&self, c: char) -> Option {\n crate::char_data::bidi_matched_opening_bracket(c)\n }\n}\n" }, { "package": "unicode-bidi@0.3.18", "sourcePath": "src/deprecated.rs", "sha256": "f94c0e75dec7e70cb9802e26b7f82fe618dcdd50e9973927bacd4eccc6899c62", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Servo Project Developers. See the\n// COPYRIGHT file at the top-level directory of this distribution.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! This module holds deprecated assets only.\n\nuse super::*;\n\n/// Find the level runs within a line and return them in visual order.\n///\n/// NOTE: This implementation is incomplete. The algorithm needs information about the text,\n/// including original `BidiClass` property of each character, to be able to perform correctly.\n/// Please see [`BidiInfo::visual_runs()`](../struct.BidiInfo.html#method.visual_runs) for the\n/// improved implementation.\n///\n/// `line` is a range of bytes indices within `levels`.\n///\n/// \n#[deprecated(\n since = \"0.3.0\",\n note = \"please use `BidiInfo::visual_runs()` instead.\"\n)]\npub fn visual_runs(line: Range, levels: &[Level]) -> Vec {\n assert!(line.start <= levels.len());\n assert!(line.end <= levels.len());\n\n let mut runs = Vec::new();\n\n // Find consecutive level runs.\n let mut start = line.start;\n let mut run_level = levels[start];\n let mut min_level = run_level;\n let mut max_level = run_level;\n\n for (i, &new_level) in levels.iter().enumerate().take(line.end).skip(start + 1) {\n if new_level != run_level {\n // End of the previous run, start of a new one.\n runs.push(start..i);\n start = i;\n run_level = new_level;\n\n min_level = cmp::min(run_level, min_level);\n max_level = cmp::max(run_level, max_level);\n }\n }\n runs.push(start..line.end);\n\n let run_count = runs.len();\n\n // Re-order the odd runs.\n // \n\n // Stop at the lowest *odd* level.\n min_level = min_level.new_lowest_ge_rtl().expect(\"Level error\");\n\n while max_level >= min_level {\n // Look for the start of a sequence of consecutive runs of max_level or higher.\n let mut seq_start = 0;\n while seq_start < run_count {\n if levels[runs[seq_start].start] < max_level {\n seq_start += 1;\n continue;\n }\n\n // Found the start of a sequence. Now find the end.\n let mut seq_end = seq_start + 1;\n\n while seq_end < run_count && levels[runs[seq_end].start] >= max_level {\n seq_end += 1;\n }\n\n // Reverse the runs within this sequence.\n runs[seq_start..seq_end].reverse();\n\n seq_start = seq_end;\n }\n\n max_level\n .lower(1)\n .expect(\"Lowering embedding level below zero\");\n }\n\n runs\n}\n" }, { "package": "unicode-bidi@0.3.18", "sourcePath": "src/explicit.rs", "sha256": "86c3c55bf2cc90aab1411aac6cf05de505ca74e44a76fe829572dd7dc4dd2aa3", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Servo Project Developers. See the\n// COPYRIGHT file at the top-level directory of this distribution.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! 3.3.2 Explicit Levels and Directions\n//!\n//! \n\n#[cfg(feature = \"smallvec\")]\nuse smallvec::{smallvec, SmallVec};\n\nuse super::char_data::{\n is_rtl,\n BidiClass::{self, *},\n};\nuse super::level::Level;\nuse super::prepare::removed_by_x9;\nuse super::LevelRunVec;\nuse super::TextSource;\n\n/// Compute explicit embedding levels for one paragraph of text (X1-X8), and identify\n/// level runs (BD7) for use when determining Isolating Run Sequences (X10).\n///\n/// `processing_classes[i]` must contain the `BidiClass` of the char at byte index `i`,\n/// for each char in `text`.\n///\n/// `runs` returns the list of level runs (BD7) of the text.\n#[cfg_attr(feature = \"flame_it\", flamer::flame)]\npub fn compute<'a, T: TextSource<'a> + ?Sized>(\n text: &'a T,\n para_level: Level,\n original_classes: &[BidiClass],\n levels: &mut [Level],\n processing_classes: &mut [BidiClass],\n runs: &mut LevelRunVec,\n) {\n assert_eq!(text.len(), original_classes.len());\n\n // \n #[cfg(feature = \"smallvec\")]\n let mut stack: SmallVec<[Status; 8]> = smallvec![Status {\n level: para_level,\n status: OverrideStatus::Neutral,\n }];\n #[cfg(not(feature = \"smallvec\"))]\n let mut stack = vec![Status {\n level: para_level,\n status: OverrideStatus::Neutral,\n }];\n\n let mut overflow_isolate_count = 0u32;\n let mut overflow_embedding_count = 0u32;\n let mut valid_isolate_count = 0u32;\n\n let mut current_run_level = Level::ltr();\n let mut current_run_start = 0;\n\n for (i, len) in text.indices_lengths() {\n let last = stack.last().unwrap();\n\n match original_classes[i] {\n // Rules X2-X5c\n RLE | LRE | RLO | LRO | RLI | LRI | FSI => {\n // \n levels[i] = last.level;\n\n // X5a-X5c: Isolate initiators get the level of the last entry on the stack.\n let is_isolate = matches!(original_classes[i], RLI | LRI | FSI);\n if is_isolate {\n // Redundant due to \"Retaining explicit formatting characters\" step.\n // levels[i] = last.level;\n match last.status {\n OverrideStatus::RTL => processing_classes[i] = R,\n OverrideStatus::LTR => processing_classes[i] = L,\n _ => {}\n }\n }\n\n let new_level = if is_rtl(original_classes[i]) {\n last.level.new_explicit_next_rtl()\n } else {\n last.level.new_explicit_next_ltr()\n };\n\n if new_level.is_ok() && overflow_isolate_count == 0 && overflow_embedding_count == 0\n {\n let new_level = new_level.unwrap();\n\n stack.push(Status {\n level: new_level,\n status: match original_classes[i] {\n RLO => OverrideStatus::RTL,\n LRO => OverrideStatus::LTR,\n RLI | LRI | FSI => OverrideStatus::Isolate,\n _ => OverrideStatus::Neutral,\n },\n });\n\n if is_isolate {\n valid_isolate_count += 1;\n } else {\n // The spec doesn't explicitly mention this step, but it is necessary.\n // See the reference implementations for comparison.\n levels[i] = new_level;\n }\n } else if is_isolate {\n overflow_isolate_count += 1;\n } else if overflow_isolate_count == 0 {\n overflow_embedding_count += 1;\n }\n\n if !is_isolate {\n // X9 +\n // \n // (PDF handled below)\n processing_classes[i] = BN;\n }\n }\n\n // \n PDI => {\n if overflow_isolate_count > 0 {\n overflow_isolate_count -= 1;\n } else if valid_isolate_count > 0 {\n overflow_embedding_count = 0;\n\n while !matches!(\n stack.pop(),\n None | Some(Status {\n status: OverrideStatus::Isolate,\n ..\n })\n ) {}\n\n valid_isolate_count -= 1;\n }\n\n let last = stack.last().unwrap();\n levels[i] = last.level;\n\n match last.status {\n OverrideStatus::RTL => processing_classes[i] = R,\n OverrideStatus::LTR => processing_classes[i] = L,\n _ => {}\n }\n }\n\n // \n PDF => {\n if overflow_isolate_count > 0 {\n // do nothing\n } else if overflow_embedding_count > 0 {\n overflow_embedding_count -= 1;\n } else if last.status != OverrideStatus::Isolate && stack.len() >= 2 {\n stack.pop();\n }\n\n // \n levels[i] = stack.last().unwrap().level;\n // X9 part of retaining explicit formatting characters.\n processing_classes[i] = BN;\n }\n\n // Nothing.\n // BN case moved down to X6, see \n B => {}\n\n // \n _ => {\n levels[i] = last.level;\n\n // This condition is not in the spec, but I am pretty sure that is a spec bug.\n // https://www.unicode.org/L2/L2023/23014-amd-to-uax9.pdf\n if original_classes[i] != BN {\n match last.status {\n OverrideStatus::RTL => processing_classes[i] = R,\n OverrideStatus::LTR => processing_classes[i] = L,\n _ => {}\n }\n }\n }\n }\n\n // Handle multi-byte characters.\n for j in 1..len {\n levels[i + j] = levels[i];\n processing_classes[i + j] = processing_classes[i];\n }\n\n // Identify level runs to be passed to prepare::isolating_run_sequences().\n if i == 0 {\n // Initialize for the first (or only) run.\n current_run_level = levels[i];\n } else {\n // Check if we need to start a new level run.\n // \n if !removed_by_x9(original_classes[i]) && levels[i] != current_run_level {\n // End the last run and start a new one.\n runs.push(current_run_start..i);\n current_run_level = levels[i];\n current_run_start = i;\n }\n }\n }\n\n // Append the trailing level run, if non-empty.\n if levels.len() > current_run_start {\n runs.push(current_run_start..levels.len());\n }\n}\n\n/// Entries in the directional status stack:\nstruct Status {\n level: Level,\n status: OverrideStatus,\n}\n\n#[derive(PartialEq)]\nenum OverrideStatus {\n Neutral,\n RTL,\n LTR,\n Isolate,\n}\n" }, { "package": "unicode-bidi@0.3.18", "sourcePath": "src/format_chars.rs", "sha256": "678399fec3f4bfaf4093f38cfdb8956288313386dc3511dab9fb58164e8dc01b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2017 The Servo Project Developers. See the\n// COPYRIGHT file at the top-level directory of this distribution.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Directional Formatting Characters\n//!\n//! \n\n// == Implicit ==\n/// ARABIC LETTER MARK\npub const ALM: char = '\\u{061C}';\n/// LEFT-TO-RIGHT MARK\npub const LRM: char = '\\u{200E}';\n/// RIGHT-TO-LEFT MARK\npub const RLM: char = '\\u{200F}';\n\n// == Explicit Isolates ==\n/// LEFT‑TO‑RIGHT ISOLATE\npub const LRI: char = '\\u{2066}';\n/// RIGHT‑TO‑LEFT ISOLATE\npub const RLI: char = '\\u{2067}';\n/// FIRST STRONG ISOLATE\npub const FSI: char = '\\u{2068}';\n/// POP DIRECTIONAL ISOLATE\npub const PDI: char = '\\u{2069}';\n\n// == Explicit Embeddings and Overrides ==\n/// LEFT-TO-RIGHT EMBEDDING\npub const LRE: char = '\\u{202A}';\n/// RIGHT-TO-LEFT EMBEDDING\npub const RLE: char = '\\u{202B}';\n/// POP DIRECTIONAL FORMATTING\npub const PDF: char = '\\u{202C}';\n/// LEFT-TO-RIGHT OVERRIDE\npub const LRO: char = '\\u{202D}';\n/// RIGHT-TO-LEFT OVERRIDE\npub const RLO: char = '\\u{202E}';\n" }, { "package": "unicode-bidi@0.3.18", "sourcePath": "src/implicit.rs", "sha256": "8d5b003464aee3f333785c6170a884945251f39601e4ea658e669a2ad575d588", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Servo Project Developers. See the\n// COPYRIGHT file at the top-level directory of this distribution.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! 3.3.4 - 3.3.6. Resolve implicit levels and types.\n\n#[cfg(not(feature = \"smallvec\"))]\nuse alloc::vec::Vec;\nuse core::cmp::max;\n#[cfg(feature = \"smallvec\")]\nuse smallvec::SmallVec;\n\nuse super::char_data::BidiClass::{self, *};\nuse super::level::Level;\nuse super::prepare::{not_removed_by_x9, IsolatingRunSequence};\nuse super::{BidiDataSource, TextSource};\n\n/// 3.3.4 Resolving Weak Types\n///\n/// \n#[cfg_attr(feature = \"flame_it\", flamer::flame)]\npub fn resolve_weak<'a, T: TextSource<'a> + ?Sized>(\n text: &'a T,\n sequence: &IsolatingRunSequence,\n processing_classes: &mut [BidiClass],\n) {\n // Note: The spec treats these steps as individual passes that are applied one after the other\n // on the entire IsolatingRunSequence at once. We instead collapse it into a single iteration,\n // which is straightforward for rules that are based on the state of the current character, but not\n // for rules that care about surrounding characters. To deal with them, we retain additional state\n // about previous character classes that may have since been changed by later rules.\n\n // The previous class for the purposes of rule W4/W6, not tracking changes made after or during W4.\n let mut prev_class_before_w4 = sequence.sos;\n // The previous class for the purposes of rule W5.\n let mut prev_class_before_w5 = sequence.sos;\n // The previous class for the purposes of rule W1, not tracking changes from any other rules.\n let mut prev_class_before_w1 = sequence.sos;\n let mut last_strong_is_al = false;\n #[cfg(feature = \"smallvec\")]\n let mut et_run_indices = SmallVec::<[usize; 8]>::new(); // for W5\n #[cfg(not(feature = \"smallvec\"))]\n let mut et_run_indices = Vec::new(); // for W5\n #[cfg(feature = \"smallvec\")]\n let mut bn_run_indices = SmallVec::<[usize; 8]>::new(); // for W5 + \n #[cfg(not(feature = \"smallvec\"))]\n let mut bn_run_indices = Vec::new(); // for W5 + \n\n for (run_index, level_run) in sequence.runs.iter().enumerate() {\n for i in &mut level_run.clone() {\n if processing_classes[i] == BN {\n // \n // Keeps track of bn runs for W5 in case we see an ET.\n bn_run_indices.push(i);\n // BNs aren't real, skip over them.\n continue;\n }\n\n // Store the processing class of all rules before W2/W1.\n // Used to keep track of the last strong character for W2. W3 is able to insert new strong\n // characters, so we don't want to be misled by it.\n let mut w2_processing_class = processing_classes[i];\n\n // \n //\n\n if processing_classes[i] == NSM {\n processing_classes[i] = match prev_class_before_w1 {\n RLI | LRI | FSI | PDI => ON,\n _ => prev_class_before_w1,\n };\n // W1 occurs before W2, update this.\n w2_processing_class = processing_classes[i];\n }\n\n prev_class_before_w1 = processing_classes[i];\n\n // \n // \n //\n match processing_classes[i] {\n EN => {\n if last_strong_is_al {\n // W2. If previous strong char was AL, change EN to AN.\n processing_classes[i] = AN;\n }\n }\n // W3.\n AL => processing_classes[i] = R,\n _ => {}\n }\n\n // update last_strong_is_al.\n match w2_processing_class {\n L | R => {\n last_strong_is_al = false;\n }\n AL => {\n last_strong_is_al = true;\n }\n _ => {}\n }\n\n let class_before_w456 = processing_classes[i];\n\n // \n // \n // (separators only)\n // (see below for W6 terminator code)\n //\n match processing_classes[i] {\n // \n EN => {\n // W5. If a run of ETs is adjacent to an EN, change the ETs to EN.\n for j in &et_run_indices {\n processing_classes[*j] = EN;\n }\n et_run_indices.clear();\n }\n\n // \n // \n ES | CS => {\n // See https://github.com/servo/unicode-bidi/issues/86 for improving this.\n // We want to make sure we check the correct next character by skipping past the rest\n // of this one.\n if let Some((_, char_len)) = text.char_at(i) {\n let mut next_class = sequence\n .iter_forwards_from(i + char_len, run_index)\n .map(|j| processing_classes[j])\n // \n .find(not_removed_by_x9)\n .unwrap_or(sequence.eos);\n if next_class == EN && last_strong_is_al {\n // Apply W2 to next_class. We know that last_strong_is_al\n // has no chance of changing on this character so we can still assume its value\n // will be the same by the time we get to it.\n next_class = AN;\n }\n processing_classes[i] =\n match (prev_class_before_w4, processing_classes[i], next_class) {\n // W4\n (EN, ES, EN) | (EN, CS, EN) => EN,\n // W4\n (AN, CS, AN) => AN,\n // W6 (separators only)\n (_, _, _) => ON,\n };\n\n // W6 + \n // We have to do this before W5 gets its grubby hands on these characters and thinks\n // they're part of an ET run.\n // We check for ON to ensure that we had hit the W6 branch above, since this `ES | CS` match\n // arm handles both W4 and W6.\n if processing_classes[i] == ON {\n for idx in sequence.iter_backwards_from(i, run_index) {\n let class = &mut processing_classes[idx];\n if *class != BN {\n break;\n }\n *class = ON;\n }\n for idx in sequence.iter_forwards_from(i + char_len, run_index) {\n let class = &mut processing_classes[idx];\n if *class != BN {\n break;\n }\n *class = ON;\n }\n }\n } else {\n // We're in the middle of a character, copy over work done for previous bytes\n // since it's going to be the same answer.\n processing_classes[i] = processing_classes[i - 1];\n }\n }\n // \n ET => {\n match prev_class_before_w5 {\n EN => processing_classes[i] = EN,\n _ => {\n // \n // If there was a BN run before this, that's now a part of this ET run.\n et_run_indices.extend(bn_run_indices.clone());\n\n // In case this is followed by an EN.\n et_run_indices.push(i);\n }\n }\n }\n _ => {}\n }\n\n // Common loop iteration code\n //\n\n // \n // BN runs would have already continued the loop, clear them before we get to the next one.\n bn_run_indices.clear();\n\n // W6 above only deals with separators, so it doesn't change anything W5 cares about,\n // so we still can update this after running that part of W6.\n prev_class_before_w5 = processing_classes[i];\n\n // (terminators only)\n // (see above for W6 separator code)\n //\n if prev_class_before_w5 != ET {\n // W6. If we didn't find an adjacent EN, turn any ETs into ON instead.\n for j in &et_run_indices {\n processing_classes[*j] = ON;\n }\n et_run_indices.clear();\n }\n\n // We stashed this before W4/5/6 could get their grubby hands on it, and it's not\n // used in the W6 terminator code below so we can update it now.\n prev_class_before_w4 = class_before_w456;\n }\n }\n // Rerun this check in case we ended with a sequence of BNs (i.e., we'd never\n // hit the end of the for loop above).\n // W6. If we didn't find an adjacent EN, turn any ETs into ON instead.\n for j in &et_run_indices {\n processing_classes[*j] = ON;\n }\n et_run_indices.clear();\n\n // W7. If the previous strong char was L, change EN to L.\n let mut last_strong_is_l = sequence.sos == L;\n for i in sequence.runs.iter().cloned().flatten() {\n match processing_classes[i] {\n EN if last_strong_is_l => {\n processing_classes[i] = L;\n }\n L => {\n last_strong_is_l = true;\n }\n R | AL => {\n last_strong_is_l = false;\n }\n // \n // Already scanning past BN here.\n _ => {}\n }\n }\n}\n\n#[cfg(feature = \"smallvec\")]\ntype BracketPairVec = SmallVec<[BracketPair; 8]>;\n#[cfg(not(feature = \"smallvec\"))]\ntype BracketPairVec = Vec;\n\n/// 3.3.5 Resolving Neutral Types\n///\n/// \n#[cfg_attr(feature = \"flame_it\", flamer::flame)]\npub fn resolve_neutral<'a, D: BidiDataSource, T: TextSource<'a> + ?Sized>(\n text: &'a T,\n data_source: &D,\n sequence: &IsolatingRunSequence,\n levels: &[Level],\n original_classes: &[BidiClass],\n processing_classes: &mut [BidiClass],\n) {\n // e = embedding direction\n let e: BidiClass = levels[sequence.runs[0].start].bidi_class();\n let not_e = if e == BidiClass::L {\n BidiClass::R\n } else {\n BidiClass::L\n };\n // N0. Process bracket pairs.\n\n // > Identify the bracket pairs in the current isolating run sequence according to BD16.\n // We use processing_classes, not original_classes, due to BD14/BD15\n let mut bracket_pairs = BracketPairVec::new();\n identify_bracket_pairs(\n text,\n data_source,\n sequence,\n processing_classes,\n &mut bracket_pairs,\n );\n\n // > For each bracket-pair element in the list of pairs of text positions\n //\n // Note: Rust ranges are interpreted as [start..end), be careful using `pair` directly\n // for indexing as it will include the opening bracket pair but not the closing one.\n for pair in bracket_pairs {\n #[cfg(feature = \"std\")]\n debug_assert!(\n pair.start < processing_classes.len(),\n \"identify_bracket_pairs returned a range that is out of bounds!\"\n );\n #[cfg(feature = \"std\")]\n debug_assert!(\n pair.end < processing_classes.len(),\n \"identify_bracket_pairs returned a range that is out of bounds!\"\n );\n let mut found_e = false;\n let mut found_not_e = false;\n let mut class_to_set = None;\n\n let start_char_len =\n T::char_len(text.subrange(pair.start..pair.end).chars().next().unwrap());\n // > Inspect the bidirectional types of the characters enclosed within the bracket pair.\n //\n // `pair` is [start, end) so we will end up processing the opening character but not the closing one.\n //\n for enclosed_i in sequence.iter_forwards_from(pair.start + start_char_len, pair.start_run) {\n if enclosed_i >= pair.end {\n #[cfg(feature = \"std\")]\n debug_assert!(\n enclosed_i == pair.end,\n \"If we skipped past this, the iterator is broken\"\n );\n break;\n }\n let class = processing_classes[enclosed_i];\n if class == e {\n found_e = true;\n } else if class == not_e {\n found_not_e = true;\n } else if matches!(class, BidiClass::EN | BidiClass::AN) {\n // > Within this scope, bidirectional types EN and AN are treated as R.\n if e == BidiClass::L {\n found_not_e = true;\n } else {\n found_e = true;\n }\n }\n\n // If we have found a character with the class of the embedding direction\n // we can bail early.\n if found_e {\n break;\n }\n }\n // > If any strong type (either L or R) matching the embedding direction is found\n if found_e {\n // > .. set the type for both brackets in the pair to match the embedding direction\n class_to_set = Some(e);\n // > Otherwise, if there is a strong type it must be opposite the embedding direction\n } else if found_not_e {\n // > Therefore, test for an established context with a preceding strong type by\n // > checking backwards before the opening paired bracket\n // > until the first strong type (L, R, or sos) is found.\n // (see note above about processing_classes and character boundaries)\n let mut previous_strong = sequence\n .iter_backwards_from(pair.start, pair.start_run)\n .map(|i| processing_classes[i])\n .find(|class| {\n matches!(\n class,\n BidiClass::L | BidiClass::R | BidiClass::EN | BidiClass::AN\n )\n })\n .unwrap_or(sequence.sos);\n\n // > Within this scope, bidirectional types EN and AN are treated as R.\n if matches!(previous_strong, BidiClass::EN | BidiClass::AN) {\n previous_strong = BidiClass::R;\n }\n\n // > If the preceding strong type is also opposite the embedding direction,\n // > context is established,\n // > so set the type for both brackets in the pair to that direction.\n // AND\n // > Otherwise set the type for both brackets in the pair to the embedding direction.\n // > Either way it gets set to previous_strong\n //\n // Both branches amount to setting the type to the strong type.\n class_to_set = Some(previous_strong);\n }\n\n if let Some(class_to_set) = class_to_set {\n // Update all processing classes corresponding to the start and end elements, as requested.\n // We should include all bytes of the character, not the first one.\n let end_char_len =\n T::char_len(text.subrange(pair.end..text.len()).chars().next().unwrap());\n for class in &mut processing_classes[pair.start..pair.start + start_char_len] {\n *class = class_to_set;\n }\n for class in &mut processing_classes[pair.end..pair.end + end_char_len] {\n *class = class_to_set;\n }\n // \n for idx in sequence.iter_backwards_from(pair.start, pair.start_run) {\n let class = &mut processing_classes[idx];\n if *class != BN {\n break;\n }\n *class = class_to_set;\n }\n // > Any number of characters that had original bidirectional character type NSM prior to the application of\n // > W1 that immediately follow a paired bracket which changed to L or R under N0 should change to match the type of their preceding bracket.\n\n // This rule deals with sequences of NSMs, so we can just update them all at once, we don't need to worry\n // about character boundaries. We do need to be careful to skip the full set of bytes for the parentheses characters.\n let nsm_start = pair.start + start_char_len;\n for idx in sequence.iter_forwards_from(nsm_start, pair.start_run) {\n let class = original_classes[idx];\n if class == BidiClass::NSM || processing_classes[idx] == BN {\n processing_classes[idx] = class_to_set;\n } else {\n break;\n }\n }\n let nsm_end = pair.end + end_char_len;\n for idx in sequence.iter_forwards_from(nsm_end, pair.end_run) {\n let class = original_classes[idx];\n if class == BidiClass::NSM || processing_classes[idx] == BN {\n processing_classes[idx] = class_to_set;\n } else {\n break;\n }\n }\n }\n // > Otherwise, there are no strong types within the bracket pair\n // > Therefore, do not set the type for that bracket pair\n }\n\n // N1 and N2.\n // Indices of every byte in this isolating run sequence\n let mut indices = sequence.runs.iter().flat_map(Clone::clone);\n let mut prev_class = sequence.sos;\n while let Some(mut i) = indices.next() {\n // Process sequences of NI characters.\n #[cfg(feature = \"smallvec\")]\n let mut ni_run = SmallVec::<[usize; 8]>::new();\n #[cfg(not(feature = \"smallvec\"))]\n let mut ni_run = Vec::new();\n // The BN is for \n if is_NI(processing_classes[i]) || processing_classes[i] == BN {\n // Consume a run of consecutive NI characters.\n ni_run.push(i);\n let mut next_class;\n loop {\n match indices.next() {\n Some(j) => {\n i = j;\n next_class = processing_classes[j];\n // The BN is for \n if is_NI(next_class) || next_class == BN {\n ni_run.push(i);\n } else {\n break;\n }\n }\n None => {\n next_class = sequence.eos;\n break;\n }\n };\n }\n // N1-N2.\n //\n // \n // \n let new_class = match (prev_class, next_class) {\n (L, L) => L,\n (R, R)\n | (R, AN)\n | (R, EN)\n | (AN, R)\n | (AN, AN)\n | (AN, EN)\n | (EN, R)\n | (EN, AN)\n | (EN, EN) => R,\n (_, _) => e,\n };\n for j in &ni_run {\n processing_classes[*j] = new_class;\n }\n ni_run.clear();\n }\n prev_class = processing_classes[i];\n }\n}\n\nstruct BracketPair {\n /// The text-relative index of the opening bracket.\n start: usize,\n /// The text-relative index of the closing bracket.\n end: usize,\n /// The index of the run (in the run sequence) that the opening bracket is in.\n start_run: usize,\n /// The index of the run (in the run sequence) that the closing bracket is in.\n end_run: usize,\n}\n/// 3.1.3 Identifying Bracket Pairs\n///\n/// Returns all paired brackets in the source, as indices into the\n/// text source.\n///\n/// \nfn identify_bracket_pairs<'a, T: TextSource<'a> + ?Sized, D: BidiDataSource>(\n text: &'a T,\n data_source: &D,\n run_sequence: &IsolatingRunSequence,\n original_classes: &[BidiClass],\n bracket_pairs: &mut BracketPairVec,\n) {\n #[cfg(feature = \"smallvec\")]\n let mut stack = SmallVec::<[(char, usize, usize); 8]>::new();\n #[cfg(not(feature = \"smallvec\"))]\n let mut stack = Vec::new();\n\n for (run_index, level_run) in run_sequence.runs.iter().enumerate() {\n for (i, ch) in text.subrange(level_run.clone()).char_indices() {\n let actual_index = level_run.start + i;\n\n // All paren characters are ON.\n // From BidiBrackets.txt:\n // > The Unicode property value stability policy guarantees that characters\n // > which have bpt=o or bpt=c also have bc=ON and Bidi_M=Y\n if original_classes[actual_index] != BidiClass::ON {\n continue;\n }\n\n if let Some(matched) = data_source.bidi_matched_opening_bracket(ch) {\n if matched.is_open {\n // > If an opening paired bracket is found ...\n\n // > ... and there is no room in the stack,\n // > stop processing BD16 for the remainder of the isolating run sequence.\n if stack.len() >= 63 {\n break;\n }\n // > ... push its Bidi_Paired_Bracket property value and its text position onto the stack\n stack.push((matched.opening, actual_index, run_index))\n } else {\n // > If a closing paired bracket is found, do the following\n\n // > Declare a variable that holds a reference to the current stack element\n // > and initialize it with the top element of the stack.\n // AND\n // > Else, if the current stack element is not at the bottom of the stack\n for (stack_index, element) in stack.iter().enumerate().rev() {\n // > Compare the closing paired bracket being inspected or its canonical\n // > equivalent to the bracket in the current stack element.\n if element.0 == matched.opening {\n // > If the values match, meaning the two characters form a bracket pair, then\n\n // > Append the text position in the current stack element together with the\n // > text position of the closing paired bracket to the list.\n let pair = BracketPair {\n start: element.1,\n end: actual_index,\n start_run: element.2,\n end_run: run_index,\n };\n bracket_pairs.push(pair);\n\n // > Pop the stack through the current stack element inclusively.\n stack.truncate(stack_index);\n break;\n }\n }\n }\n }\n }\n }\n // > Sort the list of pairs of text positions in ascending order based on\n // > the text position of the opening paired bracket.\n bracket_pairs.sort_by_key(|r| r.start);\n}\n\n/// 3.3.6 Resolving Implicit Levels\n///\n/// Returns the maximum embedding level in the paragraph.\n///\n/// \n#[cfg_attr(feature = \"flame_it\", flamer::flame)]\npub fn resolve_levels(processing_classes: &[BidiClass], levels: &mut [Level]) -> Level {\n let mut max_level = Level::ltr();\n assert_eq!(processing_classes.len(), levels.len());\n for i in 0..levels.len() {\n match (levels[i].is_rtl(), processing_classes[i]) {\n (false, AN) | (false, EN) => levels[i].raise(2).expect(\"Level number error\"),\n (false, R) | (true, L) | (true, EN) | (true, AN) => {\n levels[i].raise(1).expect(\"Level number error\")\n }\n // handled here\n (_, _) => {}\n }\n max_level = max(max_level, levels[i]);\n }\n\n max_level\n}\n\n/// Neutral or Isolate formatting character (B, S, WS, ON, FSI, LRI, RLI, PDI)\n///\n/// \n#[allow(non_snake_case)]\nfn is_NI(class: BidiClass) -> bool {\n matches!(class, B | S | WS | ON | FSI | LRI | RLI | PDI)\n}\n" }, { "package": "unicode-bidi@0.3.18", "sourcePath": "src/level.rs", "sha256": "ce1eaa9940f1b90bc59aba296488b8cd128aefeb4b6b2e3ecc34da26c569150b", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2017 The Servo Project Developers. See the\n// COPYRIGHT file at the top-level directory of this distribution.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! Bidi Embedding Level\n//!\n//! See [`Level`](struct.Level.html) for more details.\n//!\n//! \n\nuse alloc::{\n string::{String, ToString},\n vec::Vec,\n};\nuse core::slice;\n\nuse super::char_data::BidiClass;\n\n/// Embedding Level\n///\n/// Embedding Levels are numbers between 0 and 126 (inclusive), where even values denote a\n/// left-to-right (LTR) direction and odd values a right-to-left (RTL) direction.\n///\n/// This struct maintains a *valid* status for level numbers, meaning that creating a new level, or\n/// mutating an existing level, with the value smaller than `0` (before conversion to `u8`) or\n/// larger than 125 results in an `Error`.\n///\n/// \n#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]\n#[cfg_attr(feature = \"serde\", derive(serde::Serialize, serde::Deserialize))]\n#[repr(transparent)]\npub struct Level(u8);\n\npub const LTR_LEVEL: Level = Level(0);\npub const RTL_LEVEL: Level = Level(1);\n\nconst MAX_DEPTH: u8 = 125;\n/// During explicit level resolution, embedding level can go as high as `max_depth`.\npub const MAX_EXPLICIT_DEPTH: u8 = MAX_DEPTH;\n/// During implicit level resolution, embedding level can go as high as `max_depth + 1`.\npub const MAX_IMPLICIT_DEPTH: u8 = MAX_DEPTH + 1;\n\n/// Errors that can occur on Level creation or mutation\n#[derive(Debug, PartialEq)]\npub enum Error {\n /// Out-of-range (invalid) embedding level number.\n OutOfRangeNumber,\n}\n\nimpl Level {\n /// New LTR level with smallest number value (0).\n #[inline]\n pub fn ltr() -> Level {\n LTR_LEVEL\n }\n\n /// New RTL level with smallest number value (1).\n #[inline]\n pub fn rtl() -> Level {\n RTL_LEVEL\n }\n\n /// Maximum depth of the directional status stack during implicit resolutions.\n pub fn max_implicit_depth() -> u8 {\n MAX_IMPLICIT_DEPTH\n }\n\n /// Maximum depth of the directional status stack during explicit resolutions.\n pub fn max_explicit_depth() -> u8 {\n MAX_EXPLICIT_DEPTH\n }\n\n // == Inquiries ==\n\n /// Create new level, fail if number is larger than `max_depth + 1`.\n #[inline]\n pub fn new(number: u8) -> Result {\n if number <= MAX_IMPLICIT_DEPTH {\n Ok(Level(number))\n } else {\n Err(Error::OutOfRangeNumber)\n }\n }\n\n /// Create new level, fail if number is larger than `max_depth`.\n #[inline]\n pub fn new_explicit(number: u8) -> Result {\n if number <= MAX_EXPLICIT_DEPTH {\n Ok(Level(number))\n } else {\n Err(Error::OutOfRangeNumber)\n }\n }\n\n // == Inquiries ==\n\n /// The level number.\n #[inline]\n pub fn number(&self) -> u8 {\n self.0\n }\n\n /// If this level is left-to-right.\n #[inline]\n pub fn is_ltr(&self) -> bool {\n self.0 % 2 == 0\n }\n\n /// If this level is right-to-left.\n #[inline]\n pub fn is_rtl(&self) -> bool {\n self.0 % 2 == 1\n }\n\n // == Mutators ==\n\n /// Raise level by `amount`, fail if number is larger than `max_depth + 1`.\n #[inline]\n pub fn raise(&mut self, amount: u8) -> Result<(), Error> {\n match self.0.checked_add(amount) {\n Some(number) => {\n if number <= MAX_IMPLICIT_DEPTH {\n self.0 = number;\n Ok(())\n } else {\n Err(Error::OutOfRangeNumber)\n }\n }\n None => Err(Error::OutOfRangeNumber),\n }\n }\n\n /// Raise level by `amount`, fail if number is larger than `max_depth`.\n #[inline]\n pub fn raise_explicit(&mut self, amount: u8) -> Result<(), Error> {\n match self.0.checked_add(amount) {\n Some(number) => {\n if number <= MAX_EXPLICIT_DEPTH {\n self.0 = number;\n Ok(())\n } else {\n Err(Error::OutOfRangeNumber)\n }\n }\n None => Err(Error::OutOfRangeNumber),\n }\n }\n\n /// Lower level by `amount`, fail if number goes below zero.\n #[inline]\n pub fn lower(&mut self, amount: u8) -> Result<(), Error> {\n match self.0.checked_sub(amount) {\n Some(number) => {\n self.0 = number;\n Ok(())\n }\n None => Err(Error::OutOfRangeNumber),\n }\n }\n\n // == Helpers ==\n\n /// The next LTR (even) level greater than this, or fail if number is larger than `max_depth`.\n #[inline]\n pub fn new_explicit_next_ltr(&self) -> Result {\n Level::new_explicit((self.0 + 2) & !1)\n }\n\n /// The next RTL (odd) level greater than this, or fail if number is larger than `max_depth`.\n #[inline]\n pub fn new_explicit_next_rtl(&self) -> Result {\n Level::new_explicit((self.0 + 1) | 1)\n }\n\n /// The lowest RTL (odd) level greater than or equal to this, or fail if number is larger than\n /// `max_depth + 1`.\n #[inline]\n pub fn new_lowest_ge_rtl(&self) -> Result {\n Level::new(self.0 | 1)\n }\n\n /// Generate a character type based on a level (as specified in steps X10 and N2).\n #[inline]\n pub fn bidi_class(&self) -> BidiClass {\n if self.is_rtl() {\n BidiClass::R\n } else {\n BidiClass::L\n }\n }\n\n pub fn vec(v: &[u8]) -> Vec {\n v.iter().map(|&x| x.into()).collect()\n }\n\n /// Converts a byte slice to a slice of Levels\n ///\n /// Does _not_ check if each level is within bounds (`<=` [`MAX_IMPLICIT_DEPTH`]),\n /// which is not a requirement for safety but is a requirement for correctness of the algorithm.\n pub fn from_slice_unchecked(v: &[u8]) -> &[Level] {\n debug_assert_eq!(core::mem::size_of::(), core::mem::size_of::());\n unsafe {\n // Safety: The two arrays are the same size and layout-compatible since\n // Level is `repr(transparent)` over `u8`\n slice::from_raw_parts(v as *const [u8] as *const u8 as *const Level, v.len())\n }\n }\n}\n\n/// If levels has any RTL (odd) level\n///\n/// This information is usually used to skip re-ordering of text when no RTL level is present\n#[inline]\npub fn has_rtl(levels: &[Level]) -> bool {\n levels.iter().any(|&lvl| lvl.is_rtl())\n}\n\nimpl From for u8 {\n /// Convert to the level number\n #[inline]\n fn from(val: Level) -> Self {\n val.number()\n }\n}\n\nimpl From for Level {\n /// Create level by number\n #[inline]\n fn from(number: u8) -> Level {\n Level::new(number).expect(\"Level number error\")\n }\n}\n\n/// Used for matching levels in conformance tests\nimpl<'a> PartialEq<&'a str> for Level {\n #[inline]\n fn eq(&self, s: &&'a str) -> bool {\n *s == \"x\" || *s == self.0.to_string()\n }\n}\n\n/// Used for matching levels in conformance tests\nimpl PartialEq for Level {\n #[inline]\n fn eq(&self, s: &String) -> bool {\n self == &s.as_str()\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn test_new() {\n assert_eq!(Level::new(0), Ok(Level(0)));\n assert_eq!(Level::new(1), Ok(Level(1)));\n assert_eq!(Level::new(10), Ok(Level(10)));\n assert_eq!(Level::new(125), Ok(Level(125)));\n assert_eq!(Level::new(126), Ok(Level(126)));\n assert_eq!(Level::new(127), Err(Error::OutOfRangeNumber));\n assert_eq!(Level::new(255), Err(Error::OutOfRangeNumber));\n }\n\n #[test]\n fn test_new_explicit() {\n assert_eq!(Level::new_explicit(0), Ok(Level(0)));\n assert_eq!(Level::new_explicit(1), Ok(Level(1)));\n assert_eq!(Level::new_explicit(10), Ok(Level(10)));\n assert_eq!(Level::new_explicit(125), Ok(Level(125)));\n assert_eq!(Level::new_explicit(126), Err(Error::OutOfRangeNumber));\n assert_eq!(Level::new_explicit(255), Err(Error::OutOfRangeNumber));\n }\n\n #[test]\n fn test_is_ltr() {\n assert_eq!(Level(0).is_ltr(), true);\n assert_eq!(Level(1).is_ltr(), false);\n assert_eq!(Level(10).is_ltr(), true);\n assert_eq!(Level(11).is_ltr(), false);\n assert_eq!(Level(124).is_ltr(), true);\n assert_eq!(Level(125).is_ltr(), false);\n }\n\n #[test]\n fn test_is_rtl() {\n assert_eq!(Level(0).is_rtl(), false);\n assert_eq!(Level(1).is_rtl(), true);\n assert_eq!(Level(10).is_rtl(), false);\n assert_eq!(Level(11).is_rtl(), true);\n assert_eq!(Level(124).is_rtl(), false);\n assert_eq!(Level(125).is_rtl(), true);\n }\n\n #[test]\n fn test_raise() {\n let mut level = Level::ltr();\n assert_eq!(level.number(), 0);\n assert!(level.raise(100).is_ok());\n assert_eq!(level.number(), 100);\n assert!(level.raise(26).is_ok());\n assert_eq!(level.number(), 126);\n assert!(level.raise(1).is_err()); // invalid!\n assert!(level.raise(250).is_err()); // overflow!\n assert_eq!(level.number(), 126);\n }\n\n #[test]\n fn test_raise_explicit() {\n let mut level = Level::ltr();\n assert_eq!(level.number(), 0);\n assert!(level.raise_explicit(100).is_ok());\n assert_eq!(level.number(), 100);\n assert!(level.raise_explicit(25).is_ok());\n assert_eq!(level.number(), 125);\n assert!(level.raise_explicit(1).is_err()); // invalid!\n assert!(level.raise_explicit(250).is_err()); // overflow!\n assert_eq!(level.number(), 125);\n }\n\n #[test]\n fn test_lower() {\n let mut level = Level::rtl();\n assert_eq!(level.number(), 1);\n assert!(level.lower(1).is_ok());\n assert_eq!(level.number(), 0);\n assert!(level.lower(1).is_err()); // underflow!\n assert!(level.lower(250).is_err()); // underflow!\n assert_eq!(level.number(), 0);\n }\n\n #[test]\n fn test_has_rtl() {\n assert_eq!(has_rtl(&Level::vec(&[0, 0, 0])), false);\n assert_eq!(has_rtl(&Level::vec(&[0, 1, 0])), true);\n assert_eq!(has_rtl(&Level::vec(&[0, 2, 0])), false);\n assert_eq!(has_rtl(&Level::vec(&[0, 125, 0])), true);\n assert_eq!(has_rtl(&Level::vec(&[0, 126, 0])), false);\n }\n\n #[test]\n fn test_into() {\n let level = Level::rtl();\n let number: u8 = level.into();\n assert_eq!(1u8, number);\n }\n\n #[test]\n fn test_vec() {\n assert_eq!(\n Level::vec(&[0, 1, 125]),\n vec![Level(0), Level(1), Level(125)]\n );\n }\n\n #[test]\n fn test_str_eq() {\n assert_eq!(Level::vec(&[0, 1, 4, 125]), vec![\"0\", \"1\", \"x\", \"125\"]);\n assert_ne!(Level::vec(&[0, 1, 4, 125]), vec![\"0\", \"1\", \"5\", \"125\"]);\n }\n\n #[test]\n fn test_string_eq() {\n assert_eq!(\n Level::vec(&[0, 1, 4, 125]),\n vec![\"0\".to_string(), \"1\".to_string(), \"x\".to_string(), \"125\".to_string()]\n );\n }\n}\n\n#[cfg(all(feature = \"serde\", test))]\nmod serde_tests {\n use super::*;\n use serde_test::{assert_tokens, Token};\n\n #[test]\n fn test_statics() {\n assert_tokens(\n &Level::ltr(),\n &[Token::NewtypeStruct { name: \"Level\" }, Token::U8(0)],\n );\n assert_tokens(\n &Level::rtl(),\n &[Token::NewtypeStruct { name: \"Level\" }, Token::U8(1)],\n );\n }\n\n #[test]\n fn test_new() {\n let level = Level::new(42).unwrap();\n assert_tokens(\n &level,\n &[Token::NewtypeStruct { name: \"Level\" }, Token::U8(42)],\n );\n }\n}\n" }, { "package": "unicode-bidi@0.3.18", "sourcePath": "src/lib.rs", "sha256": "8a611ccb9d56c176f8428edc14b1e6fc1fbe8ed22bc72118fb987e6a29638197", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Servo Project Developers. See the\n// COPYRIGHT file at the top-level directory of this distribution.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! This crate implements the [Unicode Bidirectional Algorithm][tr9] for display of mixed\n//! right-to-left and left-to-right text. It is written in safe Rust, compatible with the\n//! current stable release.\n//!\n//! ## Example\n//!\n//! ```rust\n//! # #[cfg(feature = \"hardcoded-data\")] {\n//! use unicode_bidi::BidiInfo;\n//!\n//! // This example text is defined using `concat!` because some browsers\n//! // and text editors have trouble displaying bidi strings.\n//! let text = concat![\n//! \"א\",\n//! \"ב\",\n//! \"ג\",\n//! \"a\",\n//! \"b\",\n//! \"c\",\n//! ];\n//!\n//! // Resolve embedding levels within the text. Pass `None` to detect the\n//! // paragraph level automatically.\n//! let bidi_info = BidiInfo::new(&text, None);\n//!\n//! // This paragraph has embedding level 1 because its first strong character is RTL.\n//! assert_eq!(bidi_info.paragraphs.len(), 1);\n//! let para = &bidi_info.paragraphs[0];\n//! assert_eq!(para.level.number(), 1);\n//! assert_eq!(para.level.is_rtl(), true);\n//!\n//! // Re-ordering is done after wrapping each paragraph into a sequence of\n//! // lines. For this example, I'll just use a single line that spans the\n//! // entire paragraph.\n//! let line = para.range.clone();\n//!\n//! let display = bidi_info.reorder_line(para, line);\n//! assert_eq!(display, concat![\n//! \"a\",\n//! \"b\",\n//! \"c\",\n//! \"ג\",\n//! \"ב\",\n//! \"א\",\n//! ]);\n//! # } // feature = \"hardcoded-data\"\n//! ```\n//!\n//! # Features\n//!\n//! - `std`: Enabled by default, but can be disabled to make `unicode_bidi`\n//! `#![no_std]` + `alloc` compatible.\n//! - `hardcoded-data`: Enabled by default. Includes hardcoded Unicode bidi data and more convenient APIs.\n//! - `serde`: Adds [`serde::Serialize`] and [`serde::Deserialize`]\n//! implementations to relevant types.\n//!\n//! [tr9]: \n\n#![no_std]\n// We need to link to std to make doc tests work on older Rust versions\n#[cfg(feature = \"std\")]\nextern crate std;\n#[macro_use]\nextern crate alloc;\n#[cfg(feature = \"smallvec\")]\nextern crate smallvec;\n\npub mod data_source;\npub mod deprecated;\npub mod format_chars;\npub mod level;\npub mod utf16;\n\nmod char_data;\nmod explicit;\nmod implicit;\nmod prepare;\n\npub use crate::char_data::{BidiClass, UNICODE_VERSION};\npub use crate::data_source::BidiDataSource;\npub use crate::level::{Level, LTR_LEVEL, RTL_LEVEL};\npub use crate::prepare::{LevelRun, LevelRunVec};\n\n#[cfg(feature = \"hardcoded-data\")]\npub use crate::char_data::{bidi_class, HardcodedBidiData};\n\nuse alloc::borrow::Cow;\nuse alloc::string::String;\nuse alloc::vec::Vec;\nuse core::char;\nuse core::cmp;\nuse core::iter::repeat;\nuse core::ops::Range;\nuse core::str::CharIndices;\n#[cfg(feature = \"smallvec\")]\nuse smallvec::SmallVec;\n\nuse crate::format_chars as chars;\nuse crate::BidiClass::*;\n\n/// Trait that abstracts over a text source for use by the bidi algorithms.\n/// We implement this for str (UTF-8) and for [u16] (UTF-16, native-endian).\n/// (For internal unicode-bidi use; API may be unstable.)\n/// This trait is sealed and cannot be implemented for types outside this crate.\npub trait TextSource<'text>: private::Sealed {\n type CharIter: Iterator;\n type CharIndexIter: Iterator;\n type IndexLenIter: Iterator;\n\n /// Return the length of the text in code units.\n #[doc(hidden)]\n fn len(&self) -> usize;\n\n /// Get the character at a given code unit index, along with its length in code units.\n /// Returns None if index is out of range, or points inside a multi-code-unit character.\n /// Returns REPLACEMENT_CHARACTER for any unpaired surrogates in UTF-16.\n #[doc(hidden)]\n fn char_at(&self, index: usize) -> Option<(char, usize)>;\n\n /// Return a subrange of the text, indexed by code units.\n /// (We don't implement all of the Index trait, just the minimum we use.)\n #[doc(hidden)]\n fn subrange(&self, range: Range) -> &Self;\n\n /// An iterator over the text returning Unicode characters,\n /// REPLACEMENT_CHAR for invalid code units.\n #[doc(hidden)]\n fn chars(&'text self) -> Self::CharIter;\n\n /// An iterator over the text returning (index, char) tuples,\n /// where index is the starting code-unit index of the character,\n /// and char is its Unicode value (or REPLACEMENT_CHAR if invalid).\n #[doc(hidden)]\n fn char_indices(&'text self) -> Self::CharIndexIter;\n\n /// An iterator over the text returning (index, length) tuples,\n /// where index is the starting code-unit index of the character,\n /// and length is its length in code units.\n #[doc(hidden)]\n fn indices_lengths(&'text self) -> Self::IndexLenIter;\n\n /// Number of code units the given character uses.\n #[doc(hidden)]\n fn char_len(ch: char) -> usize;\n}\n\nmod private {\n pub trait Sealed {}\n\n // Implement for str and [u16] only.\n impl Sealed for str {}\n impl Sealed for [u16] {}\n}\n\n#[derive(PartialEq, Debug)]\npub enum Direction {\n Ltr,\n Rtl,\n Mixed,\n}\n\n/// Bidi information about a single paragraph\n#[derive(Clone, Debug, PartialEq)]\npub struct ParagraphInfo {\n /// The paragraphs boundaries within the text, as byte indices.\n ///\n /// TODO: Shrink this to only include the starting index?\n pub range: Range,\n\n /// The paragraph embedding level.\n ///\n /// \n pub level: Level,\n}\n\nimpl ParagraphInfo {\n /// Gets the length of the paragraph in the source text.\n pub fn len(&self) -> usize {\n self.range.end - self.range.start\n }\n}\n\n/// Initial bidi information of the text.\n///\n/// Contains the text paragraphs and `BidiClass` of its characters.\n#[derive(PartialEq, Debug)]\npub struct InitialInfo<'text> {\n /// The text\n pub text: &'text str,\n\n /// The BidiClass of the character at each byte in the text.\n /// If a character is multiple bytes, its class will appear multiple times in the vector.\n pub original_classes: Vec,\n\n /// The boundaries and level of each paragraph within the text.\n pub paragraphs: Vec,\n}\n\nimpl<'text> InitialInfo<'text> {\n /// Find the paragraphs and BidiClasses in a string of text.\n ///\n /// \n ///\n /// Also sets the class for each First Strong Isolate initiator (FSI) to LRI or RLI if a strong\n /// character is found before the matching PDI. If no strong character is found, the class will\n /// remain FSI, and it's up to later stages to treat these as LRI when needed.\n ///\n /// The `hardcoded-data` Cargo feature (enabled by default) must be enabled to use this.\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[cfg(feature = \"hardcoded-data\")]\n pub fn new(text: &str, default_para_level: Option) -> InitialInfo<'_> {\n Self::new_with_data_source(&HardcodedBidiData, text, default_para_level)\n }\n\n /// Find the paragraphs and BidiClasses in a string of text, with a custom [`BidiDataSource`]\n /// for Bidi data. If you just wish to use the hardcoded Bidi data, please use [`InitialInfo::new()`]\n /// instead (enabled with tbe default `hardcoded-data` Cargo feature)\n ///\n /// \n ///\n /// Also sets the class for each First Strong Isolate initiator (FSI) to LRI or RLI if a strong\n /// character is found before the matching PDI. If no strong character is found, the class will\n /// remain FSI, and it's up to later stages to treat these as LRI when needed.\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn new_with_data_source<'a, D: BidiDataSource>(\n data_source: &D,\n text: &'a str,\n default_para_level: Option,\n ) -> InitialInfo<'a> {\n InitialInfoExt::new_with_data_source(data_source, text, default_para_level).base\n }\n}\n\n/// Extended version of InitialInfo (not public API).\n#[derive(PartialEq, Debug)]\nstruct InitialInfoExt<'text> {\n /// The base InitialInfo for the text, recording its paragraphs and bidi classes.\n base: InitialInfo<'text>,\n\n /// Parallel to base.paragraphs, records whether each paragraph is \"pure LTR\" that\n /// requires no further bidi processing (i.e. there are no RTL characters or bidi\n /// control codes present), and whether any bidi isolation controls are present.\n flags: Vec,\n}\n\n#[derive(PartialEq, Debug)]\nstruct ParagraphInfoFlags {\n is_pure_ltr: bool,\n has_isolate_controls: bool,\n}\n\nimpl<'text> InitialInfoExt<'text> {\n /// Find the paragraphs and BidiClasses in a string of text, with a custom [`BidiDataSource`]\n /// for Bidi data. If you just wish to use the hardcoded Bidi data, please use [`InitialInfo::new()`]\n /// instead (enabled with tbe default `hardcoded-data` Cargo feature)\n ///\n /// \n ///\n /// Also sets the class for each First Strong Isolate initiator (FSI) to LRI or RLI if a strong\n /// character is found before the matching PDI. If no strong character is found, the class will\n /// remain FSI, and it's up to later stages to treat these as LRI when needed.\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn new_with_data_source<'a, D: BidiDataSource>(\n data_source: &D,\n text: &'a str,\n default_para_level: Option,\n ) -> InitialInfoExt<'a> {\n let mut paragraphs = Vec::::new();\n let mut flags = Vec::::new();\n let (original_classes, _, _, _) = compute_initial_info(\n data_source,\n text,\n default_para_level,\n Some((&mut paragraphs, &mut flags)),\n );\n\n InitialInfoExt {\n base: InitialInfo {\n text,\n original_classes,\n paragraphs,\n },\n flags,\n }\n }\n}\n\n/// Implementation of initial-info computation for both BidiInfo and ParagraphBidiInfo.\n/// To treat the text as (potentially) multiple paragraphs, the caller should pass the\n/// pair of optional outparam arrays to receive the ParagraphInfo and pure-ltr flags\n/// for each paragraph. Passing None for split_paragraphs will ignore any paragraph-\n/// separator characters in the text, treating it just as a single paragraph.\n/// Returns the array of BidiClass values for each code unit of the text, along with\n/// the embedding level and pure-ltr flag for the *last* (or only) paragraph.\nfn compute_initial_info<'a, D: BidiDataSource, T: TextSource<'a> + ?Sized>(\n data_source: &D,\n text: &'a T,\n default_para_level: Option,\n mut split_paragraphs: Option<(&mut Vec, &mut Vec)>,\n) -> (Vec, Level, bool, bool) {\n let mut original_classes = Vec::with_capacity(text.len());\n\n // The stack contains the starting code unit index for each nested isolate we're inside.\n #[cfg(feature = \"smallvec\")]\n let mut isolate_stack = SmallVec::<[usize; 8]>::new();\n #[cfg(not(feature = \"smallvec\"))]\n let mut isolate_stack = Vec::new();\n\n debug_assert!(\n if let Some((ref paragraphs, ref flags)) = split_paragraphs {\n paragraphs.is_empty() && flags.is_empty()\n } else {\n true\n }\n );\n\n let mut para_start = 0;\n let mut para_level = default_para_level;\n\n // Per-paragraph flag: can subsequent processing be skipped? Set to false if any\n // RTL characters or bidi control characters are encountered in the paragraph.\n let mut is_pure_ltr = true;\n // Set to true if any bidi isolation controls are present in the paragraph.\n let mut has_isolate_controls = false;\n\n #[cfg(feature = \"flame_it\")]\n flame::start(\"compute_initial_info(): iter text.char_indices()\");\n\n for (i, c) in text.char_indices() {\n let class = data_source.bidi_class(c);\n\n #[cfg(feature = \"flame_it\")]\n flame::start(\"original_classes.extend()\");\n\n let len = T::char_len(c);\n original_classes.extend(repeat(class).take(len));\n\n #[cfg(feature = \"flame_it\")]\n flame::end(\"original_classes.extend()\");\n\n match class {\n B => {\n if let Some((ref mut paragraphs, ref mut flags)) = split_paragraphs {\n // P1. Split the text into separate paragraphs. The paragraph separator is kept\n // with the previous paragraph.\n let para_end = i + len;\n paragraphs.push(ParagraphInfo {\n range: para_start..para_end,\n // P3. If no character is found in p2, set the paragraph level to zero.\n level: para_level.unwrap_or(LTR_LEVEL),\n });\n flags.push(ParagraphInfoFlags {\n is_pure_ltr,\n has_isolate_controls,\n });\n // Reset state for the start of the next paragraph.\n para_start = para_end;\n // TODO: Support defaulting to direction of previous paragraph\n //\n // \n para_level = default_para_level;\n is_pure_ltr = true;\n has_isolate_controls = false;\n isolate_stack.clear();\n }\n }\n\n L | R | AL => {\n if class != L {\n is_pure_ltr = false;\n }\n match isolate_stack.last() {\n Some(&start) => {\n if original_classes[start] == FSI {\n // X5c. If the first strong character between FSI and its matching\n // PDI is R or AL, treat it as RLI. Otherwise, treat it as LRI.\n for j in 0..T::char_len(chars::FSI) {\n original_classes[start + j] = if class == L { LRI } else { RLI };\n }\n }\n }\n\n None => {\n if para_level.is_none() {\n // P2. Find the first character of type L, AL, or R, while skipping\n // any characters between an isolate initiator and its matching\n // PDI.\n para_level = Some(if class != L { RTL_LEVEL } else { LTR_LEVEL });\n }\n }\n }\n }\n\n AN | LRE | RLE | LRO | RLO => {\n is_pure_ltr = false;\n }\n\n RLI | LRI | FSI => {\n is_pure_ltr = false;\n has_isolate_controls = true;\n isolate_stack.push(i);\n }\n\n PDI => {\n isolate_stack.pop();\n }\n\n _ => {}\n }\n }\n\n if let Some((paragraphs, flags)) = split_paragraphs {\n if para_start < text.len() {\n paragraphs.push(ParagraphInfo {\n range: para_start..text.len(),\n level: para_level.unwrap_or(LTR_LEVEL),\n });\n flags.push(ParagraphInfoFlags {\n is_pure_ltr,\n has_isolate_controls,\n });\n }\n debug_assert_eq!(paragraphs.len(), flags.len());\n }\n debug_assert_eq!(original_classes.len(), text.len());\n\n #[cfg(feature = \"flame_it\")]\n flame::end(\"compute_initial_info(): iter text.char_indices()\");\n\n (\n original_classes,\n para_level.unwrap_or(LTR_LEVEL),\n is_pure_ltr,\n has_isolate_controls,\n )\n}\n\n/// Bidi information of the text.\n///\n/// The `original_classes` and `levels` vectors are indexed by byte offsets into the text. If a\n/// character is multiple bytes wide, then its class and level will appear multiple times in these\n/// vectors.\n// TODO: Impl `struct StringProperty { values: Vec }` and use instead of Vec\n#[derive(Debug, PartialEq)]\npub struct BidiInfo<'text> {\n /// The text\n pub text: &'text str,\n\n /// The BidiClass of the character at each byte in the text.\n pub original_classes: Vec,\n\n /// The directional embedding level of each byte in the text.\n pub levels: Vec,\n\n /// The boundaries and paragraph embedding level of each paragraph within the text.\n ///\n /// TODO: Use SmallVec or similar to avoid overhead when there are only one or two paragraphs?\n /// Or just don't include the first paragraph, which always starts at 0?\n pub paragraphs: Vec,\n}\n\nimpl<'text> BidiInfo<'text> {\n /// Split the text into paragraphs and determine the bidi embedding levels for each paragraph.\n ///\n ///\n /// The `hardcoded-data` Cargo feature (enabled by default) must be enabled to use this.\n ///\n /// TODO: In early steps, check for special cases that allow later steps to be skipped. like\n /// text that is entirely LTR. See the `nsBidi` class from Gecko for comparison.\n ///\n /// TODO: Support auto-RTL base direction\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[cfg(feature = \"hardcoded-data\")]\n #[inline]\n pub fn new(text: &str, default_para_level: Option) -> BidiInfo<'_> {\n Self::new_with_data_source(&HardcodedBidiData, text, default_para_level)\n }\n\n /// Split the text into paragraphs and determine the bidi embedding levels for each paragraph, with a custom [`BidiDataSource`]\n /// for Bidi data. If you just wish to use the hardcoded Bidi data, please use [`BidiInfo::new()`]\n /// instead (enabled with tbe default `hardcoded-data` Cargo feature).\n ///\n /// TODO: In early steps, check for special cases that allow later steps to be skipped. like\n /// text that is entirely LTR. See the `nsBidi` class from Gecko for comparison.\n ///\n /// TODO: Support auto-RTL base direction\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn new_with_data_source<'a, D: BidiDataSource>(\n data_source: &D,\n text: &'a str,\n default_para_level: Option,\n ) -> BidiInfo<'a> {\n let InitialInfoExt { base, flags, .. } =\n InitialInfoExt::new_with_data_source(data_source, text, default_para_level);\n\n let mut levels = Vec::::with_capacity(text.len());\n let mut processing_classes = base.original_classes.clone();\n\n for (para, flags) in base.paragraphs.iter().zip(flags.iter()) {\n let text = &text[para.range.clone()];\n let original_classes = &base.original_classes[para.range.clone()];\n\n compute_bidi_info_for_para(\n data_source,\n para,\n flags.is_pure_ltr,\n flags.has_isolate_controls,\n text,\n original_classes,\n &mut processing_classes,\n &mut levels,\n );\n }\n\n BidiInfo {\n text,\n original_classes: base.original_classes,\n paragraphs: base.paragraphs,\n levels,\n }\n }\n\n /// Produce the levels for this paragraph as needed for reordering, one level per *byte*\n /// in the paragraph. The returned vector includes bytes that are not included\n /// in the `line`, but will not adjust them.\n ///\n /// This runs [Rule L1], you can run\n /// [Rule L2] by calling [`Self::reorder_visual()`].\n /// If doing so, you may prefer to use [`Self::reordered_levels_per_char()`] instead\n /// to avoid non-byte indices.\n ///\n /// For an all-in-one reordering solution, consider using [`Self::reorder_visual()`].\n ///\n /// [Rule L1]: https://www.unicode.org/reports/tr9/#L1\n /// [Rule L2]: https://www.unicode.org/reports/tr9/#L2\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reordered_levels(&self, para: &ParagraphInfo, line: Range) -> Vec {\n assert!(line.start <= self.levels.len());\n assert!(line.end <= self.levels.len());\n\n let mut levels = self.levels.clone();\n let line_classes = &self.original_classes[line.clone()];\n let line_levels = &mut levels[line.clone()];\n\n reorder_levels(\n line_classes,\n line_levels,\n self.text.subrange(line),\n para.level,\n );\n\n levels\n }\n\n /// Produce the levels for this paragraph as needed for reordering, one level per *character*\n /// in the paragraph. The returned vector includes characters that are not included\n /// in the `line`, but will not adjust them.\n ///\n /// This runs [Rule L1], you can run\n /// [Rule L2] by calling [`Self::reorder_visual()`].\n /// If doing so, you may prefer to use [`Self::reordered_levels_per_char()`] instead\n /// to avoid non-byte indices.\n ///\n /// For an all-in-one reordering solution, consider using [`Self::reorder_visual()`].\n ///\n /// [Rule L1]: https://www.unicode.org/reports/tr9/#L1\n /// [Rule L2]: https://www.unicode.org/reports/tr9/#L2\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reordered_levels_per_char(\n &self,\n para: &ParagraphInfo,\n line: Range,\n ) -> Vec {\n let levels = self.reordered_levels(para, line);\n self.text.char_indices().map(|(i, _)| levels[i]).collect()\n }\n\n /// Re-order a line based on resolved levels and return the line in display order.\n ///\n /// This does not apply [Rule L3] or [Rule L4] around combining characters or mirroring.\n ///\n /// [Rule L3]: https://www.unicode.org/reports/tr9/#L3\n /// [Rule L4]: https://www.unicode.org/reports/tr9/#L4\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reorder_line(&self, para: &ParagraphInfo, line: Range) -> Cow<'text, str> {\n if !level::has_rtl(&self.levels[line.clone()]) {\n return self.text[line].into();\n }\n let (levels, runs) = self.visual_runs(para, line.clone());\n reorder_line(self.text, line, levels, runs)\n }\n\n /// Reorders pre-calculated levels of a sequence of characters.\n ///\n /// NOTE: This is a convenience method that does not use a `Paragraph` object. It is\n /// intended to be used when an application has determined the levels of the objects (character sequences)\n /// and just needs to have them reordered.\n ///\n /// the index map will result in `indexMap[visualIndex]==logicalIndex`.\n ///\n /// This only runs [Rule L2](http://www.unicode.org/reports/tr9/#L2) as it does not have\n /// information about the actual text.\n ///\n /// Furthermore, if `levels` is an array that is aligned with code units, bytes within a codepoint may be\n /// reversed. You may need to fix up the map to deal with this. Alternatively, only pass in arrays where each `Level`\n /// is for a single code point.\n ///\n ///\n /// # # Example\n /// ```\n /// use unicode_bidi::BidiInfo;\n /// use unicode_bidi::Level;\n ///\n /// let l0 = Level::from(0);\n /// let l1 = Level::from(1);\n /// let l2 = Level::from(2);\n ///\n /// let levels = vec![l0, l0, l0, l0];\n /// let index_map = BidiInfo::reorder_visual(&levels);\n /// assert_eq!(levels.len(), index_map.len());\n /// assert_eq!(index_map, [0, 1, 2, 3]);\n ///\n /// let levels: Vec = vec![l0, l0, l0, l1, l1, l1, l2, l2];\n /// let index_map = BidiInfo::reorder_visual(&levels);\n /// assert_eq!(levels.len(), index_map.len());\n /// assert_eq!(index_map, [0, 1, 2, 6, 7, 5, 4, 3]);\n /// ```\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[inline]\n pub fn reorder_visual(levels: &[Level]) -> Vec {\n reorder_visual(levels)\n }\n\n /// Find the level runs within a line and return them in visual order.\n ///\n /// `line` is a range of bytes indices within `levels`.\n ///\n /// The first return value is a vector of levels used by the reordering algorithm,\n /// i.e. the result of [Rule L1]. The second return value is a vector of level runs,\n /// the result of [Rule L2], showing the visual order that each level run (a run of text with the\n /// same level) should be displayed. Within each run, the display order can be checked\n /// against the Level vector.\n ///\n /// This does not handle [Rule L3] (combining characters) or [Rule L4] (mirroring),\n /// as that should be handled by the engine using this API.\n ///\n /// Conceptually, this is the same as running [`Self::reordered_levels()`] followed by\n /// [`Self::reorder_visual()`], however it returns the result as a list of level runs instead\n /// of producing a level map, since one may wish to deal with the fact that this is operating on\n /// byte rather than character indices.\n ///\n /// \n ///\n /// [Rule L1]: https://www.unicode.org/reports/tr9/#L1\n /// [Rule L2]: https://www.unicode.org/reports/tr9/#L2\n /// [Rule L3]: https://www.unicode.org/reports/tr9/#L3\n /// [Rule L4]: https://www.unicode.org/reports/tr9/#L4\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[inline]\n pub fn visual_runs(\n &self,\n para: &ParagraphInfo,\n line: Range,\n ) -> (Vec, Vec) {\n let levels = self.reordered_levels(para, line.clone());\n visual_runs_for_line(levels, &line)\n }\n\n /// If processed text has any computed RTL levels\n ///\n /// This information is usually used to skip re-ordering of text when no RTL level is present\n #[inline]\n pub fn has_rtl(&self) -> bool {\n level::has_rtl(&self.levels)\n }\n}\n\n/// Bidi information of text treated as a single paragraph.\n///\n/// The `original_classes` and `levels` vectors are indexed by byte offsets into the text. If a\n/// character is multiple bytes wide, then its class and level will appear multiple times in these\n/// vectors.\n#[derive(Debug, PartialEq)]\npub struct ParagraphBidiInfo<'text> {\n /// The text\n pub text: &'text str,\n\n /// The BidiClass of the character at each byte in the text.\n pub original_classes: Vec,\n\n /// The directional embedding level of each byte in the text.\n pub levels: Vec,\n\n /// The paragraph embedding level.\n pub paragraph_level: Level,\n\n /// Whether the paragraph is purely LTR.\n pub is_pure_ltr: bool,\n}\n\nimpl<'text> ParagraphBidiInfo<'text> {\n /// Determine the bidi embedding level.\n ///\n ///\n /// The `hardcoded-data` Cargo feature (enabled by default) must be enabled to use this.\n ///\n /// TODO: In early steps, check for special cases that allow later steps to be skipped. like\n /// text that is entirely LTR. See the `nsBidi` class from Gecko for comparison.\n ///\n /// TODO: Support auto-RTL base direction\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[cfg(feature = \"hardcoded-data\")]\n #[inline]\n pub fn new(text: &str, default_para_level: Option) -> ParagraphBidiInfo<'_> {\n Self::new_with_data_source(&HardcodedBidiData, text, default_para_level)\n }\n\n /// Determine the bidi embedding level, with a custom [`BidiDataSource`]\n /// for Bidi data. If you just wish to use the hardcoded Bidi data, please use [`BidiInfo::new()`]\n /// instead (enabled with tbe default `hardcoded-data` Cargo feature).\n ///\n /// (This is the single-paragraph equivalent of BidiInfo::new_with_data_source,\n /// and should be kept in sync with it.\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn new_with_data_source<'a, D: BidiDataSource>(\n data_source: &D,\n text: &'a str,\n default_para_level: Option,\n ) -> ParagraphBidiInfo<'a> {\n // Here we could create a ParagraphInitialInfo struct to parallel the one\n // used by BidiInfo, but there doesn't seem any compelling reason for it.\n let (original_classes, paragraph_level, is_pure_ltr, has_isolate_controls) =\n compute_initial_info(data_source, text, default_para_level, None);\n\n let mut levels = Vec::::with_capacity(text.len());\n let mut processing_classes = original_classes.clone();\n\n let para_info = ParagraphInfo {\n range: Range {\n start: 0,\n end: text.len(),\n },\n level: paragraph_level,\n };\n\n compute_bidi_info_for_para(\n data_source,\n ¶_info,\n is_pure_ltr,\n has_isolate_controls,\n text,\n &original_classes,\n &mut processing_classes,\n &mut levels,\n );\n\n ParagraphBidiInfo {\n text,\n original_classes,\n levels,\n paragraph_level,\n is_pure_ltr,\n }\n }\n\n /// Produce the levels for this paragraph as needed for reordering, one level per *byte*\n /// in the paragraph. The returned vector includes bytes that are not included\n /// in the `line`, but will not adjust them.\n ///\n /// See BidiInfo::reordered_levels for details.\n ///\n /// (This should be kept in sync with BidiInfo::reordered_levels.)\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reordered_levels(&self, line: Range) -> Vec {\n assert!(line.start <= self.levels.len());\n assert!(line.end <= self.levels.len());\n\n let mut levels = self.levels.clone();\n let line_classes = &self.original_classes[line.clone()];\n let line_levels = &mut levels[line.clone()];\n\n reorder_levels(\n line_classes,\n line_levels,\n self.text.subrange(line),\n self.paragraph_level,\n );\n\n levels\n }\n\n /// Produce the levels for this paragraph as needed for reordering, one level per *character*\n /// in the paragraph. The returned vector includes characters that are not included\n /// in the `line`, but will not adjust them.\n ///\n /// See BidiInfo::reordered_levels_per_char for details.\n ///\n /// (This should be kept in sync with BidiInfo::reordered_levels_per_char.)\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reordered_levels_per_char(&self, line: Range) -> Vec {\n let levels = self.reordered_levels(line);\n self.text.char_indices().map(|(i, _)| levels[i]).collect()\n }\n\n /// Re-order a line based on resolved levels and return the line in display order.\n ///\n /// See BidiInfo::reorder_line for details.\n ///\n /// (This should be kept in sync with BidiInfo::reorder_line.)\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reorder_line(&self, line: Range) -> Cow<'text, str> {\n if !level::has_rtl(&self.levels[line.clone()]) {\n return self.text[line].into();\n }\n\n let (levels, runs) = self.visual_runs(line.clone());\n\n reorder_line(self.text, line, levels, runs)\n }\n\n /// Reorders pre-calculated levels of a sequence of characters.\n ///\n /// See BidiInfo::reorder_visual for details.\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[inline]\n pub fn reorder_visual(levels: &[Level]) -> Vec {\n reorder_visual(levels)\n }\n\n /// Find the level runs within a line and return them in visual order.\n ///\n /// `line` is a range of bytes indices within `levels`.\n ///\n /// See BidiInfo::visual_runs for details.\n ///\n /// (This should be kept in sync with BidiInfo::visual_runs.)\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[inline]\n pub fn visual_runs(&self, line: Range) -> (Vec, Vec) {\n let levels = self.reordered_levels(line.clone());\n visual_runs_for_line(levels, &line)\n }\n\n /// If processed text has any computed RTL levels\n ///\n /// This information is usually used to skip re-ordering of text when no RTL level is present\n #[inline]\n pub fn has_rtl(&self) -> bool {\n !self.is_pure_ltr\n }\n\n /// Return the paragraph's Direction (Ltr, Rtl, or Mixed) based on its levels.\n #[inline]\n pub fn direction(&self) -> Direction {\n para_direction(&self.levels)\n }\n}\n\n/// Return a line of the text in display order based on resolved levels.\n///\n/// `text` the full text passed to the `BidiInfo` or `ParagraphBidiInfo` for analysis\n/// `line` a range of byte indices within `text` corresponding to one line\n/// `levels` array of `Level` values, with `line`'s levels reordered into visual order\n/// `runs` array of `LevelRun`s in visual order\n///\n/// (`levels` and `runs` are the result of calling `BidiInfo::visual_runs()` or\n/// `ParagraphBidiInfo::visual_runs()` for the line of interest.)\n///\n/// Returns: the reordered text of the line.\n///\n/// This does not apply [Rule L3] or [Rule L4] around combining characters or mirroring.\n///\n/// [Rule L3]: https://www.unicode.org/reports/tr9/#L3\n/// [Rule L4]: https://www.unicode.org/reports/tr9/#L4\nfn reorder_line(\n text: &str,\n line: Range,\n levels: Vec,\n runs: Vec,\n) -> Cow<'_, str> {\n // If all isolating run sequences are LTR, no reordering is needed\n if runs.iter().all(|run| levels[run.start].is_ltr()) {\n return text[line].into();\n }\n\n let mut result = String::with_capacity(line.len());\n for run in runs {\n if levels[run.start].is_rtl() {\n result.extend(text[run].chars().rev());\n } else {\n result.push_str(&text[run]);\n }\n }\n result.into()\n}\n\n/// Find the level runs within a line and return them in visual order.\n///\n/// `line` is a range of code-unit indices within `levels`.\n///\n/// The first return value is a vector of levels used by the reordering algorithm,\n/// i.e. the result of [Rule L1]. The second return value is a vector of level runs,\n/// the result of [Rule L2], showing the visual order that each level run (a run of text with the\n/// same level) should be displayed. Within each run, the display order can be checked\n/// against the Level vector.\n///\n/// This does not handle [Rule L3] (combining characters) or [Rule L4] (mirroring),\n/// as that should be handled by the engine using this API.\n///\n/// Conceptually, this is the same as running [`reordered_levels()`] followed by\n/// [`reorder_visual()`], however it returns the result as a list of level runs instead\n/// of producing a level map, since one may wish to deal with the fact that this is operating on\n/// byte rather than character indices.\n///\n/// \n///\n/// [Rule L1]: https://www.unicode.org/reports/tr9/#L1\n/// [Rule L2]: https://www.unicode.org/reports/tr9/#L2\n/// [Rule L3]: https://www.unicode.org/reports/tr9/#L3\n/// [Rule L4]: https://www.unicode.org/reports/tr9/#L4\nfn visual_runs_for_line(levels: Vec, line: &Range) -> (Vec, Vec) {\n // Find consecutive level runs.\n let mut runs = Vec::new();\n let mut start = line.start;\n let mut run_level = levels[start];\n let mut min_level = run_level;\n let mut max_level = run_level;\n\n for (i, &new_level) in levels.iter().enumerate().take(line.end).skip(start + 1) {\n if new_level != run_level {\n // End of the previous run, start of a new one.\n runs.push(start..i);\n start = i;\n run_level = new_level;\n min_level = cmp::min(run_level, min_level);\n max_level = cmp::max(run_level, max_level);\n }\n }\n runs.push(start..line.end);\n\n let run_count = runs.len();\n\n // Re-order the odd runs.\n // \n\n // Stop at the lowest *odd* level.\n min_level = min_level.new_lowest_ge_rtl().expect(\"Level error\");\n // This loop goes through contiguous chunks of level runs that have a level\n // ≥ max_level and reverses their contents, reducing max_level by 1 each time.\n while max_level >= min_level {\n // Look for the start of a sequence of consecutive runs of max_level or higher.\n let mut seq_start = 0;\n while seq_start < run_count {\n if levels[runs[seq_start].start] < max_level {\n seq_start += 1;\n continue;\n }\n\n // Found the start of a sequence. Now find the end.\n let mut seq_end = seq_start + 1;\n while seq_end < run_count {\n if levels[runs[seq_end].start] < max_level {\n break;\n }\n seq_end += 1;\n }\n // Reverse the runs within this sequence.\n runs[seq_start..seq_end].reverse();\n\n seq_start = seq_end;\n }\n max_level\n .lower(1)\n .expect(\"Lowering embedding level below zero\");\n }\n (levels, runs)\n}\n\n/// Reorders pre-calculated levels of a sequence of characters.\n///\n/// NOTE: This is a convenience method that does not use a `Paragraph` object. It is\n/// intended to be used when an application has determined the levels of the objects (character sequences)\n/// and just needs to have them reordered.\n///\n/// the index map will result in `indexMap[visualIndex]==logicalIndex`.\n///\n/// This only runs [Rule L2](http://www.unicode.org/reports/tr9/#L2) as it does not have\n/// information about the actual text.\n///\n/// Furthermore, if `levels` is an array that is aligned with code units, bytes within a codepoint may be\n/// reversed. You may need to fix up the map to deal with this. Alternatively, only pass in arrays where each `Level`\n/// is for a single code point.\nfn reorder_visual(levels: &[Level]) -> Vec {\n // Gets the next range of characters after start_index with a level greater\n // than or equal to `max`\n fn next_range(levels: &[level::Level], mut start_index: usize, max: Level) -> Range {\n if levels.is_empty() || start_index >= levels.len() {\n return start_index..start_index;\n }\n while let Some(l) = levels.get(start_index) {\n if *l >= max {\n break;\n }\n start_index += 1;\n }\n\n if levels.get(start_index).is_none() {\n // If at the end of the array, adding one will\n // produce an out-of-range end element\n return start_index..start_index;\n }\n\n let mut end_index = start_index + 1;\n while let Some(l) = levels.get(end_index) {\n if *l < max {\n return start_index..end_index;\n }\n end_index += 1;\n }\n\n start_index..end_index\n }\n\n // This implementation is similar to the L2 implementation in `visual_runs()`\n // but it cannot benefit from a precalculated LevelRun vector so needs to be different.\n\n if levels.is_empty() {\n return vec![];\n }\n\n // Get the min and max levels\n let (mut min, mut max) = levels\n .iter()\n .fold((levels[0], levels[0]), |(min, max), &l| {\n (cmp::min(min, l), cmp::max(max, l))\n });\n\n // Initialize an index map\n let mut result: Vec = (0..levels.len()).collect();\n\n if min == max && min.is_ltr() {\n // Everything is LTR and at the same level, do nothing\n return result;\n }\n\n // Stop at the lowest *odd* level, since everything below that\n // is LTR and does not need further reordering\n min = min.new_lowest_ge_rtl().expect(\"Level error\");\n\n // For each max level, take all contiguous chunks of\n // levels ≥ max and reverse them\n //\n // We can do this check with the original levels instead of checking reorderings because all\n // prior reorderings will have been for contiguous chunks of levels >> max, which will\n // be a subset of these chunks anyway.\n while min <= max {\n let mut range = 0..0;\n loop {\n range = next_range(levels, range.end, max);\n result[range.clone()].reverse();\n\n if range.end >= levels.len() {\n break;\n }\n }\n\n max.lower(1).expect(\"Level error\");\n }\n\n result\n}\n\n/// The core of BidiInfo initialization, factored out into a function that both\n/// the utf-8 and utf-16 versions of BidiInfo can use.\nfn compute_bidi_info_for_para<'a, D: BidiDataSource, T: TextSource<'a> + ?Sized>(\n data_source: &D,\n para: &ParagraphInfo,\n is_pure_ltr: bool,\n has_isolate_controls: bool,\n text: &'a T,\n original_classes: &[BidiClass],\n processing_classes: &mut [BidiClass],\n levels: &mut Vec,\n) {\n let new_len = levels.len() + para.range.len();\n levels.resize(new_len, para.level);\n if para.level == LTR_LEVEL && is_pure_ltr {\n return;\n }\n\n let processing_classes = &mut processing_classes[para.range.clone()];\n let levels = &mut levels[para.range.clone()];\n let mut level_runs = LevelRunVec::new();\n\n explicit::compute(\n text,\n para.level,\n original_classes,\n levels,\n processing_classes,\n &mut level_runs,\n );\n\n let mut sequences = prepare::IsolatingRunSequenceVec::new();\n prepare::isolating_run_sequences(\n para.level,\n original_classes,\n levels,\n level_runs,\n has_isolate_controls,\n &mut sequences,\n );\n for sequence in &sequences {\n implicit::resolve_weak(text, sequence, processing_classes);\n implicit::resolve_neutral(\n text,\n data_source,\n sequence,\n levels,\n original_classes,\n processing_classes,\n );\n }\n\n implicit::resolve_levels(processing_classes, levels);\n\n assign_levels_to_removed_chars(para.level, original_classes, levels);\n}\n\n/// Produce the levels for this paragraph as needed for reordering, one level per *code unit*\n/// in the paragraph. The returned vector includes code units that are not included\n/// in the `line`, but will not adjust them.\n///\n/// This runs [Rule L1]\n///\n/// [Rule L1]: https://www.unicode.org/reports/tr9/#L1\nfn reorder_levels<'a, T: TextSource<'a> + ?Sized>(\n line_classes: &[BidiClass],\n line_levels: &mut [Level],\n line_text: &'a T,\n para_level: Level,\n) {\n // Reset some whitespace chars to paragraph level.\n // \n let mut reset_from: Option = Some(0);\n let mut reset_to: Option = None;\n let mut prev_level = para_level;\n for ((i, c), (_, length)) in line_text.char_indices().zip(line_text.indices_lengths()) {\n match line_classes[i] {\n // Segment separator, Paragraph separator\n B | S => {\n assert_eq!(reset_to, None);\n reset_to = Some(i + T::char_len(c));\n if reset_from.is_none() {\n reset_from = Some(i);\n }\n }\n // Whitespace, isolate formatting\n WS | FSI | LRI | RLI | PDI => {\n if reset_from.is_none() {\n reset_from = Some(i);\n }\n }\n // \n // same as above + set the level\n RLE | LRE | RLO | LRO | PDF | BN => {\n if reset_from.is_none() {\n reset_from = Some(i);\n }\n // also set the level to previous\n for level in &mut line_levels[i..i + length] {\n *level = prev_level;\n }\n }\n _ => {\n reset_from = None;\n }\n }\n if let (Some(from), Some(to)) = (reset_from, reset_to) {\n for level in &mut line_levels[from..to] {\n *level = para_level;\n }\n reset_from = None;\n reset_to = None;\n }\n prev_level = line_levels[i];\n }\n if let Some(from) = reset_from {\n for level in &mut line_levels[from..] {\n *level = para_level;\n }\n }\n}\n\n/// Contains a reference of `BidiInfo` and one of its `paragraphs`.\n/// And it supports all operation in the `Paragraph` that needs also its\n/// `BidiInfo` such as `direction`.\n#[derive(Debug)]\npub struct Paragraph<'a, 'text> {\n pub info: &'a BidiInfo<'text>,\n pub para: &'a ParagraphInfo,\n}\n\nimpl<'a, 'text> Paragraph<'a, 'text> {\n #[inline]\n pub fn new(info: &'a BidiInfo<'text>, para: &'a ParagraphInfo) -> Paragraph<'a, 'text> {\n Paragraph { info, para }\n }\n\n /// Returns if the paragraph is Left direction, right direction or mixed.\n #[inline]\n pub fn direction(&self) -> Direction {\n para_direction(&self.info.levels[self.para.range.clone()])\n }\n\n /// Returns the `Level` of a certain character in the paragraph.\n #[inline]\n pub fn level_at(&self, pos: usize) -> Level {\n let actual_position = self.para.range.start + pos;\n self.info.levels[actual_position]\n }\n}\n\n/// Return the directionality of the paragraph (Left, Right or Mixed) from its levels.\n#[cfg_attr(feature = \"flame_it\", flamer::flame)]\nfn para_direction(levels: &[Level]) -> Direction {\n let mut ltr = false;\n let mut rtl = false;\n for level in levels {\n if level.is_ltr() {\n ltr = true;\n if rtl {\n return Direction::Mixed;\n }\n }\n\n if level.is_rtl() {\n rtl = true;\n if ltr {\n return Direction::Mixed;\n }\n }\n }\n\n if ltr {\n return Direction::Ltr;\n }\n\n Direction::Rtl\n}\n\n/// Assign levels to characters removed by rule X9.\n///\n/// The levels assigned to these characters are not specified by the algorithm. This function\n/// assigns each one the level of the previous character, to avoid breaking level runs.\n#[cfg_attr(feature = \"flame_it\", flamer::flame)]\nfn assign_levels_to_removed_chars(para_level: Level, classes: &[BidiClass], levels: &mut [Level]) {\n for i in 0..levels.len() {\n if prepare::removed_by_x9(classes[i]) {\n levels[i] = if i > 0 { levels[i - 1] } else { para_level };\n }\n }\n}\n\n/// Get the base direction of the text provided according to the Unicode Bidirectional Algorithm.\n///\n/// See rules P2 and P3.\n///\n/// The base direction is derived from the first character in the string with bidi character type\n/// L, R, or AL. If the first such character has type L, Direction::Ltr is returned. If the first\n/// such character has type R or AL, Direction::Rtl is returned.\n///\n/// If the string does not contain any character of these types (outside of embedded isolate runs),\n/// then Direction::Mixed is returned (but should be considered as meaning \"neutral\" or \"unknown\",\n/// not in fact mixed directions).\n///\n/// This is a lightweight function for use when only the base direction is needed and no further\n/// bidi processing of the text is needed.\n///\n/// If the text contains paragraph separators, this function considers only the first paragraph.\n#[cfg(feature = \"hardcoded-data\")]\n#[inline]\npub fn get_base_direction<'a, T: TextSource<'a> + ?Sized>(text: &'a T) -> Direction {\n get_base_direction_with_data_source(&HardcodedBidiData, text)\n}\n\n/// Get the base direction of the text provided according to the Unicode Bidirectional Algorithm,\n/// considering the full text if the first paragraph is all-neutral.\n///\n/// This is the same as get_base_direction except that it does not stop at the first block\n/// separator, but just resets the embedding level and continues to look for a strongly-\n/// directional character. So the result will be the base direction of the first paragraph\n/// that is not purely neutral characters.\n#[cfg(feature = \"hardcoded-data\")]\n#[inline]\npub fn get_base_direction_full<'a, T: TextSource<'a> + ?Sized>(text: &'a T) -> Direction {\n get_base_direction_full_with_data_source(&HardcodedBidiData, text)\n}\n\n#[inline]\npub fn get_base_direction_with_data_source<'a, D: BidiDataSource, T: TextSource<'a> + ?Sized>(\n data_source: &D,\n text: &'a T,\n) -> Direction {\n get_base_direction_impl(data_source, text, false)\n}\n\n#[inline]\npub fn get_base_direction_full_with_data_source<\n 'a,\n D: BidiDataSource,\n T: TextSource<'a> + ?Sized,\n>(\n data_source: &D,\n text: &'a T,\n) -> Direction {\n get_base_direction_impl(data_source, text, true)\n}\n\nfn get_base_direction_impl<'a, D: BidiDataSource, T: TextSource<'a> + ?Sized>(\n data_source: &D,\n text: &'a T,\n use_full_text: bool,\n) -> Direction {\n let mut isolate_level = 0;\n for c in text.chars() {\n match data_source.bidi_class(c) {\n LRI | RLI | FSI => isolate_level += 1,\n PDI if isolate_level > 0 => isolate_level -= 1,\n L if isolate_level == 0 => return Direction::Ltr,\n R | AL if isolate_level == 0 => return Direction::Rtl,\n B if !use_full_text => break,\n B if use_full_text => isolate_level = 0,\n _ => (),\n }\n }\n // If no strong char was found, return Mixed. Normally this will be treated as Ltr by callers\n // (see rule P3), but we don't map this to Ltr here so that a caller that wants to apply other\n // heuristics to an all-neutral paragraph can tell the difference.\n Direction::Mixed\n}\n\n/// Implementation of TextSource for UTF-8 text (a string slice).\nimpl<'text> TextSource<'text> for str {\n type CharIter = core::str::Chars<'text>;\n type CharIndexIter = core::str::CharIndices<'text>;\n type IndexLenIter = Utf8IndexLenIter<'text>;\n\n #[inline]\n fn len(&self) -> usize {\n (self as &str).len()\n }\n #[inline]\n fn char_at(&self, index: usize) -> Option<(char, usize)> {\n if let Some(slice) = self.get(index..) {\n if let Some(ch) = slice.chars().next() {\n return Some((ch, ch.len_utf8()));\n }\n }\n None\n }\n #[inline]\n fn subrange(&self, range: Range) -> &Self {\n &(self as &str)[range]\n }\n #[inline]\n fn chars(&'text self) -> Self::CharIter {\n (self as &str).chars()\n }\n #[inline]\n fn char_indices(&'text self) -> Self::CharIndexIter {\n (self as &str).char_indices()\n }\n #[inline]\n fn indices_lengths(&'text self) -> Self::IndexLenIter {\n Utf8IndexLenIter::new(self)\n }\n #[inline]\n fn char_len(ch: char) -> usize {\n ch.len_utf8()\n }\n}\n\n/// Iterator over (UTF-8) string slices returning (index, char_len) tuple.\n#[derive(Debug)]\npub struct Utf8IndexLenIter<'text> {\n iter: CharIndices<'text>,\n}\n\nimpl<'text> Utf8IndexLenIter<'text> {\n #[inline]\n pub fn new(text: &'text str) -> Self {\n Utf8IndexLenIter {\n iter: text.char_indices(),\n }\n }\n}\n\nimpl Iterator for Utf8IndexLenIter<'_> {\n type Item = (usize, usize);\n\n #[inline]\n fn next(&mut self) -> Option {\n if let Some((pos, ch)) = self.iter.next() {\n return Some((pos, ch.len_utf8()));\n }\n None\n }\n}\n\n#[cfg(test)]\nfn to_utf16(s: &str) -> Vec {\n s.encode_utf16().collect()\n}\n\n#[cfg(test)]\n#[cfg(feature = \"hardcoded-data\")]\nmod tests {\n use super::*;\n\n use utf16::{\n BidiInfo as BidiInfoU16, InitialInfo as InitialInfoU16, Paragraph as ParagraphU16,\n ParagraphBidiInfo as ParagraphBidiInfoU16,\n };\n\n #[test]\n fn test_utf16_text_source() {\n let text: &[u16] =\n &[0x41, 0xD801, 0xDC01, 0x20, 0xD800, 0x20, 0xDFFF, 0x20, 0xDC00, 0xD800];\n assert_eq!(text.char_at(0), Some(('A', 1)));\n assert_eq!(text.char_at(1), Some(('\\u{10401}', 2)));\n assert_eq!(text.char_at(2), None);\n assert_eq!(text.char_at(3), Some((' ', 1)));\n assert_eq!(text.char_at(4), Some((char::REPLACEMENT_CHARACTER, 1)));\n assert_eq!(text.char_at(5), Some((' ', 1)));\n assert_eq!(text.char_at(6), Some((char::REPLACEMENT_CHARACTER, 1)));\n assert_eq!(text.char_at(7), Some((' ', 1)));\n assert_eq!(text.char_at(8), Some((char::REPLACEMENT_CHARACTER, 1)));\n assert_eq!(text.char_at(9), Some((char::REPLACEMENT_CHARACTER, 1)));\n assert_eq!(text.char_at(10), None);\n }\n\n #[test]\n fn test_utf16_char_iter() {\n let text: &[u16] =\n &[0x41, 0xD801, 0xDC01, 0x20, 0xD800, 0x20, 0xDFFF, 0x20, 0xDC00, 0xD800];\n assert_eq!(text.len(), 10);\n assert_eq!(text.chars().count(), 9);\n let mut chars = text.chars();\n assert_eq!(chars.next(), Some('A'));\n assert_eq!(chars.next(), Some('\\u{10401}'));\n assert_eq!(chars.next(), Some(' '));\n assert_eq!(chars.next(), Some('\\u{FFFD}'));\n assert_eq!(chars.next(), Some(' '));\n assert_eq!(chars.next(), Some('\\u{FFFD}'));\n assert_eq!(chars.next(), Some(' '));\n assert_eq!(chars.next(), Some('\\u{FFFD}'));\n assert_eq!(chars.next(), Some('\\u{FFFD}'));\n assert_eq!(chars.next(), None);\n }\n\n #[test]\n fn test_initial_text_info() {\n let tests = vec![\n (\n // text\n \"a1\",\n // expected bidi classes per utf-8 byte\n vec![L, EN],\n // expected paragraph-info for utf-8\n vec![ParagraphInfo {\n range: 0..2,\n level: LTR_LEVEL,\n }],\n // expected bidi classes per utf-16 code unit\n vec![L, EN],\n // expected paragraph-info for utf-16\n vec![ParagraphInfo {\n range: 0..2,\n level: LTR_LEVEL,\n }],\n ),\n (\n // Arabic, space, Hebrew\n \"\\u{0639} \\u{05D0}\",\n vec![AL, AL, WS, R, R],\n vec![ParagraphInfo {\n range: 0..5,\n level: RTL_LEVEL,\n }],\n vec![AL, WS, R],\n vec![ParagraphInfo {\n range: 0..3,\n level: RTL_LEVEL,\n }],\n ),\n (\n // SMP characters from Kharoshthi, Cuneiform, Adlam:\n \"\\u{10A00}\\u{12000}\\u{1E900}\",\n vec![R, R, R, R, L, L, L, L, R, R, R, R],\n vec![ParagraphInfo {\n range: 0..12,\n level: RTL_LEVEL,\n }],\n vec![R, R, L, L, R, R],\n vec![ParagraphInfo {\n range: 0..6,\n level: RTL_LEVEL,\n }],\n ),\n (\n \"a\\u{2029}b\",\n vec![L, B, B, B, L],\n vec![\n ParagraphInfo {\n range: 0..4,\n level: LTR_LEVEL,\n },\n ParagraphInfo {\n range: 4..5,\n level: LTR_LEVEL,\n },\n ],\n vec![L, B, L],\n vec![\n ParagraphInfo {\n range: 0..2,\n level: LTR_LEVEL,\n },\n ParagraphInfo {\n range: 2..3,\n level: LTR_LEVEL,\n },\n ],\n ),\n (\n \"\\u{2068}א\\u{2069}a\", // U+2068 FSI, U+2069 PDI\n vec![RLI, RLI, RLI, R, R, PDI, PDI, PDI, L],\n vec![ParagraphInfo {\n range: 0..9,\n level: LTR_LEVEL,\n }],\n vec![RLI, R, PDI, L],\n vec![ParagraphInfo {\n range: 0..4,\n level: LTR_LEVEL,\n }],\n ),\n ];\n\n for t in tests {\n assert_eq!(\n InitialInfo::new(t.0, None),\n InitialInfo {\n text: t.0,\n original_classes: t.1,\n paragraphs: t.2,\n }\n );\n let text = &to_utf16(t.0);\n assert_eq!(\n InitialInfoU16::new(text, None),\n InitialInfoU16 {\n text,\n original_classes: t.3,\n paragraphs: t.4,\n }\n );\n }\n }\n\n #[test]\n #[cfg(feature = \"hardcoded-data\")]\n fn test_process_text() {\n let tests = vec![\n (\n // text\n \"\",\n // base level\n Some(RTL_LEVEL),\n // levels\n Level::vec(&[]),\n // original_classes\n vec![],\n // paragraphs\n vec![],\n // levels_u16\n Level::vec(&[]),\n // original_classes_u16\n vec![],\n // paragraphs_u16\n vec![],\n ),\n (\n // text\n \"abc123\",\n // base level\n Some(LTR_LEVEL),\n // levels\n Level::vec(&[0, 0, 0, 0, 0, 0]),\n // original_classes\n vec![L, L, L, EN, EN, EN],\n // paragraphs\n vec![ParagraphInfo {\n range: 0..6,\n level: LTR_LEVEL,\n }],\n // levels_u16\n Level::vec(&[0, 0, 0, 0, 0, 0]),\n // original_classes_u16\n vec![L, L, L, EN, EN, EN],\n // paragraphs_u16\n vec![ParagraphInfo {\n range: 0..6,\n level: LTR_LEVEL,\n }],\n ),\n (\n \"abc \\u{05D0}\\u{05D1}\\u{05D2}\",\n Some(LTR_LEVEL),\n Level::vec(&[0, 0, 0, 0, 1, 1, 1, 1, 1, 1]),\n vec![L, L, L, WS, R, R, R, R, R, R],\n vec![ParagraphInfo {\n range: 0..10,\n level: LTR_LEVEL,\n }],\n Level::vec(&[0, 0, 0, 0, 1, 1, 1]),\n vec![L, L, L, WS, R, R, R],\n vec![ParagraphInfo {\n range: 0..7,\n level: LTR_LEVEL,\n }],\n ),\n (\n \"abc \\u{05D0}\\u{05D1}\\u{05D2}\",\n Some(RTL_LEVEL),\n Level::vec(&[2, 2, 2, 1, 1, 1, 1, 1, 1, 1]),\n vec![L, L, L, WS, R, R, R, R, R, R],\n vec![ParagraphInfo {\n range: 0..10,\n level: RTL_LEVEL,\n }],\n Level::vec(&[2, 2, 2, 1, 1, 1, 1]),\n vec![L, L, L, WS, R, R, R],\n vec![ParagraphInfo {\n range: 0..7,\n level: RTL_LEVEL,\n }],\n ),\n (\n \"\\u{05D0}\\u{05D1}\\u{05D2} abc\",\n Some(LTR_LEVEL),\n Level::vec(&[1, 1, 1, 1, 1, 1, 0, 0, 0, 0]),\n vec![R, R, R, R, R, R, WS, L, L, L],\n vec![ParagraphInfo {\n range: 0..10,\n level: LTR_LEVEL,\n }],\n Level::vec(&[1, 1, 1, 0, 0, 0, 0]),\n vec![R, R, R, WS, L, L, L],\n vec![ParagraphInfo {\n range: 0..7,\n level: LTR_LEVEL,\n }],\n ),\n (\n \"\\u{05D0}\\u{05D1}\\u{05D2} abc\",\n None,\n Level::vec(&[1, 1, 1, 1, 1, 1, 1, 2, 2, 2]),\n vec![R, R, R, R, R, R, WS, L, L, L],\n vec![ParagraphInfo {\n range: 0..10,\n level: RTL_LEVEL,\n }],\n Level::vec(&[1, 1, 1, 1, 2, 2, 2]),\n vec![R, R, R, WS, L, L, L],\n vec![ParagraphInfo {\n range: 0..7,\n level: RTL_LEVEL,\n }],\n ),\n (\n \"\\u{063A}2\\u{0638} \\u{05D0}2\\u{05D2}\",\n Some(LTR_LEVEL),\n Level::vec(&[1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1]),\n vec![AL, AL, EN, AL, AL, WS, R, R, EN, R, R],\n vec![ParagraphInfo {\n range: 0..11,\n level: LTR_LEVEL,\n }],\n Level::vec(&[1, 2, 1, 1, 1, 2, 1]),\n vec![AL, EN, AL, WS, R, EN, R],\n vec![ParagraphInfo {\n range: 0..7,\n level: LTR_LEVEL,\n }],\n ),\n (\n \"a א.\\nג\",\n None,\n Level::vec(&[0, 0, 1, 1, 0, 0, 1, 1]),\n vec![L, WS, R, R, CS, B, R, R],\n vec![\n ParagraphInfo {\n range: 0..6,\n level: LTR_LEVEL,\n },\n ParagraphInfo {\n range: 6..8,\n level: RTL_LEVEL,\n },\n ],\n Level::vec(&[0, 0, 1, 0, 0, 1]),\n vec![L, WS, R, CS, B, R],\n vec![\n ParagraphInfo {\n range: 0..5,\n level: LTR_LEVEL,\n },\n ParagraphInfo {\n range: 5..6,\n level: RTL_LEVEL,\n },\n ],\n ),\n // BidiTest:69635 (AL ET EN)\n (\n \"\\u{060B}\\u{20CF}\\u{06F9}\",\n None,\n Level::vec(&[1, 1, 1, 1, 1, 2, 2]),\n vec![AL, AL, ET, ET, ET, EN, EN],\n vec![ParagraphInfo {\n range: 0..7,\n level: RTL_LEVEL,\n }],\n Level::vec(&[1, 1, 2]),\n vec![AL, ET, EN],\n vec![ParagraphInfo {\n range: 0..3,\n level: RTL_LEVEL,\n }],\n ),\n ];\n\n for t in tests {\n assert_eq!(\n BidiInfo::new(t.0, t.1),\n BidiInfo {\n text: t.0,\n levels: t.2.clone(),\n original_classes: t.3.clone(),\n paragraphs: t.4.clone(),\n }\n );\n // If it was empty, also test that ParagraphBidiInfo handles it safely.\n if t.4.len() == 0 {\n assert_eq!(\n ParagraphBidiInfo::new(t.0, t.1),\n ParagraphBidiInfo {\n text: t.0,\n original_classes: t.3.clone(),\n levels: t.2.clone(),\n paragraph_level: RTL_LEVEL,\n is_pure_ltr: true,\n }\n )\n }\n // If it was a single paragraph, also test ParagraphBidiInfo.\n if t.4.len() == 1 {\n assert_eq!(\n ParagraphBidiInfo::new(t.0, t.1),\n ParagraphBidiInfo {\n text: t.0,\n original_classes: t.3,\n levels: t.2.clone(),\n paragraph_level: t.4[0].level,\n is_pure_ltr: !level::has_rtl(&t.2),\n }\n )\n }\n let text = &to_utf16(t.0);\n assert_eq!(\n BidiInfoU16::new(text, t.1),\n BidiInfoU16 {\n text,\n levels: t.5.clone(),\n original_classes: t.6.clone(),\n paragraphs: t.7.clone(),\n }\n );\n if t.7.len() == 1 {\n assert_eq!(\n ParagraphBidiInfoU16::new(text, t.1),\n ParagraphBidiInfoU16 {\n text: text,\n original_classes: t.6.clone(),\n levels: t.5.clone(),\n paragraph_level: t.7[0].level,\n is_pure_ltr: !level::has_rtl(&t.5),\n }\n )\n }\n }\n }\n\n #[test]\n #[cfg(feature = \"hardcoded-data\")]\n fn test_paragraph_bidi_info() {\n // Passing text that includes a paragraph break to the ParagraphBidiInfo API:\n // this is a misuse of the API by the client, but our behavior is safe &\n // consistent. The embedded paragraph break acts like a separator (tab) would.\n let tests = vec![\n (\n \"a א.\\nג\",\n None,\n // utf-8 results:\n vec![L, WS, R, R, CS, B, R, R],\n Level::vec(&[0, 0, 1, 1, 1, 1, 1, 1]),\n // utf-16 results:\n vec![L, WS, R, CS, B, R],\n Level::vec(&[0, 0, 1, 1, 1, 1]),\n // paragraph level; is_pure_ltr\n LTR_LEVEL,\n false,\n ),\n (\n \"\\u{5d1} a.\\nb.\",\n None,\n // utf-8 results:\n vec![R, R, WS, L, CS, B, L, CS],\n Level::vec(&[1, 1, 1, 2, 2, 2, 2, 1]),\n // utf-16 results:\n vec![R, WS, L, CS, B, L, CS],\n Level::vec(&[1, 1, 2, 2, 2, 2, 1]),\n // paragraph level; is_pure_ltr\n RTL_LEVEL,\n false,\n ),\n (\n \"a א.\\tג\",\n None,\n // utf-8 results:\n vec![L, WS, R, R, CS, S, R, R],\n Level::vec(&[0, 0, 1, 1, 1, 1, 1, 1]),\n // utf-16 results:\n vec![L, WS, R, CS, S, R],\n Level::vec(&[0, 0, 1, 1, 1, 1]),\n // paragraph level; is_pure_ltr\n LTR_LEVEL,\n false,\n ),\n (\n \"\\u{5d1} a.\\tb.\",\n None,\n // utf-8 results:\n vec![R, R, WS, L, CS, S, L, CS],\n Level::vec(&[1, 1, 1, 2, 2, 2, 2, 1]),\n // utf-16 results:\n vec![R, WS, L, CS, S, L, CS],\n Level::vec(&[1, 1, 2, 2, 2, 2, 1]),\n // paragraph level; is_pure_ltr\n RTL_LEVEL,\n false,\n ),\n ];\n\n for t in tests {\n assert_eq!(\n ParagraphBidiInfo::new(t.0, t.1),\n ParagraphBidiInfo {\n text: t.0,\n original_classes: t.2,\n levels: t.3,\n paragraph_level: t.6,\n is_pure_ltr: t.7,\n }\n );\n let text = &to_utf16(t.0);\n assert_eq!(\n ParagraphBidiInfoU16::new(text, t.1),\n ParagraphBidiInfoU16 {\n text: text,\n original_classes: t.4,\n levels: t.5,\n paragraph_level: t.6,\n is_pure_ltr: t.7,\n }\n );\n }\n }\n\n #[test]\n #[cfg(feature = \"hardcoded-data\")]\n fn test_bidi_info_has_rtl() {\n let tests = vec![\n // ASCII only\n (\"123\", None, false),\n (\"123\", Some(LTR_LEVEL), false),\n (\"123\", Some(RTL_LEVEL), false),\n (\"abc\", None, false),\n (\"abc\", Some(LTR_LEVEL), false),\n (\"abc\", Some(RTL_LEVEL), false),\n (\"abc 123\", None, false),\n (\"abc\\n123\", None, false),\n // With Hebrew\n (\"\\u{05D0}\\u{05D1}\\u{05BC}\\u{05D2}\", None, true),\n (\"\\u{05D0}\\u{05D1}\\u{05BC}\\u{05D2}\", Some(LTR_LEVEL), true),\n (\"\\u{05D0}\\u{05D1}\\u{05BC}\\u{05D2}\", Some(RTL_LEVEL), true),\n (\"abc \\u{05D0}\\u{05D1}\\u{05BC}\\u{05D2}\", None, true),\n (\"abc\\n\\u{05D0}\\u{05D1}\\u{05BC}\\u{05D2}\", None, true),\n (\"\\u{05D0}\\u{05D1}\\u{05BC}\\u{05D2} abc\", None, true),\n (\"\\u{05D0}\\u{05D1}\\u{05BC}\\u{05D2}\\nabc\", None, true),\n (\"\\u{05D0}\\u{05D1}\\u{05BC}\\u{05D2} 123\", None, true),\n (\"\\u{05D0}\\u{05D1}\\u{05BC}\\u{05D2}\\n123\", None, true),\n ];\n\n for t in tests {\n assert_eq!(BidiInfo::new(t.0, t.1).has_rtl(), t.2);\n assert_eq!(BidiInfoU16::new(&to_utf16(t.0), t.1).has_rtl(), t.2);\n }\n }\n\n #[cfg(feature = \"hardcoded-data\")]\n fn reorder_paras(text: &str) -> Vec> {\n let bidi_info = BidiInfo::new(text, None);\n bidi_info\n .paragraphs\n .iter()\n .map(|para| bidi_info.reorder_line(para, para.range.clone()))\n .collect()\n }\n\n #[cfg(feature = \"hardcoded-data\")]\n fn reorder_paras_u16(text: &[u16]) -> Vec> {\n let bidi_info = BidiInfoU16::new(text, None);\n bidi_info\n .paragraphs\n .iter()\n .map(|para| bidi_info.reorder_line(para, para.range.clone()))\n .collect()\n }\n\n #[test]\n #[cfg(feature = \"hardcoded-data\")]\n fn test_reorder_line() {\n let tests = vec![\n // Bidi_Class: L L L B L L L B L L L\n (\"abc\\ndef\\nghi\", vec![\"abc\\n\", \"def\\n\", \"ghi\"]),\n // Bidi_Class: L L EN B L L EN B L L EN\n (\"ab1\\nde2\\ngh3\", vec![\"ab1\\n\", \"de2\\n\", \"gh3\"]),\n // Bidi_Class: L L L B AL AL AL\n (\"abc\\nابج\", vec![\"abc\\n\", \"جبا\"]),\n // Bidi_Class: AL AL AL B L L L\n (\n \"\\u{0627}\\u{0628}\\u{062C}\\nabc\",\n vec![\"\\n\\u{062C}\\u{0628}\\u{0627}\", \"abc\"],\n ),\n (\"1.-2\", vec![\"1.-2\"]),\n (\"1-.2\", vec![\"1-.2\"]),\n (\"abc אבג\", vec![\"abc גבא\"]),\n // Numbers being weak LTR characters, cannot reorder strong RTL\n (\"123 \\u{05D0}\\u{05D1}\\u{05D2}\", vec![\"גבא 123\"]),\n (\"abc\\u{202A}def\", vec![\"abc\\u{202A}def\"]),\n (\n \"abc\\u{202A}def\\u{202C}ghi\",\n vec![\"abc\\u{202A}def\\u{202C}ghi\"],\n ),\n (\n \"abc\\u{2066}def\\u{2069}ghi\",\n vec![\"abc\\u{2066}def\\u{2069}ghi\"],\n ),\n // Testing for RLE Character\n (\"\\u{202B}abc אבג\\u{202C}\", vec![\"\\u{202b}גבא abc\\u{202c}\"]),\n // Testing neutral characters\n (\"\\u{05D0}בג? אבג\", vec![\"גבא ?גבא\"]),\n // Testing neutral characters with special case\n (\"A אבג?\", vec![\"A גבא?\"]),\n // Testing neutral characters with Implicit RTL Marker\n (\"A אבג?\\u{200F}\", vec![\"A \\u{200F}?גבא\"]),\n (\"\\u{05D0}בג abc\", vec![\"abc גבא\"]),\n (\"abc\\u{2067}.-\\u{2069}ghi\", vec![\"abc\\u{2067}-.\\u{2069}ghi\"]),\n (\n \"Hello, \\u{2068}\\u{202E}world\\u{202C}\\u{2069}!\",\n vec![\"Hello, \\u{2068}\\u{202E}\\u{202C}dlrow\\u{2069}!\"],\n ),\n // With mirrorable characters in RTL run\n (\"\\u{05D0}(ב)ג.\", vec![\".ג)ב(א\"]),\n // With mirrorable characters on level boundary\n (\"\\u{05D0}ב(גד[&ef].)gh\", vec![\"gh).]ef&[דג(בא\"]),\n ];\n\n for t in tests {\n assert_eq!(reorder_paras(t.0), t.1);\n let expect_utf16 = t.1.iter().map(|v| to_utf16(v)).collect::>();\n assert_eq!(reorder_paras_u16(&to_utf16(t.0)), expect_utf16);\n }\n }\n\n fn reordered_levels_for_paras(text: &str) -> Vec> {\n let bidi_info = BidiInfo::new(text, None);\n bidi_info\n .paragraphs\n .iter()\n .map(|para| bidi_info.reordered_levels(para, para.range.clone()))\n .collect()\n }\n\n fn reordered_levels_per_char_for_paras(text: &str) -> Vec> {\n let bidi_info = BidiInfo::new(text, None);\n bidi_info\n .paragraphs\n .iter()\n .map(|para| bidi_info.reordered_levels_per_char(para, para.range.clone()))\n .collect()\n }\n\n fn reordered_levels_for_paras_u16(text: &[u16]) -> Vec> {\n let bidi_info = BidiInfoU16::new(text, None);\n bidi_info\n .paragraphs\n .iter()\n .map(|para| bidi_info.reordered_levels(para, para.range.clone()))\n .collect()\n }\n\n fn reordered_levels_per_char_for_paras_u16(text: &[u16]) -> Vec> {\n let bidi_info = BidiInfoU16::new(text, None);\n bidi_info\n .paragraphs\n .iter()\n .map(|para| bidi_info.reordered_levels_per_char(para, para.range.clone()))\n .collect()\n }\n\n #[test]\n #[cfg(feature = \"hardcoded-data\")]\n // See issue #138\n fn test_reordered_levels_range() {\n // |---------------|\n let s = \"\\u{202a}A\\u{202c}\\u{202a}A\\u{202c}\";\n let range = 4..11;\n assert!(s.get(range.clone()).is_some());\n\n let bidi = BidiInfo::new(s, None);\n let (_, runs) = bidi.visual_runs(&bidi.paragraphs[0], range);\n\n for run in runs {\n let _ = &s[run]; // should be valid slice of s\n }\n }\n\n #[test]\n #[cfg(feature = \"hardcoded-data\")]\n fn test_reordered_levels() {\n let tests = vec![\n // BidiTest:946 (LRI PDI)\n (\n \"\\u{2067}\\u{2069}\",\n vec![Level::vec(&[0, 0, 0, 0, 0, 0])],\n vec![Level::vec(&[0, 0])],\n vec![Level::vec(&[0, 0])],\n ),\n // BidiTest:69635 (AL ET EN)\n (\n \"\\u{060B}\\u{20CF}\\u{06F9}\",\n vec![Level::vec(&[1, 1, 1, 1, 1, 2, 2])],\n vec![Level::vec(&[1, 1, 2])],\n vec![Level::vec(&[1, 1, 2])],\n ),\n ];\n\n for t in tests {\n assert_eq!(reordered_levels_for_paras(t.0), t.1);\n assert_eq!(reordered_levels_per_char_for_paras(t.0), t.2);\n let text = &to_utf16(t.0);\n assert_eq!(reordered_levels_for_paras_u16(text), t.3);\n assert_eq!(reordered_levels_per_char_for_paras_u16(text), t.2);\n }\n\n let tests = vec![\n // BidiTest:291284 (AN RLI PDF R)\n (\n \"\\u{0605}\\u{2067}\\u{202C}\\u{0590}\",\n vec![&[\"2\", \"2\", \"0\", \"0\", \"0\", \"x\", \"x\", \"x\", \"1\", \"1\"]],\n vec![&[\"2\", \"0\", \"x\", \"1\"]],\n vec![&[\"2\", \"0\", \"x\", \"1\"]],\n ),\n ];\n\n for t in tests {\n assert_eq!(reordered_levels_for_paras(t.0), t.1);\n assert_eq!(reordered_levels_per_char_for_paras(t.0), t.2);\n let text = &to_utf16(t.0);\n assert_eq!(reordered_levels_for_paras_u16(text), t.3);\n assert_eq!(reordered_levels_per_char_for_paras_u16(text), t.2);\n }\n\n let text = \"aa טֶ\";\n let bidi_info = BidiInfo::new(text, None);\n assert_eq!(\n bidi_info.reordered_levels(&bidi_info.paragraphs[0], 3..7),\n Level::vec(&[0, 0, 0, 1, 1, 1, 1]),\n );\n\n let text = &to_utf16(text);\n let bidi_info = BidiInfoU16::new(text, None);\n assert_eq!(\n bidi_info.reordered_levels(&bidi_info.paragraphs[0], 1..4),\n Level::vec(&[0, 0, 0, 1, 1]),\n );\n }\n\n #[test]\n fn test_paragraph_info_len() {\n let text = \"hello world\";\n let bidi_info = BidiInfo::new(text, None);\n assert_eq!(bidi_info.paragraphs.len(), 1);\n assert_eq!(bidi_info.paragraphs[0].len(), text.len());\n\n let text2 = \"How are you\";\n let whole_text = format!(\"{}\\n{}\", text, text2);\n let bidi_info = BidiInfo::new(&whole_text, None);\n assert_eq!(bidi_info.paragraphs.len(), 2);\n\n // The first paragraph include the paragraph separator.\n // TODO: investigate if the paragraph separator character\n // should not be part of any paragraph.\n assert_eq!(bidi_info.paragraphs[0].len(), text.len() + 1);\n assert_eq!(bidi_info.paragraphs[1].len(), text2.len());\n\n let text = &to_utf16(text);\n let bidi_info = BidiInfoU16::new(text, None);\n assert_eq!(bidi_info.paragraphs.len(), 1);\n assert_eq!(bidi_info.paragraphs[0].len(), text.len());\n\n let text2 = &to_utf16(text2);\n let whole_text = &to_utf16(&whole_text);\n let bidi_info = BidiInfoU16::new(&whole_text, None);\n assert_eq!(bidi_info.paragraphs.len(), 2);\n\n assert_eq!(bidi_info.paragraphs[0].len(), text.len() + 1);\n assert_eq!(bidi_info.paragraphs[1].len(), text2.len());\n }\n\n #[test]\n fn test_direction() {\n let ltr_text = \"hello world\";\n let rtl_text = \"أهلا بكم\";\n let all_paragraphs = format!(\"{}\\n{}\\n{}{}\", ltr_text, rtl_text, ltr_text, rtl_text);\n let bidi_info = BidiInfo::new(&all_paragraphs, None);\n assert_eq!(bidi_info.paragraphs.len(), 3);\n let p_ltr = Paragraph::new(&bidi_info, &bidi_info.paragraphs[0]);\n let p_rtl = Paragraph::new(&bidi_info, &bidi_info.paragraphs[1]);\n let p_mixed = Paragraph::new(&bidi_info, &bidi_info.paragraphs[2]);\n assert_eq!(p_ltr.direction(), Direction::Ltr);\n assert_eq!(p_rtl.direction(), Direction::Rtl);\n assert_eq!(p_mixed.direction(), Direction::Mixed);\n\n let all_paragraphs = &to_utf16(&all_paragraphs);\n let bidi_info = BidiInfoU16::new(&all_paragraphs, None);\n assert_eq!(bidi_info.paragraphs.len(), 3);\n let p_ltr = ParagraphU16::new(&bidi_info, &bidi_info.paragraphs[0]);\n let p_rtl = ParagraphU16::new(&bidi_info, &bidi_info.paragraphs[1]);\n let p_mixed = ParagraphU16::new(&bidi_info, &bidi_info.paragraphs[2]);\n assert_eq!(p_ltr.direction(), Direction::Ltr);\n assert_eq!(p_rtl.direction(), Direction::Rtl);\n assert_eq!(p_mixed.direction(), Direction::Mixed);\n }\n\n #[test]\n fn test_edge_cases_direction() {\n // No paragraphs for empty text.\n let empty = \"\";\n let bidi_info = BidiInfo::new(empty, Option::from(RTL_LEVEL));\n assert_eq!(bidi_info.paragraphs.len(), 0);\n\n let empty = &to_utf16(empty);\n let bidi_info = BidiInfoU16::new(empty, Option::from(RTL_LEVEL));\n assert_eq!(bidi_info.paragraphs.len(), 0);\n\n let tests = vec![\n // The paragraph separator will take the value of the default direction\n // which is left to right.\n (\"\\n\", None, Direction::Ltr),\n // The paragraph separator will take the value of the given initial direction\n // which is left to right.\n (\"\\n\", Option::from(LTR_LEVEL), Direction::Ltr),\n // The paragraph separator will take the value of the given initial direction\n // which is right to left.\n (\"\\n\", Option::from(RTL_LEVEL), Direction::Rtl),\n ];\n\n for t in tests {\n let bidi_info = BidiInfo::new(t.0, t.1);\n assert_eq!(bidi_info.paragraphs.len(), 1);\n let p = Paragraph::new(&bidi_info, &bidi_info.paragraphs[0]);\n assert_eq!(p.direction(), t.2);\n let text = &to_utf16(t.0);\n let bidi_info = BidiInfoU16::new(text, t.1);\n let p = ParagraphU16::new(&bidi_info, &bidi_info.paragraphs[0]);\n assert_eq!(p.direction(), t.2);\n }\n }\n\n #[test]\n fn test_level_at() {\n let ltr_text = \"hello world\";\n let rtl_text = \"أهلا بكم\";\n let all_paragraphs = format!(\"{}\\n{}\\n{}{}\", ltr_text, rtl_text, ltr_text, rtl_text);\n let bidi_info = BidiInfo::new(&all_paragraphs, None);\n assert_eq!(bidi_info.paragraphs.len(), 3);\n\n let p_ltr = Paragraph::new(&bidi_info, &bidi_info.paragraphs[0]);\n let p_rtl = Paragraph::new(&bidi_info, &bidi_info.paragraphs[1]);\n let p_mixed = Paragraph::new(&bidi_info, &bidi_info.paragraphs[2]);\n\n assert_eq!(p_ltr.level_at(0), LTR_LEVEL);\n assert_eq!(p_rtl.level_at(0), RTL_LEVEL);\n assert_eq!(p_mixed.level_at(0), LTR_LEVEL);\n assert_eq!(p_mixed.info.levels.len(), 54);\n assert_eq!(p_mixed.para.range.start, 28);\n assert_eq!(p_mixed.level_at(ltr_text.len()), RTL_LEVEL);\n\n let all_paragraphs = &to_utf16(&all_paragraphs);\n let bidi_info = BidiInfoU16::new(&all_paragraphs, None);\n assert_eq!(bidi_info.paragraphs.len(), 3);\n\n let p_ltr = ParagraphU16::new(&bidi_info, &bidi_info.paragraphs[0]);\n let p_rtl = ParagraphU16::new(&bidi_info, &bidi_info.paragraphs[1]);\n let p_mixed = ParagraphU16::new(&bidi_info, &bidi_info.paragraphs[2]);\n\n assert_eq!(p_ltr.level_at(0), LTR_LEVEL);\n assert_eq!(p_rtl.level_at(0), RTL_LEVEL);\n assert_eq!(p_mixed.level_at(0), LTR_LEVEL);\n assert_eq!(p_mixed.info.levels.len(), 40);\n assert_eq!(p_mixed.para.range.start, 21);\n assert_eq!(p_mixed.level_at(ltr_text.len()), RTL_LEVEL);\n }\n\n #[test]\n fn test_get_base_direction() {\n let tests = vec![\n (\"\", Direction::Mixed), // return Mixed if no strong character found\n (\"123[]-+\\u{2019}\\u{2060}\\u{00bf}?\", Direction::Mixed),\n (\"3.14\\npi\", Direction::Mixed), // only first paragraph is considered\n (\"[123 'abc']\", Direction::Ltr),\n (\"[123 '\\u{0628}' abc\", Direction::Rtl),\n (\"[123 '\\u{2066}abc\\u{2069}'\\u{0628}]\", Direction::Rtl), // embedded isolate is ignored\n (\"[123 '\\u{2066}abc\\u{2068}'\\u{0628}]\", Direction::Mixed),\n ];\n\n for t in tests {\n assert_eq!(get_base_direction(t.0), t.1);\n let text = &to_utf16(t.0);\n assert_eq!(get_base_direction(text.as_slice()), t.1);\n }\n }\n\n #[test]\n fn test_get_base_direction_full() {\n let tests = vec![\n (\"\", Direction::Mixed), // return Mixed if no strong character found\n (\"123[]-+\\u{2019}\\u{2060}\\u{00bf}?\", Direction::Mixed),\n (\"3.14\\npi\", Direction::Ltr), // direction taken from the second paragraph\n (\"3.14\\n\\u{05D0}\", Direction::Rtl), // direction taken from the second paragraph\n (\"[123 'abc']\", Direction::Ltr),\n (\"[123 '\\u{0628}' abc\", Direction::Rtl),\n (\"[123 '\\u{2066}abc\\u{2069}'\\u{0628}]\", Direction::Rtl), // embedded isolate is ignored\n (\"[123 '\\u{2066}abc\\u{2068}'\\u{0628}]\", Direction::Mixed),\n (\"[123 '\\u{2066}abc\\u{2068}'\\n\\u{0628}]\", Direction::Rtl), // \\n resets embedding level\n ];\n\n for t in tests {\n assert_eq!(get_base_direction_full(t.0), t.1);\n let text = &to_utf16(t.0);\n assert_eq!(get_base_direction_full(text.as_slice()), t.1);\n }\n }\n}\n\n#[cfg(all(feature = \"serde\", feature = \"hardcoded-data\", test))]\nmod serde_tests {\n use super::*;\n use serde_test::{assert_tokens, Token};\n\n #[test]\n fn test_levels() {\n let text = \"abc אבג\";\n let bidi_info = BidiInfo::new(text, None);\n let levels = bidi_info.levels;\n assert_eq!(text.as_bytes().len(), 10);\n assert_eq!(levels.len(), 10);\n assert_tokens(\n &levels,\n &[\n Token::Seq { len: Some(10) },\n Token::NewtypeStruct { name: \"Level\" },\n Token::U8(0),\n Token::NewtypeStruct { name: \"Level\" },\n Token::U8(0),\n Token::NewtypeStruct { name: \"Level\" },\n Token::U8(0),\n Token::NewtypeStruct { name: \"Level\" },\n Token::U8(0),\n Token::NewtypeStruct { name: \"Level\" },\n Token::U8(1),\n Token::NewtypeStruct { name: \"Level\" },\n Token::U8(1),\n Token::NewtypeStruct { name: \"Level\" },\n Token::U8(1),\n Token::NewtypeStruct { name: \"Level\" },\n Token::U8(1),\n Token::NewtypeStruct { name: \"Level\" },\n Token::U8(1),\n Token::NewtypeStruct { name: \"Level\" },\n Token::U8(1),\n Token::SeqEnd,\n ],\n );\n }\n}\n" }, { "package": "unicode-bidi@0.3.18", "sourcePath": "src/prepare.rs", "sha256": "aeb8b88cfb2d2e6b74473f5903205dd3683d57abcc8801de7b9fdea6a432a0fe", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2015 The Servo Project Developers. See the\n// COPYRIGHT file at the top-level directory of this distribution.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n//! 3.3.3 Preparations for Implicit Processing\n//!\n//! \n\nuse alloc::vec::Vec;\nuse core::cmp::max;\nuse core::ops::Range;\n#[cfg(feature = \"smallvec\")]\nuse smallvec::{smallvec, SmallVec};\n\nuse super::level::Level;\nuse super::BidiClass::{self, *};\n\n/// A maximal substring of characters with the same embedding level.\n///\n/// Represented as a range of byte indices.\npub type LevelRun = Range;\n\n#[cfg(feature = \"smallvec\")]\npub type LevelRunVec = SmallVec<[LevelRun; 8]>;\n#[cfg(not(feature = \"smallvec\"))]\npub type LevelRunVec = Vec;\n\n/// Output of `isolating_run_sequences` (steps X9-X10)\n#[derive(Debug, PartialEq)]\npub struct IsolatingRunSequence {\n pub runs: Vec,\n pub sos: BidiClass, // Start-of-sequence type.\n pub eos: BidiClass, // End-of-sequence type.\n}\n\n#[cfg(feature = \"smallvec\")]\npub type IsolatingRunSequenceVec = SmallVec<[IsolatingRunSequence; 8]>;\n#[cfg(not(feature = \"smallvec\"))]\npub type IsolatingRunSequenceVec = Vec;\n\n/// Compute the set of isolating run sequences.\n///\n/// An isolating run sequence is a maximal sequence of level runs such that for all level runs\n/// except the last one in the sequence, the last character of the run is an isolate initiator\n/// whose matching PDI is the first character of the next level run in the sequence.\n///\n/// Note: This function does *not* return the sequences in order by their first characters.\n#[cfg_attr(feature = \"flame_it\", flamer::flame)]\npub fn isolating_run_sequences(\n para_level: Level,\n original_classes: &[BidiClass],\n levels: &[Level],\n runs: LevelRunVec,\n has_isolate_controls: bool,\n isolating_run_sequences: &mut IsolatingRunSequenceVec,\n) {\n // Per http://www.unicode.org/reports/tr9/#BD13:\n // \"In the absence of isolate initiators, each isolating run sequence in a paragraph\n // consists of exactly one level run, and each level run constitutes a separate\n // isolating run sequence.\"\n // We can take a simplified path to handle this case.\n if !has_isolate_controls {\n isolating_run_sequences.reserve_exact(runs.len());\n for run in runs {\n // Determine the `sos` and `eos` class for the sequence.\n // \n\n let run_levels = &levels[run.clone()];\n let run_classes = &original_classes[run.clone()];\n let seq_level = run_levels[run_classes\n .iter()\n .position(|c| not_removed_by_x9(c))\n .unwrap_or(0)];\n\n let end_level = run_levels[run_classes\n .iter()\n .rposition(|c| not_removed_by_x9(c))\n .unwrap_or(run.end - run.start - 1)];\n\n // Get the level of the last non-removed char before the run.\n let pred_level = match original_classes[..run.start]\n .iter()\n .rposition(not_removed_by_x9)\n {\n Some(idx) => levels[idx],\n None => para_level,\n };\n\n // Get the level of the next non-removed char after the run.\n let succ_level = match original_classes[run.end..]\n .iter()\n .position(not_removed_by_x9)\n {\n Some(idx) => levels[run.end + idx],\n None => para_level,\n };\n\n isolating_run_sequences.push(IsolatingRunSequence {\n runs: vec![run],\n sos: max(seq_level, pred_level).bidi_class(),\n eos: max(end_level, succ_level).bidi_class(),\n });\n }\n return;\n }\n\n // Compute the set of isolating run sequences.\n // \n let mut sequences = Vec::with_capacity(runs.len());\n\n // When we encounter an isolate initiator, we push the current sequence onto the\n // stack so we can resume it after the matching PDI.\n #[cfg(feature = \"smallvec\")]\n let mut stack: SmallVec<[Vec>; 8]> = smallvec![vec![]];\n #[cfg(not(feature = \"smallvec\"))]\n let mut stack = vec![vec![]];\n\n for run in runs {\n assert!(!run.is_empty());\n assert!(!stack.is_empty());\n\n let start_class = original_classes[run.start];\n // > In rule X10, [..] skip over any BNs when [..].\n // > Do the same when determining if the last character of the sequence is an isolate initiator.\n //\n // \n let end_class = original_classes[run.start..run.end]\n .iter()\n .copied()\n .rev()\n .find(not_removed_by_x9)\n .unwrap_or(start_class);\n\n let mut sequence = if start_class == PDI && stack.len() > 1 {\n // Continue a previous sequence interrupted by an isolate.\n stack.pop().unwrap()\n } else {\n // Start a new sequence.\n Vec::new()\n };\n\n sequence.push(run);\n\n if matches!(end_class, RLI | LRI | FSI) {\n // Resume this sequence after the isolate.\n stack.push(sequence);\n } else {\n // This sequence is finished.\n sequences.push(sequence);\n }\n }\n // Pop any remaining sequences off the stack.\n sequences.extend(stack.into_iter().rev().filter(|seq| !seq.is_empty()));\n\n // Determine the `sos` and `eos` class for each sequence.\n // \n for sequence in sequences {\n assert!(!sequence.is_empty());\n\n let start_of_seq = sequence[0].start;\n let runs_len = sequence.len();\n let end_of_seq = sequence[runs_len - 1].end;\n\n let mut result = IsolatingRunSequence {\n runs: sequence,\n sos: L,\n eos: L,\n };\n\n // > (not counting characters removed by X9)\n let seq_level = levels[result\n .iter_forwards_from(start_of_seq, 0)\n .find(|i| not_removed_by_x9(&original_classes[*i]))\n .unwrap_or(start_of_seq)];\n\n // XXXManishearth the spec talks of a start and end level,\n // but for a given IRS the two should be equivalent, yes?\n let end_level = levels[result\n .iter_backwards_from(end_of_seq, runs_len - 1)\n .find(|i| not_removed_by_x9(&original_classes[*i]))\n .unwrap_or(end_of_seq - 1)];\n\n #[cfg(test)]\n for idx in result.runs.clone().into_iter().flatten() {\n if not_removed_by_x9(&original_classes[idx]) {\n assert_eq!(seq_level, levels[idx]);\n }\n }\n\n // Get the level of the last non-removed char before the runs.\n let pred_level = match original_classes[..start_of_seq]\n .iter()\n .rposition(not_removed_by_x9)\n {\n Some(idx) => levels[idx],\n None => para_level,\n };\n\n // Get the last non-removed character to check if it is an isolate initiator.\n // The spec calls for an unmatched one, but matched isolate initiators\n // will never be at the end of a level run (otherwise there would be more to the run).\n // We unwrap_or(BN) because BN marks removed classes and it won't matter for the check.\n let last_non_removed = original_classes[..end_of_seq]\n .iter()\n .copied()\n .rev()\n .find(not_removed_by_x9)\n .unwrap_or(BN);\n\n // Get the level of the next non-removed char after the runs.\n let succ_level = if matches!(last_non_removed, RLI | LRI | FSI) {\n para_level\n } else {\n match original_classes[end_of_seq..]\n .iter()\n .position(not_removed_by_x9)\n {\n Some(idx) => levels[end_of_seq + idx],\n None => para_level,\n }\n };\n\n result.sos = max(seq_level, pred_level).bidi_class();\n result.eos = max(end_level, succ_level).bidi_class();\n\n isolating_run_sequences.push(result);\n }\n}\n\nimpl IsolatingRunSequence {\n /// Given a text-relative position `pos` and an index of the level run it is in,\n /// produce an iterator of all characters after and pos (`pos..`) that are in this\n /// run sequence\n pub(crate) fn iter_forwards_from(\n &self,\n pos: usize,\n level_run_index: usize,\n ) -> impl Iterator + '_ {\n let runs = &self.runs[level_run_index..];\n\n // Check that it is in range\n // (we can't use contains() since we want an inclusive range)\n #[cfg(feature = \"std\")]\n debug_assert!(runs[0].start <= pos && pos <= runs[0].end);\n\n (pos..runs[0].end).chain(runs[1..].iter().flat_map(Clone::clone))\n }\n\n /// Given a text-relative position `pos` and an index of the level run it is in,\n /// produce an iterator of all characters before and excludingpos (`..pos`) that are in this\n /// run sequence\n pub(crate) fn iter_backwards_from(\n &self,\n pos: usize,\n level_run_index: usize,\n ) -> impl Iterator + '_ {\n let prev_runs = &self.runs[..level_run_index];\n let current = &self.runs[level_run_index];\n\n // Check that it is in range\n // (we can't use contains() since we want an inclusive range)\n #[cfg(feature = \"std\")]\n debug_assert!(current.start <= pos && pos <= current.end);\n\n (current.start..pos)\n .rev()\n .chain(prev_runs.iter().rev().flat_map(Clone::clone))\n }\n}\n\n/// Finds the level runs in a paragraph.\n///\n/// \n///\n/// This is only used by tests; normally level runs are identified during explicit::compute.\n#[cfg(test)]\nfn level_runs(levels: &[Level], original_classes: &[BidiClass]) -> Vec {\n assert_eq!(levels.len(), original_classes.len());\n\n let mut runs = Vec::new();\n if levels.is_empty() {\n return runs;\n }\n\n let mut current_run_level = levels[0];\n let mut current_run_start = 0;\n for i in 1..levels.len() {\n if !removed_by_x9(original_classes[i]) && levels[i] != current_run_level {\n // End the last run and start a new one.\n runs.push(current_run_start..i);\n current_run_level = levels[i];\n current_run_start = i;\n }\n }\n runs.push(current_run_start..levels.len());\n\n runs\n}\n\n/// Should this character be ignored in steps after X9?\n///\n/// \npub fn removed_by_x9(class: BidiClass) -> bool {\n matches!(class, RLE | LRE | RLO | LRO | PDF | BN)\n}\n\n// For use as a predicate for `position` / `rposition`\npub fn not_removed_by_x9(class: &BidiClass) -> bool {\n !removed_by_x9(*class)\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn test_level_runs() {\n assert_eq!(level_runs(&Level::vec(&[]), &[]), &[]);\n assert_eq!(\n level_runs(&Level::vec(&[0, 0, 0, 1, 1, 2, 0, 0]), &[L; 8]),\n &[0..3, 3..5, 5..6, 6..8]\n );\n }\n\n // From \n #[rustfmt::skip]\n #[test]\n fn test_isolating_run_sequences() {\n\n // == Example 1 ==\n // text1·RLE·text2·PDF·RLE·text3·PDF·text4\n // index 0 1 2 3 4 5 6 7\n let classes = &[L, RLE, L, PDF, RLE, L, PDF, L];\n let levels = &[0, 1, 1, 1, 1, 1, 1, 0];\n let para_level = Level::ltr();\n let mut sequences = IsolatingRunSequenceVec::new();\n isolating_run_sequences(\n para_level,\n classes,\n &Level::vec(levels),\n level_runs(&Level::vec(levels), classes).into(),\n false,\n &mut sequences);\n sequences.sort_by(|a, b| a.runs[0].clone().cmp(b.runs[0].clone()));\n assert_eq!(\n sequences.iter().map(|s| s.runs.clone()).collect::>(),\n vec![vec![0..2], vec![2..7], vec![7..8]]\n );\n\n // == Example 2 ==\n // text1·RLI·text2·PDI·RLI·text3·PDI·text4\n // index 0 1 2 3 4 5 6 7\n let classes = &[L, RLI, L, PDI, RLI, L, PDI, L];\n let levels = &[0, 0, 1, 0, 0, 1, 0, 0];\n let para_level = Level::ltr();\n let mut sequences = IsolatingRunSequenceVec::new();\n isolating_run_sequences(\n para_level,\n classes,\n &Level::vec(levels),\n level_runs(&Level::vec(levels), classes).into(),\n true,\n &mut sequences);\n sequences.sort_by(|a, b| a.runs[0].clone().cmp(b.runs[0].clone()));\n assert_eq!(\n sequences.iter().map(|s| s.runs.clone()).collect::>(),\n vec![vec![0..2, 3..5, 6..8], vec![2..3], vec![5..6]]\n );\n\n // == Example 3 ==\n // text1·RLI·text2·LRI·text3·RLE·text4·PDF·text5·PDI·text6·PDI·text7\n // index 0 1 2 3 4 5 6 7 8 9 10 11 12\n let classes = &[L, RLI, L, LRI, L, RLE, L, PDF, L, PDI, L, PDI, L];\n let levels = &[0, 0, 1, 1, 2, 3, 3, 3, 2, 1, 1, 0, 0];\n let para_level = Level::ltr();\n let mut sequences = IsolatingRunSequenceVec::new();\n isolating_run_sequences(\n para_level,\n classes,\n &Level::vec(levels),\n level_runs(&Level::vec(levels), classes).into(),\n true,\n &mut sequences);\n sequences.sort_by(|a, b| a.runs[0].clone().cmp(b.runs[0].clone()));\n assert_eq!(\n sequences.iter().map(|s| s.runs.clone()).collect::>(),\n vec![vec![0..2, 11..13], vec![2..4, 9..11], vec![4..6], vec![6..8], vec![8..9]]\n );\n }\n\n // From \n #[rustfmt::skip]\n #[test]\n fn test_isolating_run_sequences_sos_and_eos() {\n\n // == Example 1 ==\n // text1·RLE·text2·LRE·text3·PDF·text4·PDF·RLE·text5·PDF·text6\n // index 0 1 2 3 4 5 6 7 8 9 10 11\n let classes = &[L, RLE, L, LRE, L, PDF, L, PDF, RLE, L, PDF, L];\n let levels = &[0, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 0];\n let para_level = Level::ltr();\n let mut sequences = IsolatingRunSequenceVec::new();\n isolating_run_sequences(\n para_level,\n classes,\n &Level::vec(levels),\n level_runs(&Level::vec(levels), classes).into(),\n false,\n &mut sequences);\n sequences.sort_by(|a, b| a.runs[0].clone().cmp(b.runs[0].clone()));\n\n // text1\n assert_eq!(\n &sequences[0],\n &IsolatingRunSequence {\n runs: vec![0..2],\n sos: L,\n eos: R,\n }\n );\n\n // text2\n assert_eq!(\n &sequences[1],\n &IsolatingRunSequence {\n runs: vec![2..4],\n sos: R,\n eos: L,\n }\n );\n\n // text3\n assert_eq!(\n &sequences[2],\n &IsolatingRunSequence {\n runs: vec![4..6],\n sos: L,\n eos: L,\n }\n );\n\n // text4 text5\n assert_eq!(\n &sequences[3],\n &IsolatingRunSequence {\n runs: vec![6..11],\n sos: L,\n eos: R,\n }\n );\n\n // text6\n assert_eq!(\n &sequences[4],\n &IsolatingRunSequence {\n runs: vec![11..12],\n sos: R,\n eos: L,\n }\n );\n\n // == Example 2 ==\n // text1·RLI·text2·LRI·text3·PDI·text4·PDI·RLI·text5·PDI·text6\n // index 0 1 2 3 4 5 6 7 8 9 10 11\n let classes = &[L, RLI, L, LRI, L, PDI, L, PDI, RLI, L, PDI, L];\n let levels = &[0, 0, 1, 1, 2, 1, 1, 0, 0, 1, 0, 0];\n let para_level = Level::ltr();\n let mut sequences = IsolatingRunSequenceVec::new();\n isolating_run_sequences(\n para_level,\n classes,\n &Level::vec(levels),\n level_runs(&Level::vec(levels), classes).into(),\n true,\n &mut sequences);\n sequences.sort_by(|a, b| a.runs[0].clone().cmp(b.runs[0].clone()));\n\n // text1·RLI·PDI·RLI·PDI·text6\n assert_eq!(\n &sequences[0],\n &IsolatingRunSequence {\n runs: vec![0..2, 7..9, 10..12],\n sos: L,\n eos: L,\n }\n );\n\n // text2·LRI·PDI·text4\n assert_eq!(\n &sequences[1],\n &IsolatingRunSequence {\n runs: vec![2..4, 5..7],\n sos: R,\n eos: R,\n }\n );\n\n // text3\n assert_eq!(\n &sequences[2],\n &IsolatingRunSequence {\n runs: vec![4..5],\n sos: L,\n eos: L,\n }\n );\n\n // text5\n assert_eq!(\n &sequences[3],\n &IsolatingRunSequence {\n runs: vec![9..10],\n sos: R,\n eos: R,\n }\n );\n }\n\n #[test]\n fn test_removed_by_x9() {\n let rem_classes = &[RLE, LRE, RLO, LRO, PDF, BN];\n let not_classes = &[L, RLI, AL, LRI, PDI];\n for x in rem_classes {\n assert_eq!(removed_by_x9(*x), true);\n }\n for x in not_classes {\n assert_eq!(removed_by_x9(*x), false);\n }\n }\n\n #[test]\n fn test_not_removed_by_x9() {\n let non_x9_classes = &[L, R, AL, EN, ES, ET, AN, CS, NSM, B, S, WS, ON, LRI, RLI, FSI, PDI];\n for x in non_x9_classes {\n assert_eq!(not_removed_by_x9(&x), true);\n }\n }\n}\n" }, { "package": "unicode-bidi@0.3.18", "sourcePath": "src/utf16.rs", "sha256": "12ee177127a0b5b0350a1fcc1edf7387c26b51ec5654f724629aab723881c313", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "// Copyright 2023 The Mozilla Foundation. See the\n// COPYRIGHT file at the top-level directory of this distribution.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\nuse super::TextSource;\n\nuse alloc::borrow::Cow;\nuse alloc::vec::Vec;\nuse core::char;\nuse core::ops::Range;\n\nuse crate::{\n compute_bidi_info_for_para, compute_initial_info, level, para_direction, reorder_levels,\n reorder_visual, visual_runs_for_line,\n};\nuse crate::{\n BidiClass, BidiDataSource, Direction, Level, LevelRun, ParagraphInfo, ParagraphInfoFlags,\n};\n\n#[cfg(feature = \"hardcoded-data\")]\nuse crate::HardcodedBidiData;\n\n/// Initial bidi information of the text (UTF-16 version).\n///\n/// Contains the text paragraphs and `BidiClass` of its characters.\n#[derive(PartialEq, Debug)]\npub struct InitialInfo<'text> {\n /// The text\n pub text: &'text [u16],\n\n /// The BidiClass of the character at each code unit in the text.\n /// If a character is multiple code units, its class will appear multiple times in the vector.\n pub original_classes: Vec,\n\n /// The boundaries and level of each paragraph within the text.\n pub paragraphs: Vec,\n}\n\nimpl<'text> InitialInfo<'text> {\n /// Find the paragraphs and BidiClasses in a string of text.\n ///\n /// \n ///\n /// Also sets the class for each First Strong Isolate initiator (FSI) to LRI or RLI if a strong\n /// character is found before the matching PDI. If no strong character is found, the class will\n /// remain FSI, and it's up to later stages to treat these as LRI when needed.\n ///\n /// The `hardcoded-data` Cargo feature (enabled by default) must be enabled to use this.\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[cfg(feature = \"hardcoded-data\")]\n pub fn new(text: &[u16], default_para_level: Option) -> InitialInfo<'_> {\n Self::new_with_data_source(&HardcodedBidiData, text, default_para_level)\n }\n\n /// Find the paragraphs and BidiClasses in a string of text, with a custom [`BidiDataSource`]\n /// for Bidi data. If you just wish to use the hardcoded Bidi data, please use [`InitialInfo::new()`]\n /// instead (enabled with tbe default `hardcoded-data` Cargo feature)\n ///\n /// \n ///\n /// Also sets the class for each First Strong Isolate initiator (FSI) to LRI or RLI if a strong\n /// character is found before the matching PDI. If no strong character is found, the class will\n /// remain FSI, and it's up to later stages to treat these as LRI when needed.\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn new_with_data_source<'a, D: BidiDataSource>(\n data_source: &D,\n text: &'a [u16],\n default_para_level: Option,\n ) -> InitialInfo<'a> {\n InitialInfoExt::new_with_data_source(data_source, text, default_para_level).base\n }\n}\n\n/// Extended version of InitialInfo (not public API).\n#[derive(PartialEq, Debug)]\nstruct InitialInfoExt<'text> {\n /// The base InitialInfo for the text, recording its paragraphs and bidi classes.\n base: InitialInfo<'text>,\n\n /// Parallel to base.paragraphs, records whether each paragraph is \"pure LTR\" that\n /// requires no further bidi processing (i.e. there are no RTL characters or bidi\n /// control codes present).\n flags: Vec,\n}\n\nimpl<'text> InitialInfoExt<'text> {\n /// Find the paragraphs and BidiClasses in a string of text, with a custom [`BidiDataSource`]\n /// for Bidi data. If you just wish to use the hardcoded Bidi data, please use [`InitialInfo::new()`]\n /// instead (enabled with tbe default `hardcoded-data` Cargo feature)\n ///\n /// \n ///\n /// Also sets the class for each First Strong Isolate initiator (FSI) to LRI or RLI if a strong\n /// character is found before the matching PDI. If no strong character is found, the class will\n /// remain FSI, and it's up to later stages to treat these as LRI when needed.\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn new_with_data_source<'a, D: BidiDataSource>(\n data_source: &D,\n text: &'a [u16],\n default_para_level: Option,\n ) -> InitialInfoExt<'a> {\n let mut paragraphs = Vec::::new();\n let mut flags = Vec::::new();\n let (original_classes, _, _, _) = compute_initial_info(\n data_source,\n text,\n default_para_level,\n Some((&mut paragraphs, &mut flags)),\n );\n\n InitialInfoExt {\n base: InitialInfo {\n text,\n original_classes,\n paragraphs,\n },\n flags,\n }\n }\n}\n\n/// Bidi information of the text (UTF-16 version).\n///\n/// The `original_classes` and `levels` vectors are indexed by code unit offsets into the text. If a\n/// character is multiple code units wide, then its class and level will appear multiple times in these\n/// vectors.\n// TODO: Impl `struct StringProperty { values: Vec }` and use instead of Vec\n#[derive(Debug, PartialEq)]\npub struct BidiInfo<'text> {\n /// The text\n pub text: &'text [u16],\n\n /// The BidiClass of the character at each byte in the text.\n pub original_classes: Vec,\n\n /// The directional embedding level of each byte in the text.\n pub levels: Vec,\n\n /// The boundaries and paragraph embedding level of each paragraph within the text.\n ///\n /// TODO: Use SmallVec or similar to avoid overhead when there are only one or two paragraphs?\n /// Or just don't include the first paragraph, which always starts at 0?\n pub paragraphs: Vec,\n}\n\nimpl<'text> BidiInfo<'text> {\n /// Split the text into paragraphs and determine the bidi embedding levels for each paragraph.\n ///\n ///\n /// The `hardcoded-data` Cargo feature (enabled by default) must be enabled to use this.\n ///\n /// TODO: In early steps, check for special cases that allow later steps to be skipped. like\n /// text that is entirely LTR. See the `nsBidi` class from Gecko for comparison.\n ///\n /// TODO: Support auto-RTL base direction\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[cfg(feature = \"hardcoded-data\")]\n #[inline]\n pub fn new(text: &[u16], default_para_level: Option) -> BidiInfo<'_> {\n Self::new_with_data_source(&HardcodedBidiData, text, default_para_level)\n }\n\n /// Split the text into paragraphs and determine the bidi embedding levels for each paragraph, with a custom [`BidiDataSource`]\n /// for Bidi data. If you just wish to use the hardcoded Bidi data, please use [`BidiInfo::new()`]\n /// instead (enabled with tbe default `hardcoded-data` Cargo feature).\n ///\n /// TODO: In early steps, check for special cases that allow later steps to be skipped. like\n /// text that is entirely LTR. See the `nsBidi` class from Gecko for comparison.\n ///\n /// TODO: Support auto-RTL base direction\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn new_with_data_source<'a, D: BidiDataSource>(\n data_source: &D,\n text: &'a [u16],\n default_para_level: Option,\n ) -> BidiInfo<'a> {\n let InitialInfoExt { base, flags, .. } =\n InitialInfoExt::new_with_data_source(data_source, text, default_para_level);\n\n let mut levels = Vec::::with_capacity(text.len());\n let mut processing_classes = base.original_classes.clone();\n\n for (para, flags) in base.paragraphs.iter().zip(flags.iter()) {\n let text = &text[para.range.clone()];\n let original_classes = &base.original_classes[para.range.clone()];\n\n compute_bidi_info_for_para(\n data_source,\n para,\n flags.is_pure_ltr,\n flags.has_isolate_controls,\n text,\n original_classes,\n &mut processing_classes,\n &mut levels,\n );\n }\n\n BidiInfo {\n text,\n original_classes: base.original_classes,\n paragraphs: base.paragraphs,\n levels,\n }\n }\n\n /// Produce the levels for this paragraph as needed for reordering, one level per *byte*\n /// in the paragraph. The returned vector includes bytes that are not included\n /// in the `line`, but will not adjust them.\n ///\n /// This runs [Rule L1], you can run\n /// [Rule L2] by calling [`Self::reorder_visual()`].\n /// If doing so, you may prefer to use [`Self::reordered_levels_per_char()`] instead\n /// to avoid non-byte indices.\n ///\n /// For an all-in-one reordering solution, consider using [`Self::reorder_visual()`].\n ///\n /// [Rule L1]: https://www.unicode.org/reports/tr9/#L1\n /// [Rule L2]: https://www.unicode.org/reports/tr9/#L2\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reordered_levels(&self, para: &ParagraphInfo, line: Range) -> Vec {\n assert!(line.start <= self.levels.len());\n assert!(line.end <= self.levels.len());\n\n let mut levels = self.levels.clone();\n let line_classes = &self.original_classes[line.clone()];\n let line_levels = &mut levels[line.clone()];\n let line_str: &[u16] = &self.text[line.clone()];\n\n reorder_levels(line_classes, line_levels, line_str, para.level);\n\n levels\n }\n\n /// Produce the levels for this paragraph as needed for reordering, one level per *character*\n /// in the paragraph. The returned vector includes characters that are not included\n /// in the `line`, but will not adjust them.\n ///\n /// This runs [Rule L1], you can run\n /// [Rule L2] by calling [`Self::reorder_visual()`].\n /// If doing so, you may prefer to use [`Self::reordered_levels_per_char()`] instead\n /// to avoid non-byte indices.\n ///\n /// For an all-in-one reordering solution, consider using [`Self::reorder_visual()`].\n ///\n /// [Rule L1]: https://www.unicode.org/reports/tr9/#L1\n /// [Rule L2]: https://www.unicode.org/reports/tr9/#L2\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reordered_levels_per_char(\n &self,\n para: &ParagraphInfo,\n line: Range,\n ) -> Vec {\n let levels = self.reordered_levels(para, line);\n self.text.char_indices().map(|(i, _)| levels[i]).collect()\n }\n\n /// Re-order a line based on resolved levels and return the line in display order.\n ///\n /// This does not apply [Rule L3] or [Rule L4] around combining characters or mirroring.\n ///\n /// [Rule L3]: https://www.unicode.org/reports/tr9/#L3\n /// [Rule L4]: https://www.unicode.org/reports/tr9/#L4\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reorder_line(&self, para: &ParagraphInfo, line: Range) -> Cow<'text, [u16]> {\n if !level::has_rtl(&self.levels[line.clone()]) {\n return self.text[line].into();\n }\n let (levels, runs) = self.visual_runs(para, line.clone());\n reorder_line(self.text, line, levels, runs)\n }\n\n /// Reorders pre-calculated levels of a sequence of characters.\n ///\n /// NOTE: This is a convenience method that does not use a `Paragraph` object. It is\n /// intended to be used when an application has determined the levels of the objects (character sequences)\n /// and just needs to have them reordered.\n ///\n /// the index map will result in `indexMap[visualIndex]==logicalIndex`.\n ///\n /// This only runs [Rule L2](http://www.unicode.org/reports/tr9/#L2) as it does not have\n /// information about the actual text.\n ///\n /// Furthermore, if `levels` is an array that is aligned with code units, bytes within a codepoint may be\n /// reversed. You may need to fix up the map to deal with this. Alternatively, only pass in arrays where each `Level`\n /// is for a single code point.\n ///\n ///\n /// # # Example\n /// ```\n /// use unicode_bidi::BidiInfo;\n /// use unicode_bidi::Level;\n ///\n /// let l0 = Level::from(0);\n /// let l1 = Level::from(1);\n /// let l2 = Level::from(2);\n ///\n /// let levels = vec![l0, l0, l0, l0];\n /// let index_map = BidiInfo::reorder_visual(&levels);\n /// assert_eq!(levels.len(), index_map.len());\n /// assert_eq!(index_map, [0, 1, 2, 3]);\n ///\n /// let levels: Vec = vec![l0, l0, l0, l1, l1, l1, l2, l2];\n /// let index_map = BidiInfo::reorder_visual(&levels);\n /// assert_eq!(levels.len(), index_map.len());\n /// assert_eq!(index_map, [0, 1, 2, 6, 7, 5, 4, 3]);\n /// ```\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[inline]\n pub fn reorder_visual(levels: &[Level]) -> Vec {\n reorder_visual(levels)\n }\n\n /// Find the level runs within a line and return them in visual order.\n ///\n /// `line` is a range of bytes indices within `levels`.\n ///\n /// The first return value is a vector of levels used by the reordering algorithm,\n /// i.e. the result of [Rule L1]. The second return value is a vector of level runs,\n /// the result of [Rule L2], showing the visual order that each level run (a run of text with the\n /// same level) should be displayed. Within each run, the display order can be checked\n /// against the Level vector.\n ///\n /// This does not handle [Rule L3] (combining characters) or [Rule L4] (mirroring),\n /// as that should be handled by the engine using this API.\n ///\n /// Conceptually, this is the same as running [`Self::reordered_levels()`] followed by\n /// [`Self::reorder_visual()`], however it returns the result as a list of level runs instead\n /// of producing a level map, since one may wish to deal with the fact that this is operating on\n /// byte rather than character indices.\n ///\n /// \n ///\n /// [Rule L1]: https://www.unicode.org/reports/tr9/#L1\n /// [Rule L2]: https://www.unicode.org/reports/tr9/#L2\n /// [Rule L3]: https://www.unicode.org/reports/tr9/#L3\n /// [Rule L4]: https://www.unicode.org/reports/tr9/#L4\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[inline]\n pub fn visual_runs(\n &self,\n para: &ParagraphInfo,\n line: Range,\n ) -> (Vec, Vec) {\n let levels = self.reordered_levels(para, line.clone());\n visual_runs_for_line(levels, &line)\n }\n\n /// If processed text has any computed RTL levels\n ///\n /// This information is usually used to skip re-ordering of text when no RTL level is present\n #[inline]\n pub fn has_rtl(&self) -> bool {\n level::has_rtl(&self.levels)\n }\n}\n\n/// Bidi information of text treated as a single paragraph.\n///\n/// The `original_classes` and `levels` vectors are indexed by code unit offsets into the text. If a\n/// character is multiple code units wide, then its class and level will appear multiple times in these\n/// vectors.\n#[derive(Debug, PartialEq)]\npub struct ParagraphBidiInfo<'text> {\n /// The text\n pub text: &'text [u16],\n\n /// The BidiClass of the character at each byte in the text.\n pub original_classes: Vec,\n\n /// The directional embedding level of each byte in the text.\n pub levels: Vec,\n\n /// The paragraph embedding level.\n pub paragraph_level: Level,\n\n /// Whether the paragraph is purely LTR.\n pub is_pure_ltr: bool,\n}\n\nimpl<'text> ParagraphBidiInfo<'text> {\n /// Determine the bidi embedding level.\n ///\n ///\n /// The `hardcoded-data` Cargo feature (enabled by default) must be enabled to use this.\n ///\n /// TODO: In early steps, check for special cases that allow later steps to be skipped. like\n /// text that is entirely LTR. See the `nsBidi` class from Gecko for comparison.\n ///\n /// TODO: Support auto-RTL base direction\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[cfg(feature = \"hardcoded-data\")]\n #[inline]\n pub fn new(text: &[u16], default_para_level: Option) -> ParagraphBidiInfo<'_> {\n Self::new_with_data_source(&HardcodedBidiData, text, default_para_level)\n }\n\n /// Determine the bidi embedding level, with a custom [`BidiDataSource`]\n /// for Bidi data. If you just wish to use the hardcoded Bidi data, please use [`BidiInfo::new()`]\n /// instead (enabled with tbe default `hardcoded-data` Cargo feature).\n ///\n /// (This is the single-paragraph equivalent of BidiInfo::new_with_data_source,\n /// and should be kept in sync with it.\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn new_with_data_source<'a, D: BidiDataSource>(\n data_source: &D,\n text: &'a [u16],\n default_para_level: Option,\n ) -> ParagraphBidiInfo<'a> {\n // Here we could create a ParagraphInitialInfo struct to parallel the one\n // used by BidiInfo, but there doesn't seem any compelling reason for it.\n let (original_classes, paragraph_level, is_pure_ltr, has_isolate_controls) =\n compute_initial_info(data_source, text, default_para_level, None);\n\n let mut levels = Vec::::with_capacity(text.len());\n let mut processing_classes = original_classes.clone();\n\n let para_info = ParagraphInfo {\n range: Range {\n start: 0,\n end: text.len(),\n },\n level: paragraph_level,\n };\n\n compute_bidi_info_for_para(\n data_source,\n ¶_info,\n is_pure_ltr,\n has_isolate_controls,\n text,\n &original_classes,\n &mut processing_classes,\n &mut levels,\n );\n\n ParagraphBidiInfo {\n text,\n original_classes,\n levels,\n paragraph_level,\n is_pure_ltr,\n }\n }\n\n /// Produce the levels for this paragraph as needed for reordering, one level per *code unit*\n /// in the paragraph. The returned vector includes code units that are not included\n /// in the `line`, but will not adjust them.\n ///\n /// See BidiInfo::reordered_levels for details.\n ///\n /// (This should be kept in sync with BidiInfo::reordered_levels.)\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reordered_levels(&self, line: Range) -> Vec {\n assert!(line.start <= self.levels.len());\n assert!(line.end <= self.levels.len());\n\n let mut levels = self.levels.clone();\n let line_classes = &self.original_classes[line.clone()];\n let line_levels = &mut levels[line.clone()];\n\n reorder_levels(\n line_classes,\n line_levels,\n self.text.subrange(line),\n self.paragraph_level,\n );\n\n levels\n }\n\n /// Produce the levels for this paragraph as needed for reordering, one level per *character*\n /// in the paragraph. The returned vector includes characters that are not included\n /// in the `line`, but will not adjust them.\n ///\n /// See BidiInfo::reordered_levels_per_char for details.\n ///\n /// (This should be kept in sync with BidiInfo::reordered_levels_per_char.)\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reordered_levels_per_char(&self, line: Range) -> Vec {\n let levels = self.reordered_levels(line);\n self.text.char_indices().map(|(i, _)| levels[i]).collect()\n }\n\n /// Re-order a line based on resolved levels and return the line in display order.\n ///\n /// See BidiInfo::reorder_line for details.\n ///\n /// (This should be kept in sync with BidiInfo::reorder_line.)\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n pub fn reorder_line(&self, line: Range) -> Cow<'text, [u16]> {\n if !level::has_rtl(&self.levels[line.clone()]) {\n return self.text[line].into();\n }\n let (levels, runs) = self.visual_runs(line.clone());\n reorder_line(self.text, line, levels, runs)\n }\n\n /// Reorders pre-calculated levels of a sequence of characters.\n ///\n /// See BidiInfo::reorder_visual for details.\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[inline]\n pub fn reorder_visual(levels: &[Level]) -> Vec {\n reorder_visual(levels)\n }\n\n /// Find the level runs within a line and return them in visual order.\n ///\n /// `line` is a range of code-unit indices within `levels`.\n ///\n /// See `BidiInfo::visual_runs` for details.\n ///\n /// (This should be kept in sync with BidiInfo::visual_runs.)\n #[cfg_attr(feature = \"flame_it\", flamer::flame)]\n #[inline]\n pub fn visual_runs(&self, line: Range) -> (Vec, Vec) {\n let levels = self.reordered_levels(line.clone());\n visual_runs_for_line(levels, &line)\n }\n\n /// If processed text has any computed RTL levels\n ///\n /// This information is usually used to skip re-ordering of text when no RTL level is present\n #[inline]\n pub fn has_rtl(&self) -> bool {\n !self.is_pure_ltr\n }\n\n /// Return the paragraph's Direction (Ltr, Rtl, or Mixed) based on its levels.\n #[inline]\n pub fn direction(&self) -> Direction {\n para_direction(&self.levels)\n }\n}\n\n/// Return a line of the text in display order based on resolved levels.\n///\n/// `text` the full text passed to the `BidiInfo` or `ParagraphBidiInfo` for analysis\n/// `line` a range of byte indices within `text` corresponding to one line\n/// `levels` array of `Level` values, with `line`'s levels reordered into visual order\n/// `runs` array of `LevelRun`s in visual order\n///\n/// (`levels` and `runs` are the result of calling `BidiInfo::visual_runs()` or\n/// `ParagraphBidiInfo::visual_runs()` for the line of interest.)\n///\n/// Returns: the reordered text of the line.\n///\n/// This does not apply [Rule L3] or [Rule L4] around combining characters or mirroring.\n///\n/// [Rule L3]: https://www.unicode.org/reports/tr9/#L3\n/// [Rule L4]: https://www.unicode.org/reports/tr9/#L4\nfn reorder_line(\n text: &[u16],\n line: Range,\n levels: Vec,\n runs: Vec,\n) -> Cow<'_, [u16]> {\n // If all isolating run sequences are LTR, no reordering is needed\n if runs.iter().all(|run| levels[run.start].is_ltr()) {\n return text[line].into();\n }\n\n let mut result = Vec::::with_capacity(line.len());\n for run in runs {\n if levels[run.start].is_rtl() {\n let mut buf = [0; 2];\n for c in text[run].chars().rev() {\n result.extend(c.encode_utf16(&mut buf).iter());\n }\n } else {\n result.extend(text[run].iter());\n }\n }\n result.into()\n}\n\n/// Contains a reference of `BidiInfo` and one of its `paragraphs`.\n/// And it supports all operation in the `Paragraph` that needs also its\n/// `BidiInfo` such as `direction`.\n#[derive(Debug)]\npub struct Paragraph<'a, 'text> {\n pub info: &'a BidiInfo<'text>,\n pub para: &'a ParagraphInfo,\n}\n\nimpl<'a, 'text> Paragraph<'a, 'text> {\n #[inline]\n pub fn new(info: &'a BidiInfo<'text>, para: &'a ParagraphInfo) -> Paragraph<'a, 'text> {\n Paragraph { info, para }\n }\n\n /// Returns if the paragraph is Left direction, right direction or mixed.\n #[inline]\n pub fn direction(&self) -> Direction {\n para_direction(&self.info.levels[self.para.range.clone()])\n }\n\n /// Returns the `Level` of a certain character in the paragraph.\n #[inline]\n pub fn level_at(&self, pos: usize) -> Level {\n let actual_position = self.para.range.start + pos;\n self.info.levels[actual_position]\n }\n}\n\n/// Implementation of TextSource for UTF-16 text in a [u16] array.\n/// Note that there could be unpaired surrogates present!\n\n// Convenience functions to check whether a UTF16 code unit is a surrogate.\n#[inline]\nfn is_high_surrogate(code: u16) -> bool {\n (code & 0xFC00) == 0xD800\n}\n#[inline]\nfn is_low_surrogate(code: u16) -> bool {\n (code & 0xFC00) == 0xDC00\n}\n\nimpl<'text> TextSource<'text> for [u16] {\n type CharIter = Utf16CharIter<'text>;\n type CharIndexIter = Utf16CharIndexIter<'text>;\n type IndexLenIter = Utf16IndexLenIter<'text>;\n\n #[inline]\n fn len(&self) -> usize {\n (self as &[u16]).len()\n }\n fn char_at(&self, index: usize) -> Option<(char, usize)> {\n if index >= self.len() {\n return None;\n }\n // Get the indicated code unit and try simply converting it to a char;\n // this will fail if it is half of a surrogate pair.\n let c = self[index];\n if let Some(ch) = char::from_u32(c.into()) {\n return Some((ch, 1));\n }\n // If it's a low surrogate, and was immediately preceded by a high surrogate,\n // then we're in the middle of a (valid) character, and should return None.\n if is_low_surrogate(c) && index > 0 && is_high_surrogate(self[index - 1]) {\n return None;\n }\n // Otherwise, try to decode, returning REPLACEMENT_CHARACTER for errors.\n if let Some(ch) = char::decode_utf16(self[index..].iter().cloned()).next() {\n if let Ok(ch) = ch {\n // This must be a surrogate pair, otherwise char::from_u32() above should\n // have succeeded!\n debug_assert!(ch.len_utf16() == 2, \"BMP should have already been handled\");\n return Some((ch, ch.len_utf16()));\n }\n } else {\n debug_assert!(\n false,\n \"Why did decode_utf16 return None when we're not at the end?\"\n );\n return None;\n }\n // Failed to decode UTF-16: we must have encountered an unpaired surrogate.\n // Return REPLACEMENT_CHARACTER (not None), to continue processing the following text\n // and keep indexing correct.\n Some((char::REPLACEMENT_CHARACTER, 1))\n }\n #[inline]\n fn subrange(&self, range: Range) -> &Self {\n &(self as &[u16])[range]\n }\n #[inline]\n fn chars(&'text self) -> Self::CharIter {\n Utf16CharIter::new(self)\n }\n #[inline]\n fn char_indices(&'text self) -> Self::CharIndexIter {\n Utf16CharIndexIter::new(self)\n }\n #[inline]\n fn indices_lengths(&'text self) -> Self::IndexLenIter {\n Utf16IndexLenIter::new(self)\n }\n #[inline]\n fn char_len(ch: char) -> usize {\n ch.len_utf16()\n }\n}\n\n/// Iterator over UTF-16 text in a [u16] slice, returning (index, char_len) tuple.\n#[derive(Debug)]\npub struct Utf16IndexLenIter<'text> {\n text: &'text [u16],\n cur_pos: usize,\n}\n\nimpl<'text> Utf16IndexLenIter<'text> {\n #[inline]\n pub fn new(text: &'text [u16]) -> Self {\n Utf16IndexLenIter { text, cur_pos: 0 }\n }\n}\n\nimpl Iterator for Utf16IndexLenIter<'_> {\n type Item = (usize, usize);\n\n #[inline]\n fn next(&mut self) -> Option {\n if let Some((_, char_len)) = self.text.char_at(self.cur_pos) {\n let result = (self.cur_pos, char_len);\n self.cur_pos += char_len;\n return Some(result);\n }\n None\n }\n}\n\n/// Iterator over UTF-16 text in a [u16] slice, returning (index, char) tuple.\n#[derive(Debug)]\npub struct Utf16CharIndexIter<'text> {\n text: &'text [u16],\n cur_pos: usize,\n}\n\nimpl<'text> Utf16CharIndexIter<'text> {\n pub fn new(text: &'text [u16]) -> Self {\n Utf16CharIndexIter { text, cur_pos: 0 }\n }\n}\n\nimpl Iterator for Utf16CharIndexIter<'_> {\n type Item = (usize, char);\n\n fn next(&mut self) -> Option {\n if let Some((ch, char_len)) = self.text.char_at(self.cur_pos) {\n let result = (self.cur_pos, ch);\n self.cur_pos += char_len;\n return Some(result);\n }\n None\n }\n}\n\n/// Iterator over UTF-16 text in a [u16] slice, returning Unicode chars.\n/// (Unlike the other iterators above, this also supports reverse iteration.)\n#[derive(Debug)]\npub struct Utf16CharIter<'text> {\n text: &'text [u16],\n cur_pos: usize,\n end_pos: usize,\n}\n\nimpl<'text> Utf16CharIter<'text> {\n pub fn new(text: &'text [u16]) -> Self {\n Utf16CharIter {\n text,\n cur_pos: 0,\n end_pos: text.len(),\n }\n }\n}\n\nimpl Iterator for Utf16CharIter<'_> {\n type Item = char;\n\n fn next(&mut self) -> Option {\n if let Some((ch, char_len)) = self.text.char_at(self.cur_pos) {\n self.cur_pos += char_len;\n return Some(ch);\n }\n None\n }\n}\n\nimpl DoubleEndedIterator for Utf16CharIter<'_> {\n fn next_back(&mut self) -> Option {\n if self.end_pos <= self.cur_pos {\n return None;\n }\n self.end_pos -= 1;\n if let Some(ch) = char::from_u32(self.text[self.end_pos] as u32) {\n return Some(ch);\n }\n if self.end_pos > self.cur_pos {\n if let Some((ch, char_len)) = self.text.char_at(self.end_pos - 1) {\n if char_len == 2 {\n self.end_pos -= 1;\n return Some(ch);\n }\n }\n }\n Some(char::REPLACEMENT_CHARACTER)\n }\n}\n" }, { "package": "unicode-bidi-mirroring@0.1.0", "sourcePath": "README.md", "sha256": "b184c317d71abff063675c1a7170f155c18b37663644afe77e7dfee7f57b5cc0", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "## unicode-bidi-mirroring\n[![Crates.io](https://img.shields.io/crates/v/unicode-bidi-mirroring.svg)](https://crates.io/crates/unicode-bidi-mirroring)\n[![Documentation](https://docs.rs/unicode-bidi-mirroring/badge.svg)](https://docs.rs/unicode-bidi-mirroring)\n\nThis library implements\n[Unicode Bidi Mirroring](https://unicode.org/reports/tr44/#BidiMirroring.txt) property detection.\n\n```rust\nuse unicode_bidi_mirroring::*;\n\nassert_eq!(get_mirrored('A'), None);\nassert_eq!(get_mirrored('\\u{2039}'), Some('\\u{203A}'));\nassert_eq!(get_mirrored('\\u{203A}'), Some('\\u{2039}'));\n\nassert_eq!(is_mirroring('A'), false);\nassert_eq!(is_mirroring('\\u{29C4}'), true);\nassert_eq!(is_mirroring('\\u{22FF}'), true);\n```\n\n### License\n\nLicensed under either of\n\n- Apache License, Version 2.0\n ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n- MIT license\n ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "package": "unicode-ccc@0.1.2", "sourcePath": "README.md", "sha256": "dd9213583228412ac610e974abd07208935a797903fc7b829e91f7cabd4ddec3", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "## unicode-ccc\n[![Crates.io](https://img.shields.io/crates/v/unicode-ccc.svg)](https://crates.io/crates/unicode-ccc)\n[![Documentation](https://docs.rs/unicode-ccc/badge.svg)](https://docs.rs/unicode-ccc)\n\nThis library implements\n[Unicode Canonical Combining Class](https://unicode.org/reports/tr44/#Canonical_Combining_Class_Values) detection.\n\n```rust\nuse unicode_ccc::*;\n\nassert_eq!(get_canonical_combining_class('A'), CanonicalCombiningClass::NotReordered);\nassert_eq!(get_canonical_combining_class('\\u{0A3C}'), CanonicalCombiningClass::Nukta);\nassert_eq!(get_canonical_combining_class('\\u{18A9}'), CanonicalCombiningClass::AboveLeft);\n```\n\n### License\n\nLicensed under either of\n\n- Apache License, Version 2.0\n ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)\n- MIT license\n ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)\n\nat your option.\n\n### Contribution\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in the work by you, as defined in the Apache-2.0 license, shall be\ndual licensed as above, without any additional terms or conditions.\n" }, { "package": "unicode-ident@1.0.26", "sourcePath": "README.md", "sha256": "085da9383f04b605f1c8726dbbc2dfb134e2c0ab662f3c40c6b78be858ab651e", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "Unicode ident\n=============\n\n[\"github\"](https://github.com/dtolnay/unicode-ident)\n[\"crates.io\"](https://crates.io/crates/unicode-ident)\n[\"docs.rs\"](https://docs.rs/unicode-ident)\n[\"build](https://github.com/dtolnay/unicode-ident/actions?query=branch%3Amaster)\n\nImplementation of [Unicode Standard Annex #31][tr31] for determining which\n`char` values are valid in programming language identifiers.\n\n[tr31]: https://www.unicode.org/reports/tr31/\n\nThis crate is a better optimized implementation of the older `unicode-xid`\ncrate. This crate uses less static storage, and is able to classify both ASCII\nand non-ASCII codepoints with better performance, 6× faster than\n`unicode-xid`.\n\n
\n\n## Comparison of performance\n\nThe following table shows a comparison between five Unicode identifier\nimplementations.\n\n- `unicode-ident` is this crate;\n- [`unicode-xid`] is a widely used crate run by the \"unicode-rs\" org;\n- `ucd-trie` and `fst` are two data structures supported by the [`ucd-generate`] tool;\n- [`roaring`] is a Rust implementation of Roaring bitmap.\n\nThe *static storage* column shows the total size of `static` tables that the\ncrate bakes into your binary, measured in 1000s of bytes.\n\nThe remaining columns show the **cost per call** to evaluate whether a single\n`char` has the XID\\_Start or XID\\_Continue Unicode property, comparing across\ndifferent ratios of ASCII to non-ASCII codepoints in the input data.\n\n[`unicode-xid`]: https://github.com/unicode-rs/unicode-xid\n[`ucd-generate`]: https://github.com/BurntSushi/ucd-generate\n[`roaring`]: https://github.com/RoaringBitmap/roaring-rs\n\n| | static storage | 0% nonascii | 1% | 10% | 100% nonascii |\n|---|---|---|---|---|---|\n| **`unicode-ident`** | 10.3 K | 0.36 ns | 0.37 ns | 0.37 ns | 0.43 ns |\n| **`unicode-xid`** | 12.2 K | 1.63 ns | 1.70 ns | 1.82 ns | 4.56 ns |\n| **`ucd-trie`** | 10.8 K | 1.01 ns | 0.73 ns | 0.97 ns | 1.09 ns |\n| **`fst`** | 149 K | 22.0 ns | 21.9 ns | 20.9 ns | 10.5 ns |\n| **`roaring`** | 66.1 K | 1.91 ns | 1.90 ns | 1.94 ns | 2.67 ns |\n\nSource code for the benchmark is provided in the *bench* directory of this repo\nand may be repeated by running `cargo criterion`.\n\n
\n\n## Comparison of data structures\n\n#### unicode-xid\n\nThey use a sorted array of character ranges, and do a binary search to look up\nwhether a given character lands inside one of those ranges.\n\n```rust\nstatic XID_Continue_table: [(char, char); 763] = [\n ('\\u{30}', '\\u{39}'), // 0-9\n ('\\u{41}', '\\u{5a}'), // A-Z\n …\n ('\\u{e0100}', '\\u{e01ef}'),\n];\n```\n\nThe static storage used by this data structure scales with the number of\ncontiguous ranges of identifier codepoints in Unicode. Every table entry\nconsumes 8 bytes, because it consists of a pair of 32-bit `char` values.\n\nIn some ranges of the Unicode codepoint space, this is quite a sparse\nrepresentation – there are some ranges where tens of thousands of adjacent\ncodepoints are all valid identifier characters. In other places, the\nrepresentation is quite inefficient. A characater like `µ` (U+00B5) which is\nsurrounded by non-identifier codepoints consumes 64 bits in the table, while it\nwould be just 1 bit in a dense bitmap.\n\nOn a system with 64-byte cache lines, binary searching the table touches 7 cache\nlines on average. Each cache line fits only 8 table entries. Additionally, the\nbranching performed during the binary search is probably mostly unpredictable to\nthe branch predictor.\n\nOverall, the crate ends up being about 6× slower on non-ASCII input\ncompared to the fastest crate.\n\nA potential improvement would be to pack the table entries more compactly.\nRust's `char` type is a 21-bit integer padded to 32 bits, which means every\ntable entry is holding 22 bits of wasted space, adding up to 3.9 K. They could\ninstead fit every table entry into 6 bytes, leaving out some of the padding, for\na 25% improvement in space used. With some cleverness it may be possible to fit\nin 5 bytes or even 4 bytes by storing a low char and an extent, instead of low\nchar and high char. I don't expect that performance would improve much but this\ncould be the most efficient for space across all the libraries, needing only\nabout 7 K to store.\n\n#### ucd-trie\n\nTheir data structure is a compressed trie set specifically tailored for Unicode\ncodepoints. The design is credited to Raph Levien in [rust-lang/rust#33098].\n\n[rust-lang/rust#33098]: https://github.com/rust-lang/rust/pull/33098\n\n```rust\npub struct TrieSet {\n tree1_level1: &'static [u64; 32],\n tree2_level1: &'static [u8; 992],\n tree2_level2: &'static [u64],\n tree3_level1: &'static [u8; 256],\n tree3_level2: &'static [u8],\n tree3_level3: &'static [u64],\n}\n```\n\nIt represents codepoint sets using a trie to achieve prefix compression. The\nfinal states of the trie are embedded in leaves or \"chunks\", where each chunk is\na 64-bit integer. Each bit position of the integer corresponds to whether a\nparticular codepoint is in the set or not. These chunks are not just a compact\nrepresentation of the final states of the trie, but are also a form of suffix\ncompression. In particular, if multiple ranges of 64 contiguous codepoints have\nthe same Unicode properties, then they all map to the same chunk in the final\nlevel of the trie.\n\nBeing tailored for Unicode codepoints, this trie is partitioned into three\ndisjoint sets: tree1, tree2, tree3. The first set corresponds to codepoints \\[0,\n0x800), the second \\[0x800, 0x10000) and the third \\[0x10000, 0x110000). These\npartitions conveniently correspond to the space of 1 or 2 byte UTF-8 encoded\ncodepoints, 3 byte UTF-8 encoded codepoints and 4 byte UTF-8 encoded codepoints,\nrespectively.\n\nLookups in this data structure are significantly more efficient than binary\nsearch. A lookup touches either 1, 2, or 3 cache lines based on which of the\ntrie partitions is being accessed.\n\nOne possible performance improvement would be for this crate to expose a way to\nquery based on a UTF-8 encoded string, returning the Unicode property\ncorresponding to the first character in the string. Without such an API, the\ncaller is required to tokenize their UTF-8 encoded input data into `char`, hand\nthe `char` into `ucd-trie`, only for `ucd-trie` to undo that work by converting\nback into the variable-length representation for trie traversal.\n\n#### fst\n\nUses a [finite state transducer][fst]. This representation is built into\n[ucd-generate] but I am not aware of any advantage over the `ucd-trie`\nrepresentation. In particular `ucd-trie` is optimized for storing Unicode\nproperties while `fst` is not.\n\n[fst]: https://github.com/BurntSushi/fst\n[ucd-generate]: https://github.com/BurntSushi/ucd-generate\n\nAs far as I can tell, the main thing that causes `fst` to have large size and\nslow lookups for this use case relative to `ucd-trie` is that it does not\nspecialize for the fact that only 21 of the 32 bits in a `char` are meaningful.\nThere are some dense arrays in the structure with large ranges that could never\npossibly be used.\n\n#### roaring\n\nThis crate is a pure-Rust implementation of [Roaring Bitmap], a data structure\ndesigned for storing sets of 32-bit unsigned integers.\n\n[Roaring Bitmap]: https://roaringbitmap.org/about/\n\nRoaring bitmaps are compressed bitmaps which tend to outperform conventional\ncompressed bitmaps such as WAH, EWAH or Concise. In some instances, they can be\nhundreds of times faster and they often offer significantly better compression.\n\nIn this use case the performance was reasonably competitive but still\nsubstantially slower than the Unicode-optimized crates. Meanwhile the\ncompression was significantly worse, requiring 6× as much storage for the\ndata structure.\n\nI also benchmarked the [`croaring`] crate which is an FFI wrapper around the C\nreference implementation of Roaring Bitmap. This crate was consistently about\n15% slower than pure-Rust `roaring`, which could just be FFI overhead. I did not\ninvestigate further.\n\n[`croaring`]: https://crates.io/crates/croaring\n\n#### unicode-ident\n\nThis crate is most similar to the `ucd-trie` library, in that it's based on\nbitmaps stored in the leafs of a trie representation, achieving both prefix\ncompression and suffix compression.\n\nThe key differences are:\n\n- Uses a single 2-level trie, rather than 3 disjoint partitions of different\n depth each.\n- Uses significantly larger chunks: 512 bits rather than 64 bits.\n- Compresses the XID\\_Start and XID\\_Continue properties together\n simultaneously, rather than duplicating identical trie leaf chunks across the\n two.\n\nThe following diagram show the XID\\_Start and XID\\_Continue Unicode boolean\nproperties in uncompressed form, in row-major order:\n\n\n\n\n\n\n\n
XID_StartXID_Continue
\"XID_Start\"XID_Continue
\n\nUncompressed, these would take 140 K to store, which is beyond what would be\nreasonable. However, as you can see there is a large degree of similarity\nbetween the two bitmaps and across the rows, which lends well to compression.\n\nThis crate stores one 512-bit \"row\" of the above bitmaps in the leaf level of a\ntrie, and a single additional level to index into the leafs. There are 134\nunique 512-bit chunks across the two bitmaps.\n\nThe chunk size of 512 bits is selected as the size that minimizes the total size\nof the data structure. A smaller chunk, like 256 or 128 bits, would achieve\nbetter deduplication but require a larger index. A larger chunk would increase\nredundancy in the leaf bitmaps. 512 bit chunks are the optimum for total size of\nthe index plus leaf bitmaps.\n\nThe chunk data is compressed using the Kuhn–Munkres algorithm for bipartite\nmatching to eliminate redundancies between the second half of any chunk and the\nfirst half of any other chunk. This achieves an additional 9% compression of the\nleaf level, leaving 122 chunks that can be indexed at the half-chunk level using\nan 8-bit index. Note that this is not the same as using chunks which are half\nthe size, because it does not necessitate raising the size of the trie's first\nlevel.\n\nIn contrast to binary search or the `ucd-trie` crate, performing lookups in this\ndata structure is straight-line code with no need for branching.\n\n```asm\nis_xid_start:\n\tmov eax, edi\n\tshr eax, 9\n\tcmp edi, 261632\n\tmov ecx, offset unicode_ident::tables::TRIE_START+17\n\tlea rax, [rax + unicode_ident::tables::TRIE_START]\n\tcmovb rcx, rax\n\tmovzx eax, byte ptr [rcx]\n\tmov ecx, 1539\n\tbextr ecx, edi, ecx\n\tand edi, 7\n\tshl eax, 5\n\tmovzx eax, byte ptr [rax + rcx + unicode_ident::tables::LEAF]\n\tbt eax, edi\n\tsetb al\n\tret\n```\n\n
\n\n## License\n\nUse of the Unicode Character Database, as this crate does, is governed by the Unicode license.\n\nAll intellectual property within this crate that is **not generated** using the\nUnicode Character Database as input is licensed under either of Apache License, Version 2.0 or MIT license at your option.\n\nThe **generated** files incorporate tabular data derived from the Unicode\nCharacter Database, together with intellectual property from the original source\ncode content of the crate. One must comply with the terms of both the Unicode\nLicense Agreement and either of the Apache license or MIT license when those\ngenerated files are involved.\n\nUnless you explicitly state otherwise, any contribution intentionally submitted\nfor inclusion in this crate by you, as defined in the Apache-2.0 license, shall\nbe licensed as just described, without any additional terms or conditions.\n" }, { "package": "unicode-script@0.5.8", "sourcePath": "scripts/unicode.py", "sha256": "72f5fed57ffcbff045013c8101e875596ce8eeedde2c979f7cb904c315f3c32f", "preservation": "Whole source text retained to preserve comment blocks without rewriting or inferring ownership.", "text": "#!/usr/bin/env python\n#\n# Copyright 2011-2015 The Rust Project Developers. See the COPYRIGHT\n# file at the top-level directory of this distribution and at\n# http://rust-lang.org/COPYRIGHT.\n#\n# Licensed under the Apache License, Version 2.0 or the MIT license\n# , at your\n# option. This file may not be copied, modified, or distributed\n# except according to those terms.\n\n# This script uses the following Unicode tables:\n# - PropertyValueAliases.txt\n# - ScriptExtensions.txt\n# - Scripts.txt\n#\n# Since this should not require frequent updates, we just store this\n# out-of-line and check the unicode.rs file into git.\n\nimport fileinput, re, os, sys\n\npreamble = '''// Copyright 2012-2018 The Rust Project Developers. See the COPYRIGHT\n// file at the top-level directory of this distribution and at\n// http://rust-lang.org/COPYRIGHT.\n//\n// Licensed under the Apache License, Version 2.0 or the MIT license\n// , at your\n// option. This file may not be copied, modified, or distributed\n// except according to those terms.\n\n// NOTE: The following code was generated by \"scripts/unicode.py\", do not edit directly\n\n#![allow(missing_docs, non_upper_case_globals, non_snake_case)]\n\npub use tables_impl::*;\n\n#[rustfmt::skip]\nmod tables_impl {\nuse crate::ScriptExtension;\n'''\n\n# Close `mod impl {`\nending='''\n}\n'''\n\nUNICODE_VERSION = (17, 0, 0)\n\nUNICODE_VERSION_NUMBER = \"%s.%s.%s\" %UNICODE_VERSION\n\ndef escape_char(c):\n return \"'\\\\u{%x}'\" % c\n\ndef fetch(f):\n if not os.path.exists(os.path.basename(f)):\n if \"emoji\" in f:\n os.system(\"curl -O https://www.unicode.org/Public/emoji/%s.%s/%s\"\n % (UNICODE_VERSION[0], UNICODE_VERSION[1], f))\n else:\n os.system(\"curl -O https://www.unicode.org/Public/%s/ucd/%s\"\n % (UNICODE_VERSION_NUMBER, f))\n\n if not os.path.exists(os.path.basename(f)):\n sys.stderr.write(\"cannot load %s\" % f)\n exit(1)\n\ndef group_cats(cats):\n cats_out = {}\n for cat in cats:\n cats_out[cat] = group_cat(cats[cat])\n return cats_out\n\ndef aliases():\n \"\"\"\n Fetch the shorthand aliases for each longhand Script name\n \"\"\"\n fetch(\"PropertyValueAliases.txt\")\n longforms = {}\n shortforms = {}\n re1 = re.compile(r\"^ *sc *; *(\\w+) *; *(\\w+)\")\n for line in fileinput.input(os.path.basename(\"PropertyValueAliases.txt\")):\n m = re1.match(line)\n if m:\n l = m.group(2).strip()\n s = m.group(1).strip()\n assert(s not in longforms)\n assert(l not in shortforms)\n longforms[s] = l\n shortforms[l] = s\n else:\n continue\n\n return (longforms, shortforms)\n\ndef format_table_content(f, content, indent):\n line = \" \"*indent\n first = True\n for chunk in content.split(\",\"):\n if len(line) + len(chunk) < 98:\n if first:\n line += chunk\n else:\n line += \", \" + chunk\n first = False\n else:\n f.write(line + \",\\n\")\n line = \" \"*indent + chunk\n f.write(line)\n\n# Implementation from unicode-segmentation\ndef load_properties(f, interestingprops):\n fetch(f)\n props = {}\n # Note: these regexes are different from those in unicode-segmentation,\n # becase we need to handle spaces here\n re1 = re.compile(r\"^ *([0-9A-F]+) *; *([^#]+) *#\")\n re2 = re.compile(r\"^ *([0-9A-F]+)\\.\\.([0-9A-F]+) *; *([^#]+) *#\")\n\n for line in fileinput.input(os.path.basename(f)):\n prop = None\n d_lo = 0\n d_hi = 0\n m = re1.match(line)\n if m:\n d_lo = m.group(1)\n d_hi = m.group(1)\n prop = m.group(2).strip()\n else:\n m = re2.match(line)\n if m:\n d_lo = m.group(1)\n d_hi = m.group(2)\n prop = m.group(3).strip()\n else:\n continue\n if interestingprops and prop not in interestingprops:\n continue\n d_lo = int(d_lo, 16)\n d_hi = int(d_hi, 16)\n if prop not in props:\n props[prop] = []\n props[prop].append((d_lo, d_hi))\n\n return props\n\n# Implementation from unicode-segmentation\ndef emit_table(f, name, t_data, t_type = \"&'static [(char, char)]\", is_pub=True,\n pfun=lambda x: \"(%s,%s)\" % (escape_char(x[0]), escape_char(x[1])), is_const=True):\n pub_string = \"const\"\n if not is_const:\n pub_string = \"let\"\n if is_pub:\n pub_string = \"pub \" + pub_string\n f.write(\" %s %s: %s = &[\\n\" % (pub_string, name, t_type))\n data = \"\"\n first = True\n for dat in t_data:\n if not first:\n data += \",\"\n first = False\n data += pfun(dat)\n format_table_content(f, data, 8)\n f.write(\"\\n ];\\n\\n\")\n\ndef emit_search(f):\n f.write(\"\"\"\npub fn bsearch_range_value_table(c: char, r: &'static [(char, char, T)]) -> Option {\n use core::cmp::Ordering::{Equal, Less, Greater};\n match r.binary_search_by(|&(lo, hi, _)| {\n if lo <= c && c <= hi { Equal }\n else if hi < c { Less }\n else { Greater }\n }) {\n Ok(idx) => {\n let (_, _, cat) = r[idx];\n Some(cat)\n }\n Err(_) => None\n }\n}\n\n#[inline]\npub fn get_script(c: char) -> Option